1551 files changed, 95715 insertions, 30724 deletions

diff --git a/src/app/app.md b/src/app/app.md
new file mode 100644
index 0000000000..138e75b127
--- /dev/null
+++ b/src/app/app.md
@@ -0,0 +1,6 @@
+@dir /app
+@brief app: top-level entry point for Tor
+
+The "app" directory has Tor's main entry point and configuration logic,
+and is responsible for initializing and managing the other modules in
+Tor.
diff --git a/src/app/config/.may_include b/src/app/config/.may_include
new file mode 100644
index 0000000000..11c5ffbb14
--- /dev/null
+++ b/src/app/config/.may_include
@@ -0,0 +1,2 @@
+*.h
+*.inc
diff --git a/src/app/config/app_config.md b/src/app/config/app_config.md
new file mode 100644
index 0000000000..b359ce77f6
--- /dev/null
+++ b/src/app/config/app_config.md
@@ -0,0 +1,6 @@
+@dir /app/config
+@brief app/config: Top-level configuration code
+
+Refactoring this module is a work in progress, see
+[ticket 29211](https://trac.torproject.org/projects/tor/ticket/29211).
+
diff --git a/src/app/config/config.c b/src/app/config/config.c
index c1f1f153e7..a0c188adc4 100644
--- a/src/app/config/config.c
+++ b/src/app/config/config.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,8 @@
  *
  * To add new items to the torrc, there are a minimum of three places to edit:
  * <ul>
- *   <li>The or_options_t structure in or.h, where the options are stored.
+ *   <li>The or_options_t structure in or_options_st.h, where the options are
+ *       stored.
  *   <li>The option_vars_ array below in this module, which configures
  *       the names of the torrc options, their types, their multiplicities,
  *       and their mappings to fields in or_options_t.
@@ -32,16 +33,18 @@
  *
  * Additionally, you might need to edit these places too:
  * <ul>
- *   <li>options_validate() below, in case you want to reject some possible
+ *   <li>options_validate_cb() below, in case you want to reject some possible
  *       values of the new configuration option.
  *   <li>options_transition_allowed() below, in case you need to
  *       forbid some or all changes in the option while Tor is
  *       running.
  *   <li>options_transition_affects_workers(), in case changes in the option
  *       might require Tor to relaunch or reconfigure its worker threads.
+ *       (This function is now in the relay module.)
  *   <li>options_transition_affects_descriptor(), in case changes in the
  *       option might require a Tor relay to build and publish a new server
  *       descriptor.
+ *       (This function is now in the relay module.)
  *   <li>options_act() and/or options_act_reversible(), in case there's some
  *       action that needs to be taken immediately based on the option's
  *       value.
@@ -61,21 +64,19 @@
 #define CONFIG_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "app/config/statefile.h"
 #include "app/main/main.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
-#include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
-#include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
 #include "core/or/connection_edge.h"
-#include "core/or/connection_or.h"
 #include "core/or/dos.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
@@ -85,11 +86,9 @@
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
-#include "feature/dirauth/bwauth.h"
-#include "feature/dirauth/guardfraction.h"
-#include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_config.h"
 #include "feature/nodelist/dirlist.h"
@@ -101,20 +100,21 @@
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
 #include "feature/relay/routermode.h"
+#include "feature/relay/relay_config.h"
+#include "feature/relay/transport_config.h"
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "lib/geoip/geoip.h"
 #include "feature/stats/geoip_stats.h"
-#include "feature/stats/predict_ports.h"
-#include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
+#include "lib/confmgt/structvar.h"
 #include "lib/crypt_ops/crypto_init.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/encoding/confline.h"
-#include "lib/log/git_revision.h"
 #include "lib/net/resolve.h"
 #include "lib/sandbox/sandbox.h"
+#include "lib/version/torversion.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
@@ -144,7 +144,7 @@
 #include "lib/process/pidfile.h"
 #include "lib/process/restrict.h"
 #include "lib/process/setuid.h"
-#include "lib/process/subprocess.h"
+#include "lib/process/process.h"
 #include "lib/net/gethostname.h"
 #include "lib/thread/numcpus.h"
 
@@ -152,9 +152,8 @@
 #include "lib/fs/conffile.h"
 #include "lib/evloop/procmon.h"
 
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/recommend_pkg.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_config.h"
 
 #include "core/or/connection_st.h"
 #include "core/or/port_cfg_st.h"
@@ -181,12 +180,16 @@ static const char unix_q_socket_prefix[] = "unix:\"";
 
 /** macro to help with the bulk rename of *DownloadSchedule to
  * *DowloadInitialDelay . */
+#ifndef COCCI
 #define DOWNLOAD_SCHEDULE(name) \
-  { #name "DownloadSchedule", #name "DownloadInitialDelay", 0, 1 }
+  { (#name "DownloadSchedule"), (#name "DownloadInitialDelay"), 0, 1 }
+#else
+#define DOWNLOAD_SCHEDULE(name) { NULL, NULL, 0, 1 }
+#endif /* !defined(COCCI) */
 
 /** A list of abbreviations and aliases to map command-line options, obsolete
  * option names, or alternative option names, to their current values. */
-static config_abbrev_t option_abbrevs_[] = {
+static const config_abbrev_t option_abbrevs_[] = {
   PLURAL(AuthDirBadDirCC),
   PLURAL(AuthDirBadExitCC),
   PLURAL(AuthDirInvalidCC),
@@ -249,22 +252,43 @@ static config_abbrev_t option_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_options_t);
 
-/** An entry for config_vars: "The option <b>name</b> has type
+/** An entry for config_vars: "The option <b>varname</b> has type
  * CONFIG_TYPE_<b>conftype</b>, and corresponds to
  * or_options_t.<b>member</b>"
  */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_options_t, member),     \
-      initvalue CONF_TEST_MEMBERS(or_options_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member, 0, initvalue)
+
+/* As VAR, but uses a type definition in addition to a type enum. */
+#define VAR_D(varname,conftype,member,initvalue)                        \
+  CONFIG_VAR_DEFN(or_options_t, varname, conftype, member, 0, initvalue)
+
+#define VAR_NODUMP(varname,conftype,member,initvalue)             \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP, initvalue)
+#define VAR_NODUMP_IMMUTABLE(varname,conftype,member,initvalue)   \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP | CFLG_IMMUTABLE, initvalue)
+#define VAR_INVIS(varname,conftype,member,initvalue)              \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,       \
+                   CFLG_NODUMP | CFLG_NOSET | CFLG_NOLIST, initvalue)
+
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
-/** An entry for config_vars: "The option <b>name</b> is obsolete." */
-#ifdef TOR_UNIT_TESTS
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL, {.INT=NULL} }
-#else
-#define OBSOLETE(name) { name, CONFIG_TYPE_OBSOLETE, 0, NULL }
-#endif
+
+#define VAR_IMMUTABLE(varname, conftype, member, initvalue)             \
+  CONFIG_VAR_ETYPE(or_options_t, varname, conftype, member,             \
+                   CFLG_IMMUTABLE, initvalue)
+
+#define V_IMMUTABLE(member,conftype,initvalue)                 \
+  VAR_IMMUTABLE(#member, conftype, member, initvalue)
+
+/** As V, but uses a type definition instead of a type enum */
+#define V_D(member,type,initvalue)              \
+  VAR_D(#member, type, member, initvalue)
+
+/** An entry for config_vars: "The option <b>varname</b> is obsolete." */
+#define OBSOLETE(varname) CONFIG_VAR_OBSOLETE(varname)
 
 /**
  * Macro to declare *Port options.  Each one comes in three entries.
@@ -276,7 +300,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
 #define VPORT(member)                                           \
   VAR(#member "Lines", LINELIST_V, member ## _lines, NULL),     \
   VAR(#member, LINELIST_S, member ## _lines, NULL),             \
-  VAR("__" #member, LINELIST_S, member ## _lines, NULL)
+  VAR_NODUMP("__" #member, LINELIST_S, member ## _lines, NULL)
 
 /** UINT64_MAX as a decimal string */
 #define UINT64_MAX_STRING "18446744073709551615"
@@ -285,7 +309,7 @@ DUMMY_TYPECHECK_INSTANCE(or_options_t);
  * abbreviations, order is significant, since the first matching option will
  * be chosen first.
  */
-static config_var_t option_vars_[] = {
+static const config_var_t option_vars_[] = {
   V(AccountingMax,               MEMUNIT,  "0 bytes"),
   VAR("AccountingRule",          STRING,   AccountingRule_option,  "max"),
   V(AccountingStart,             STRING,   NULL),
@@ -305,17 +329,11 @@ static config_var_t option_vars_[] = {
   V(AuthDirBadExitCCs,           CSV,      ""),
   V(AuthDirInvalid,              LINELIST, NULL),
   V(AuthDirInvalidCCs,           CSV,      ""),
-  V(AuthDirFastGuarantee,        MEMUNIT,  "100 KB"),
-  V(AuthDirGuardBWGuarantee,     MEMUNIT,  "2 MB"),
-  V(AuthDirPinKeys,              BOOL,     "1"),
   V(AuthDirReject,               LINELIST, NULL),
   V(AuthDirRejectCCs,            CSV,      ""),
   OBSOLETE("AuthDirRejectUnlisted"),
   OBSOLETE("AuthDirListBadDirs"),
-  V(AuthDirListBadExits,         BOOL,     "0"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "2"),
   OBSOLETE("AuthDirMaxServersPerAuthAddr"),
-  V(AuthDirHasIPv6Connectivity,  BOOL,     "0"),
   VAR("AuthoritativeDirectory",  BOOL, AuthoritativeDir,    "0"),
   V(AutomapHostsOnResolve,       BOOL,     "0"),
   V(AutomapHostsSuffixes,        CSV,      ".onion,.exit"),
@@ -328,7 +346,7 @@ static config_var_t option_vars_[] = {
   V(BridgeRecordUsageByCountry,  BOOL,     "1"),
   V(BridgeRelay,                 BOOL,     "0"),
   V(BridgeDistribution,          STRING,   NULL),
-  VAR("CacheDirectory",          FILENAME, CacheDirectory_option, NULL),
+  VAR_IMMUTABLE("CacheDirectory",FILENAME, CacheDirectory_option, NULL),
   V(CacheDirectoryGroupReadable, AUTOBOOL,     "auto"),
   V(CellStatistics,              BOOL,     "0"),
   V(PaddingStatistics,           BOOL,     "1"),
@@ -339,15 +357,21 @@ static config_var_t option_vars_[] = {
   V(CircuitStreamTimeout,        INTERVAL, "0"),
   V(CircuitPriorityHalflife,     DOUBLE,  "-1.0"), /*negative:'Use default'*/
   V(ClientDNSRejectInternalAddresses, BOOL,"1"),
+#if defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS)
+  /* The unit tests expect the ClientOnly default to be 0. */
   V(ClientOnly,                  BOOL,     "0"),
+#else
+  /* We must be a Client if the relay module is disabled. */
+  V(ClientOnly,                  BOOL,     "1"),
+#endif /* defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS) */
   V(ClientPreferIPv6ORPort,      AUTOBOOL, "auto"),
   V(ClientPreferIPv6DirPort,     AUTOBOOL, "auto"),
+  OBSOLETE("ClientAutoIPv6ORPort"),
   V(ClientRejectInternalAddresses, BOOL,   "1"),
   V(ClientTransportPlugin,       LINELIST, NULL),
   V(ClientUseIPv6,               BOOL,     "0"),
   V(ClientUseIPv4,               BOOL,     "1"),
-  V(ConsensusParams,             STRING,   NULL),
-  V(ConnLimit,                   UINT,     "1000"),
+  V(ConnLimit,                   POSINT,     "1000"),
   V(ConnDirectionStatistics,     BOOL,     "0"),
   V(ConstrainedSockets,          BOOL,     "0"),
   V(ConstrainedSockSize,         MEMUNIT,  "8192"),
@@ -361,21 +385,26 @@ static config_var_t option_vars_[] = {
   V(UnixSocksGroupWritable,    BOOL,     "0"),
   V(CookieAuthentication,        BOOL,     "0"),
   V(CookieAuthFileGroupReadable, BOOL,     "0"),
-  V(CookieAuthFile,              STRING,   NULL),
+  V(CookieAuthFile,              FILENAME, NULL),
   V(CountPrivateBandwidth,       BOOL,     "0"),
-  VAR("DataDirectory",           FILENAME, DataDirectory_option, NULL),
+  VAR_IMMUTABLE("DataDirectory", FILENAME, DataDirectory_option, NULL),
   V(DataDirectoryGroupReadable,  BOOL,     "0"),
   V(DisableOOSCheck,             BOOL,     "1"),
   V(DisableNetwork,              BOOL,     "0"),
   V(DirAllowPrivateAddresses,    BOOL,     "0"),
-  V(TestingAuthDirTimeToLearnReachability, INTERVAL, "30 minutes"),
   OBSOLETE("DirListenAddress"),
   V(DirPolicy,                   LINELIST, NULL),
   VPORT(DirPort),
   V(DirPortFrontPage,            FILENAME, NULL),
   VAR("DirReqStatistics",        BOOL,     DirReqStatistics_option, "1"),
   VAR("DirAuthority",            LINELIST, DirAuthorities, NULL),
+#if defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS)
+  /* The unit tests expect the DirCache default to be 1. */
   V(DirCache,                    BOOL,     "1"),
+#else
+  /* We can't be a DirCache if the relay module is disabled. */
+  V(DirCache,                    BOOL,     "0"),
+#endif /* defined(HAVE_MODULE_RELAY) || defined(TOR_UNIT_TESTS) */
   /* A DirAuthorityFallbackRate of 0.1 means that 0.5% of clients try an
    * authority when all fallbacks are up, and 2% try an authority when 25% of
    * fallbacks are down. (We rebuild the list when 25% of fallbacks are down).
@@ -384,23 +413,27 @@ static config_var_t option_vars_[] = {
    * an order of magnitude, so there isn't too much load shifting to
    * authorities when fallbacks go down. */
   V(DirAuthorityFallbackRate,    DOUBLE,   "0.1"),
-  V(DisableAllSwap,              BOOL,     "0"),
-  V(DisableDebuggerAttachment,   BOOL,     "1"),
+  V_IMMUTABLE(DisableAllSwap,    BOOL,     "0"),
+  V_IMMUTABLE(DisableDebuggerAttachment,   BOOL,     "1"),
   OBSOLETE("DisableIOCP"),
   OBSOLETE("DisableV2DirectoryInfo_"),
   OBSOLETE("DynamicDHGroups"),
   VPORT(DNSPort),
   OBSOLETE("DNSListenAddress"),
+  V(DormantClientTimeout,         INTERVAL, "24 hours"),
+  V(DormantTimeoutDisabledByIdleStreams, BOOL,     "1"),
+  V(DormantOnFirstStartup,       BOOL,      "0"),
+  V(DormantCanceledByStartup,    BOOL,      "0"),
   /* DoS circuit creation options. */
   V(DoSCircuitCreationEnabled,   AUTOBOOL, "auto"),
-  V(DoSCircuitCreationMinConnections,      UINT, "0"),
-  V(DoSCircuitCreationRate,      UINT,     "0"),
-  V(DoSCircuitCreationBurst,     UINT,     "0"),
+  V(DoSCircuitCreationMinConnections,      POSINT, "0"),
+  V(DoSCircuitCreationRate,      POSINT,     "0"),
+  V(DoSCircuitCreationBurst,     POSINT,     "0"),
   V(DoSCircuitCreationDefenseType,         INT,  "0"),
   V(DoSCircuitCreationDefenseTimePeriod,   INTERVAL, "0"),
   /* DoS connection options. */
   V(DoSConnectionEnabled,        AUTOBOOL, "auto"),
-  V(DoSConnectionMaxConcurrentCount,       UINT, "0"),
+  V(DoSConnectionMaxConcurrentCount,       POSINT, "0"),
   V(DoSConnectionDefenseType,    INT,      "0"),
   /* DoS single hop client options. */
   V(DoSRefuseSingleHopClientRendezvous,    AUTOBOOL, "auto"),
@@ -409,13 +442,17 @@ static config_var_t option_vars_[] = {
   V(TestingEnableCellStatsEvent, BOOL,     "0"),
   OBSOLETE("TestingEnableTbEmptyEvent"),
   V(EnforceDistinctSubnets,      BOOL,     "1"),
-  V(EntryNodes,                  ROUTERSET,   NULL),
+  V_D(EntryNodes,                ROUTERSET,   NULL),
   V(EntryStatistics,             BOOL,     "0"),
-  V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "10 minutes"),
-  V(ExcludeNodes,                ROUTERSET, NULL),
-  V(ExcludeExitNodes,            ROUTERSET, NULL),
+  OBSOLETE("TestingEstimatedDescriptorPropagationTime"),
+  V_D(ExcludeNodes,              ROUTERSET, NULL),
+  V_D(ExcludeExitNodes,          ROUTERSET, NULL),
   OBSOLETE("ExcludeSingleHopRelays"),
-  V(ExitNodes,                   ROUTERSET, NULL),
+  V_D(ExitNodes,                 ROUTERSET, NULL),
+  /* Researchers need a way to tell their clients to use specific
+   * middles that they also control, to allow safe live-network
+   * experimentation with new padding machines. */
+  V_D(MiddleNodes,               ROUTERSET, NULL),
   V(ExitPolicy,                  LINELIST, NULL),
   V(ExitPolicyRejectPrivate,     BOOL,     "1"),
   V(ExitPolicyRejectLocalInterfaces, BOOL, "0"),
@@ -423,7 +460,7 @@ static config_var_t option_vars_[] = {
   V(ExtendAllowPrivateAddresses, BOOL,     "0"),
   V(ExitRelay,                   AUTOBOOL, "auto"),
   VPORT(ExtORPort),
-  V(ExtORPortCookieAuthFile,     STRING,   NULL),
+  V(ExtORPortCookieAuthFile,     FILENAME,   NULL),
   V(ExtORPortCookieAuthFileGroupReadable, BOOL, "0"),
   V(ExtraInfoStatistics,         BOOL,     "1"),
   V(ExtendByEd25519ID,           AUTOBOOL, "auto"),
@@ -453,11 +490,8 @@ static config_var_t option_vars_[] = {
 #endif /* defined(_WIN32) */
   OBSOLETE("Group"),
   V(GuardLifetime,               INTERVAL, "0 minutes"),
-  V(HardwareAccel,               BOOL,     "0"),
   V(HeartbeatPeriod,             INTERVAL, "6 hours"),
   V(MainloopStats,               BOOL,     "0"),
-  V(AccelName,                   STRING,   NULL),
-  V(AccelDir,                    FILENAME, NULL),
   V(HashedControlPassword,       LINELIST, NULL),
   OBSOLETE("HidServDirectoryV2"),
   VAR("HiddenServiceDir",    LINELIST_S, RendConfigLines,    NULL),
@@ -471,13 +505,20 @@ static config_var_t option_vars_[] = {
   VAR("HiddenServiceMaxStreamsCloseCircuit",LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceNumIntroductionPoints", LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceExportCircuitID", LINELIST_S,  RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSDefense", LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSRatePerSec",
+      LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceEnableIntroDoSBurstPerSec",
+      LINELIST_S, RendConfigLines, NULL),
+  VAR("HiddenServiceOnionBalanceInstance",
+      LINELIST_S, RendConfigLines, NULL),
   VAR("HiddenServiceStatistics", BOOL, HiddenServiceStatistics_option, "1"),
   V(HidServAuth,                 LINELIST, NULL),
   V(ClientOnionAuthDir,          FILENAME, NULL),
   OBSOLETE("CloseHSClientCircuitsImmediatelyOnTimeout"),
   OBSOLETE("CloseHSServiceRendCircuitsImmediatelyOnTimeout"),
-  V(HiddenServiceSingleHopMode,  BOOL,     "0"),
-  V(HiddenServiceNonAnonymousMode,BOOL,    "0"),
+  V_IMMUTABLE(HiddenServiceSingleHopMode,  BOOL,     "0"),
+  V_IMMUTABLE(HiddenServiceNonAnonymousMode,BOOL,    "0"),
   V(HTTPProxy,                   STRING,   NULL),
   V(HTTPProxyAuthenticator,      STRING,   NULL),
   V(HTTPSProxy,                  STRING,   NULL),
@@ -492,30 +533,30 @@ static config_var_t option_vars_[] = {
   V(Socks5Proxy,                 STRING,   NULL),
   V(Socks5ProxyUsername,         STRING,   NULL),
   V(Socks5ProxyPassword,         STRING,   NULL),
-  VAR("KeyDirectory",            FILENAME, KeyDirectory_option, NULL),
-  V(KeyDirectoryGroupReadable,   BOOL,     "0"),
-  VAR("HSLayer2Nodes",           ROUTERSET,  HSLayer2Nodes,  NULL),
-  VAR("HSLayer3Nodes",           ROUTERSET,  HSLayer3Nodes,  NULL),
+  V(TCPProxy,                    STRING,   NULL),
+  VAR_IMMUTABLE("KeyDirectory",  FILENAME, KeyDirectory_option, NULL),
+  V(KeyDirectoryGroupReadable,   AUTOBOOL, "auto"),
+  VAR_D("HSLayer2Nodes",         ROUTERSET,  HSLayer2Nodes,  NULL),
+  VAR_D("HSLayer3Nodes",         ROUTERSET,  HSLayer3Nodes,  NULL),
   V(KeepalivePeriod,             INTERVAL, "5 minutes"),
-  V(KeepBindCapabilities,            AUTOBOOL, "auto"),
+  V_IMMUTABLE(KeepBindCapabilities,        AUTOBOOL, "auto"),
   VAR("Log",                     LINELIST, Logs,             NULL),
   V(LogMessageDomains,           BOOL,     "0"),
   V(LogTimeGranularity,          MSEC_INTERVAL, "1 second"),
   V(TruncateLogFile,             BOOL,     "0"),
-  V(SyslogIdentityTag,           STRING,   NULL),
-  V(AndroidIdentityTag,          STRING,   NULL),
+  V_IMMUTABLE(SyslogIdentityTag, STRING,   NULL),
+  V_IMMUTABLE(AndroidIdentityTag,STRING,   NULL),
   V(LongLivedPorts,              CSV,
         "21,22,706,1863,5050,5190,5222,5223,6523,6667,6697,8300"),
   VAR("MapAddress",              LINELIST, AddressMap,           NULL),
   V(MaxAdvertisedBandwidth,      MEMUNIT,  "1 GB"),
   V(MaxCircuitDirtiness,         INTERVAL, "10 minutes"),
-  V(MaxClientCircuitsPending,    UINT,     "32"),
+  V(MaxClientCircuitsPending,    POSINT,     "32"),
   V(MaxConsensusAgeForDiffs,     INTERVAL, "0 seconds"),
   VAR("MaxMemInQueues",          MEMUNIT,   MaxMemInQueues_raw, "0"),
   OBSOLETE("MaxOnionsPending"),
   V(MaxOnionQueueDelay,          MSEC_INTERVAL, "1750 msec"),
   V(MaxUnparseableDescSizeToLog, MEMUNIT, "10 MB"),
-  V(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, "500"),
   VAR("MyFamily",                LINELIST, MyFamily_lines,       NULL),
   V(NewCircuitPeriod,            INTERVAL, "30 seconds"),
   OBSOLETE("NamingAuthoritativeDirectory"),
@@ -525,11 +566,11 @@ static config_var_t option_vars_[] = {
   OBSOLETE("PredictedPortsRelevanceTime"),
   OBSOLETE("WarnUnsafeSocks"),
   VAR("NodeFamily",              LINELIST, NodeFamilies,         NULL),
-  V(NoExec,                      BOOL,     "0"),
-  V(NumCPUs,                     UINT,     "0"),
-  V(NumDirectoryGuards,          UINT,     "0"),
-  V(NumEntryGuards,              UINT,     "0"),
-  V(NumPrimaryGuards,            UINT,     "0"),
+  V_IMMUTABLE(NoExec,            BOOL,     "0"),
+  V(NumCPUs,                     POSINT,     "0"),
+  V(NumDirectoryGuards,          POSINT,     "0"),
+  V(NumEntryGuards,              POSINT,     "0"),
+  V(NumPrimaryGuards,            POSINT,     "0"),
   V(OfflineMasterKey,            BOOL,     "0"),
   OBSOLETE("ORListenAddress"),
   VPORT(ORPort),
@@ -556,10 +597,8 @@ static config_var_t option_vars_[] = {
   V(PathsNeededToBuildCircuits,  DOUBLE,   "-1"),
   V(PerConnBWBurst,              MEMUNIT,  "0"),
   V(PerConnBWRate,               MEMUNIT,  "0"),
-  V(PidFile,                     STRING,   NULL),
-  V(TestingTorNetwork,           BOOL,     "0"),
-  V(TestingMinExitFlagThreshold, MEMUNIT,  "0"),
-  V(TestingMinFastFlagThreshold, MEMUNIT,  "0"),
+  V_IMMUTABLE(PidFile,           FILENAME,   NULL),
+  V_IMMUTABLE(TestingTorNetwork, BOOL,     "0"),
 
   V(TestingLinkCertLifetime,          INTERVAL, "2 days"),
   V(TestingAuthKeyLifetime,          INTERVAL, "2 days"),
@@ -577,29 +616,28 @@ static config_var_t option_vars_[] = {
   V(ReachableAddresses,          LINELIST, NULL),
   V(ReachableDirAddresses,       LINELIST, NULL),
   V(ReachableORAddresses,        LINELIST, NULL),
-  V(RecommendedVersions,         LINELIST, NULL),
-  V(RecommendedClientVersions,   LINELIST, NULL),
-  V(RecommendedServerVersions,   LINELIST, NULL),
-  V(RecommendedPackages,         LINELIST, NULL),
+  OBSOLETE("RecommendedPackages"),
   V(ReducedConnectionPadding,    BOOL,     "0"),
   V(ConnectionPadding,           AUTOBOOL, "auto"),
   V(RefuseUnknownExits,          AUTOBOOL, "auto"),
+  V(CircuitPadding,              BOOL,     "1"),
+  V(ReducedCircuitPadding,       BOOL,     "0"),
   V(RejectPlaintextPorts,        CSV,      ""),
   V(RelayBandwidthBurst,         MEMUNIT,  "0"),
   V(RelayBandwidthRate,          MEMUNIT,  "0"),
   V(RendPostPeriod,              INTERVAL, "1 hour"),
   V(RephistTrackTime,            INTERVAL, "24 hours"),
-  V(RunAsDaemon,                 BOOL,     "0"),
+  V_IMMUTABLE(RunAsDaemon,       BOOL,     "0"),
   V(ReducedExitPolicy,           BOOL,     "0"),
   OBSOLETE("RunTesting"), // currently unused
-  V(Sandbox,                     BOOL,     "0"),
+  V_IMMUTABLE(Sandbox,           BOOL,     "0"),
   V(SafeLogging,                 STRING,   "1"),
   V(SafeSocks,                   BOOL,     "0"),
   V(ServerDNSAllowBrokenConfig,  BOOL,     "1"),
   V(ServerDNSAllowNonRFC953Hostnames, BOOL,"0"),
   V(ServerDNSDetectHijacking,    BOOL,     "1"),
   V(ServerDNSRandomizeCase,      BOOL,     "1"),
-  V(ServerDNSResolvConfFile,     STRING,   NULL),
+  V(ServerDNSResolvConfFile,     FILENAME,   NULL),
   V(ServerDNSSearchDomains,      BOOL,     "0"),
   V(ServerDNSTestAddresses,      CSV,
       "www.google.com,www.mit.edu,www.yahoo.com,www.slashdot.org"),
@@ -620,7 +658,7 @@ static config_var_t option_vars_[] = {
   V(StrictNodes,                 BOOL,     "0"),
   OBSOLETE("Support022HiddenServices"),
   V(TestSocks,                   BOOL,     "0"),
-  V(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
+  V_IMMUTABLE(TokenBucketRefillInterval,   MSEC_INTERVAL, "100 msec"),
   OBSOLETE("Tor2webMode"),
   OBSOLETE("Tor2webRendezvousPoints"),
   OBSOLETE("TLSECGroup"),
@@ -637,10 +675,8 @@ static config_var_t option_vars_[] = {
   V(UseGuardFraction,            AUTOBOOL, "auto"),
   V(UseMicrodescriptors,         AUTOBOOL, "auto"),
   OBSOLETE("UseNTorHandshake"),
-  V(User,                        STRING,   NULL),
+  V_IMMUTABLE(User,              STRING,   NULL),
   OBSOLETE("UserspaceIOCPBuffers"),
-  V(AuthDirSharedRandomness,     BOOL,     "1"),
-  V(AuthDirTestEd25519LinkKeys,  BOOL,     "1"),
   OBSOLETE("V1AuthoritativeDirectory"),
   OBSOLETE("V2AuthoritativeDirectory"),
   VAR("V3AuthoritativeDirectory",BOOL, V3AuthoritativeDir,   "0"),
@@ -651,27 +687,29 @@ static config_var_t option_vars_[] = {
   V(V3AuthVotingInterval,        INTERVAL, "1 hour"),
   V(V3AuthVoteDelay,             INTERVAL, "5 minutes"),
   V(V3AuthDistDelay,             INTERVAL, "5 minutes"),
-  V(V3AuthNIntervalsValid,       UINT,     "3"),
+  V(V3AuthNIntervalsValid,       POSINT,     "3"),
   V(V3AuthUseLegacyKey,          BOOL,     "0"),
   V(V3BandwidthsFile,            FILENAME, NULL),
   V(GuardfractionFile,           FILENAME, NULL),
-  VAR("VersioningAuthoritativeDirectory",BOOL,VersioningAuthoritativeDir, "0"),
   OBSOLETE("VoteOnHidServDirectoriesV2"),
   V(VirtualAddrNetworkIPv4,      STRING,   "127.192.0.0/10"),
   V(VirtualAddrNetworkIPv6,      STRING,   "[FE80::]/10"),
   V(WarnPlaintextPorts,          CSV,      "23,109,110,143"),
   OBSOLETE("UseFilteringSSLBufferevents"),
   OBSOLETE("__UseFilteringSSLBufferevents"),
-  VAR("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
-  VAR("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
-  VAR("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
-  VAR("__DisableSignalHandlers", BOOL,  DisableSignalHandlers,    "0"),
-  VAR("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
-  VAR("__HashedControlSessionPassword", LINELIST, HashedControlSessionPassword,
+  VAR_NODUMP("__ReloadTorrcOnSIGHUP",   BOOL,  ReloadTorrcOnSIGHUP,      "1"),
+  VAR_NODUMP("__AllDirActionsPrivate",  BOOL,  AllDirActionsPrivate,     "0"),
+  VAR_NODUMP("__DisablePredictedCircuits",BOOL,DisablePredictedCircuits, "0"),
+  VAR_NODUMP_IMMUTABLE("__DisableSignalHandlers", BOOL,
+                       DisableSignalHandlers,    "0"),
+  VAR_NODUMP("__LeaveStreamsUnattached",BOOL,  LeaveStreamsUnattached,   "0"),
+  VAR_NODUMP("__HashedControlSessionPassword", LINELIST,
+             HashedControlSessionPassword,
       NULL),
-  VAR("__OwningControllerProcess",STRING,OwningControllerProcess, NULL),
-  VAR("__OwningControllerFD", UINT64, OwningControllerFD, UINT64_MAX_STRING),
-  V(MinUptimeHidServDirectoryV2, INTERVAL, "96 hours"),
+  VAR_NODUMP("__OwningControllerProcess",STRING,
+                       OwningControllerProcess, NULL),
+  VAR_NODUMP_IMMUTABLE("__OwningControllerFD", UINT64, OwningControllerFD,
+             UINT64_MAX_STRING),
   V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
   V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
@@ -700,7 +738,7 @@ static config_var_t option_vars_[] = {
    * blocked), but we also don't want to fail if only some mirrors are
    * blackholed. Clients will try 3 directories simultaneously.
    * (Relays never use simultaneous connections.) */
-  V(ClientBootstrapConsensusMaxInProgressTries, UINT, "3"),
+  V(ClientBootstrapConsensusMaxInProgressTries, POSINT, "3"),
   /* When a client has any running bridges, check each bridge occasionally,
     * whether or not that bridge is actually up. */
   V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL,"10800"),
@@ -717,56 +755,40 @@ static config_var_t option_vars_[] = {
   OBSOLETE("TestingDescriptorMaxDownloadTries"),
   OBSOLETE("TestingMicrodescMaxDownloadTries"),
   OBSOLETE("TestingCertMaxDownloadTries"),
-  V(TestingDirAuthVoteExit, ROUTERSET, NULL),
-  V(TestingDirAuthVoteExitIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteGuard, ROUTERSET, NULL),
-  V(TestingDirAuthVoteGuardIsStrict,  BOOL,     "0"),
-  V(TestingDirAuthVoteHSDir, ROUTERSET, NULL),
-  V(TestingDirAuthVoteHSDirIsStrict,  BOOL,     "0"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "0"),
+  VAR_INVIS("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_,
+            "0"),
 
   END_OF_CONFIG_VARS
 };
 
+/** List of default directory authorities */
+static const char *default_authorities[] = {
+#ifndef COCCI
+#include "auth_dirs.inc"
+#endif
+  NULL
+};
+
+/** List of fallback directory authorities. The list is generated by opt-in of
+ * relays that meet certain stability criteria.
+ */
+static const char *default_fallbacks[] = {
+#ifndef COCCI
+#include "fallback_dirs.inc"
+#endif
+  NULL
+};
+
 /** Override default values with these if the user sets the TestingTorNetwork
  * option. */
-static const config_var_t testing_tor_network_defaults[] = {
-  V(DirAllowPrivateAddresses,    BOOL,     "1"),
-  V(EnforceDistinctSubnets,      BOOL,     "0"),
-  V(AssumeReachable,             BOOL,     "1"),
-  V(AuthDirMaxServersPerAddr,    UINT,     "0"),
-  V(ClientBootstrapConsensusAuthorityDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusFallbackDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay, CSV_INTERVAL,
-    "0"),
-  V(ClientDNSRejectInternalAddresses, BOOL,"0"),
-  V(ClientRejectInternalAddresses, BOOL,   "0"),
-  V(CountPrivateBandwidth,       BOOL,     "1"),
-  V(ExitPolicyRejectPrivate,     BOOL,     "0"),
-  V(ExtendAllowPrivateAddresses, BOOL,     "1"),
-  V(V3AuthVotingInterval,        INTERVAL, "5 minutes"),
-  V(V3AuthVoteDelay,             INTERVAL, "20 seconds"),
-  V(V3AuthDistDelay,             INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialVotingInterval, INTERVAL, "150 seconds"),
-  V(TestingV3AuthInitialVoteDelay, INTERVAL, "20 seconds"),
-  V(TestingV3AuthInitialDistDelay, INTERVAL, "20 seconds"),
-  V(TestingAuthDirTimeToLearnReachability, INTERVAL, "0 minutes"),
-  V(TestingEstimatedDescriptorPropagationTime, INTERVAL, "0 minutes"),
-  V(MinUptimeHidServDirectoryV2, INTERVAL, "0 minutes"),
-  V(TestingServerDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingServerConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientConsensusDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingBridgeDownloadInitialDelay, CSV_INTERVAL, "10"),
-  V(TestingBridgeBootstrapDownloadInitialDelay, CSV_INTERVAL, "0"),
-  V(TestingClientMaxIntervalWithoutRequest, INTERVAL, "5 seconds"),
-  V(TestingDirConnectionMaxStall, INTERVAL, "30 seconds"),
-  V(TestingEnableConnBwEvent,    BOOL,     "1"),
-  V(TestingEnableCellStatsEvent, BOOL,     "1"),
-  VAR("___UsingTestNetworkDefaults", BOOL, UsingTestNetworkDefaults_, "1"),
-  V(RendPostPeriod,              INTERVAL, "2 minutes"),
-
-  END_OF_CONFIG_VARS
+static const struct {
+  const char *k;
+  const char *v;
+} testing_tor_network_defaults[] = {
+#ifndef COCCI
+#include "testnet.inc"
+#endif
+  { NULL, NULL }
 };
 
 #undef VAR
@@ -788,35 +810,24 @@ static const config_deprecation_t option_deprecation_notes_[] = {
     "effect on clients since 0.2.8." },
   /* End of options deprecated since 0.3.2.2-alpha. */
 
+  /* Options deprecated since 0.4.3.1-alpha. */
+  { "ClientAutoIPv6ORPort", "This option is unreliable if a connection isn't "
+    "reliably dual-stack."},
+  /* End of options deprecated since 0.4.3.1-alpha. */
+
   { NULL, NULL }
 };
 
 #ifdef _WIN32
 static char *get_windows_conf_root(void);
 #endif
-static int options_act_reversible(const or_options_t *old_options, char **msg);
-static int options_transition_allowed(const or_options_t *old,
-                                      const or_options_t *new,
-                                      char **msg);
-static int options_transition_affects_workers(
-      const or_options_t *old_options, const or_options_t *new_options);
-static int options_transition_affects_descriptor(
-      const or_options_t *old_options, const or_options_t *new_options);
-static int options_transition_affects_dirauth_timing(
-      const or_options_t *old_options, const or_options_t *new_options);
-static int normalize_nickname_list(config_line_t **normalized_out,
-                                   const config_line_t *lst, const char *name,
-                                   char **msg);
-static char *get_bindaddr_from_transport_listen_line(const char *line,
-                                                     const char *transport);
-static int check_server_ports(const smartlist_t *ports,
-                              const or_options_t *options,
-                              int *num_low_ports_out);
+
+static int options_check_transition_cb(const void *old,
+                                       const void *new,
+                                       char **msg);
 static int validate_data_directories(or_options_t *options);
 static int write_configuration_file(const char *fname,
                                     const or_options_t *options);
-static int options_init_logs(const or_options_t *old_options,
-                             or_options_t *options, int validate_only);
 
 static void init_libevent(const or_options_t *options);
 static int opt_streq(const char *s1, const char *s2);
@@ -824,27 +835,37 @@ static int parse_outbound_addresses(or_options_t *options, int validate_only,
                                     char **msg);
 static void config_maybe_load_geoip_files_(const or_options_t *options,
                                            const or_options_t *old_options);
-static int options_validate_cb(void *old_options, void *options,
-                               void *default_options,
-                               int from_setconf, char **msg);
-static void options_free_cb(void *options);
+static int options_validate_cb(const void *old_options, void *options,
+                               char **msg);
 static void cleanup_protocol_warning_severity_level(void);
 static void set_protocol_warning_severity_level(int warning_severity);
+static void options_clear_cb(const config_mgr_t *mgr, void *opts);
+static setopt_err_t options_validate_and_set(const or_options_t *old_options,
+                                             or_options_t *new_options,
+                                             char **msg_out);
+struct listener_transaction_t;
+static void options_rollback_listener_transaction(
+                                           struct listener_transaction_t *xn);
 
 /** Magic value for or_options_t. */
 #define OR_OPTIONS_MAGIC 9090909
 
 /** Configuration format for or_options_t. */
-STATIC config_format_t options_format = {
-  sizeof(or_options_t),
-  OR_OPTIONS_MAGIC,
-  offsetof(or_options_t, magic_),
-  option_abbrevs_,
-  option_deprecation_notes_,
-  option_vars_,
-  options_validate_cb,
-  options_free_cb,
-  NULL
+static const config_format_t options_format = {
+  .size = sizeof(or_options_t),
+  .magic = {
+   "or_options_t",
+   OR_OPTIONS_MAGIC,
+   offsetof(or_options_t, magic_),
+  },
+  .abbrevs = option_abbrevs_,
+  .deprecations = option_deprecation_notes_,
+  .vars = option_vars_,
+  .legacy_validate_fn = options_validate_cb,
+  .check_transition_fn = options_check_transition_cb,
+  .clear_fn = options_clear_cb,
+  .has_config_suite = true,
+  .config_suite_offset = offsetof(or_options_t, subconfigs_),
 };
 
 /*
@@ -860,29 +881,36 @@ static or_options_t *global_default_options = NULL;
 static char *torrc_fname = NULL;
 /** Name of the most recently read torrc-defaults file.*/
 static char *torrc_defaults_fname = NULL;
-/** Configuration options set by command line. */
-static config_line_t *global_cmdline_options = NULL;
-/** Non-configuration options set by the command line */
-static config_line_t *global_cmdline_only_options = NULL;
-/** Boolean: Have we parsed the command line? */
-static int have_parsed_cmdline = 0;
-/** Contents of most recently read DirPortFrontPage file. */
-static char *global_dirfrontpagecontents = NULL;
+/** Result of parsing the command line. */
+static parsed_cmdline_t *global_cmdline = NULL;
 /** List of port_cfg_t for all configured ports. */
 static smartlist_t *configured_ports = NULL;
 /** True iff we're currently validating options, and any calls to
  * get_options() are likely to be bugs. */
 static int in_option_validation = 0;
-/* True iff we've initialized libevent */
-static int libevent_initialized = 0;
+/** True iff we have run options_act_once_on_startup() */
+static bool have_set_startup_options = false;
 
-/** Return the contents of our frontpage string, or NULL if not configured. */
-MOCK_IMPL(const char*,
-get_dirportfrontpage, (void))
+/* A global configuration manager to handle all configuration objects. */
+static config_mgr_t *options_mgr = NULL;
+
+/** Return the global configuration manager object for torrc options. */
+STATIC const config_mgr_t *
+get_options_mgr(void)
 {
-  return global_dirfrontpagecontents;
+  if (PREDICT_UNLIKELY(options_mgr == NULL)) {
+    options_mgr = config_mgr_new(&options_format);
+    int rv = subsystems_register_options_formats(options_mgr);
+    tor_assert(rv == 0);
+    config_mgr_freeze(options_mgr);
+  }
+  return options_mgr;
 }
 
+#define CHECK_OPTIONS_MAGIC(opt) STMT_BEGIN                      \
+    config_check_toplevel_magic(get_options_mgr(), (opt));       \
+  STMT_END
+
 /** Returns the currently configured options. */
 MOCK_IMPL(or_options_t *,
 get_options_mutable, (void))
@@ -899,6 +927,32 @@ get_options,(void))
   return get_options_mutable();
 }
 
+/**
+ * True iff we have noticed that this is a testing tor network, and we
+ * should use the corresponding defaults.
+ **/
+static bool testing_network_configured = false;
+
+/** Return a set of lines for any default options that we want to override
+ * from those set in our config_var_t values. */
+static config_line_t *
+get_options_defaults(void)
+{
+  int i;
+  config_line_t *result = NULL, **next = &result;
+
+  if (testing_network_configured) {
+    for (i = 0; testing_tor_network_defaults[i].k; ++i) {
+      config_line_append(next,
+                         testing_tor_network_defaults[i].k,
+                         testing_tor_network_defaults[i].v);
+      next = &(*next)->next;
+    }
+  }
+
+  return result;
+}
+
 /** Change the current global options to contain <b>new_val</b> instead of
  * their current value; take action based on the new value; free the old value
  * as necessary.  Returns 0 on success, -1 on failure.
@@ -906,9 +960,6 @@ get_options,(void))
 int
 set_options(or_options_t *new_val, char **msg)
 {
-  int i;
-  smartlist_t *elements;
-  config_line_t *line;
   or_options_t *old_options = global_options;
   global_options = new_val;
   /* Note that we pass the *old* options below, for comparison. It
@@ -918,7 +969,8 @@ set_options(or_options_t *new_val, char **msg)
     global_options = old_options;
     return -1;
   }
-  if (options_act(old_options) < 0) { /* acting on the options failed. die. */
+  if (subsystems_set_options(get_options_mgr(), new_val) < 0 ||
+      options_act(old_options) < 0) { /* acting on the options failed. die. */
     if (! tor_event_loop_shutdown_is_pending()) {
       log_err(LD_BUG,
               "Acting on config options left us in a broken state. Dying.");
@@ -930,35 +982,10 @@ set_options(or_options_t *new_val, char **msg)
   /* Issues a CONF_CHANGED event to notify controller of the change. If Tor is
    * just starting up then the old_options will be undefined. */
   if (old_options && old_options != global_options) {
-    elements = smartlist_new();
-    for (i=0; options_format.vars[i].name; ++i) {
-      const config_var_t *var = &options_format.vars[i];
-      const char *var_name = var->name;
-      if (var->type == CONFIG_TYPE_LINELIST_S ||
-          var->type == CONFIG_TYPE_OBSOLETE) {
-        continue;
-      }
-      if (!config_is_same(&options_format, new_val, old_options, var_name)) {
-        line = config_get_assigned_option(&options_format, new_val,
-                                          var_name, 1);
-
-        if (line) {
-          config_line_t *next;
-          for (; line; line = next) {
-            next = line->next;
-            smartlist_add(elements, line->key);
-            smartlist_add(elements, line->value);
-            tor_free(line);
-          }
-        } else {
-          smartlist_add_strdup(elements, options_format.vars[i].name);
-          smartlist_add(elements, NULL);
-        }
-      }
-    }
-    control_event_conf_changed(elements);
-    SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-    smartlist_free(elements);
+    config_line_t *changes =
+      config_get_changes(get_options_mgr(), old_options, new_val);
+    control_event_conf_changed(changes);
+    config_free_lines(changes);
   }
 
   if (old_options != global_options) {
@@ -972,49 +999,14 @@ set_options(or_options_t *new_val, char **msg)
   return 0;
 }
 
-/** The version of this Tor process, as parsed. */
-static char *the_tor_version = NULL;
-/** A shorter version of this Tor process's version, for export in our router
- *  descriptor.  (Does not include the git version, if any.) */
-static char *the_short_tor_version = NULL;
-
-/** Return the current Tor version. */
-const char *
-get_version(void)
-{
-  if (the_tor_version == NULL) {
-    if (strlen(tor_git_revision)) {
-      tor_asprintf(&the_tor_version, "%s (git-%s)", get_short_version(),
-                   tor_git_revision);
-    } else {
-      the_tor_version = tor_strdup(get_short_version());
-    }
-  }
-  return the_tor_version;
-}
-
-/** Return the current Tor version, without any git tag. */
-const char *
-get_short_version(void)
-{
-
-  if (the_short_tor_version == NULL) {
-#ifdef TOR_BUILD_TAG
-    tor_asprintf(&the_short_tor_version, "%s (%s)", VERSION, TOR_BUILD_TAG);
-#else
-    the_short_tor_version = tor_strdup(VERSION);
-#endif
-  }
-  return the_short_tor_version;
-}
-
 /** Release additional memory allocated in options
  */
-STATIC void
-or_options_free_(or_options_t *options)
+static void
+options_clear_cb(const config_mgr_t *mgr, void *opts)
 {
-  if (!options)
-    return;
+  (void)mgr;
+  CHECK_OPTIONS_MAGIC(opts);
+  or_options_t *options = opts;
 
   routerset_free(options->ExcludeExitNodesUnion_);
   if (options->NodeFamilySets) {
@@ -1037,7 +1029,14 @@ or_options_free_(or_options_t *options)
   tor_free(options->command_arg);
   tor_free(options->master_key_fname);
   config_free_lines(options->MyFamily);
-  config_free(&options_format, options);
+}
+
+/** Release all memory allocated in options
+ */
+STATIC void
+or_options_free_(or_options_t *options)
+{
+  config_free(get_options_mgr(), options);
 }
 
 /** Release all memory and resources held by global configuration structures.
@@ -1050,11 +1049,7 @@ config_free_all(void)
   or_options_free(global_default_options);
   global_default_options = NULL;
 
-  config_free_lines(global_cmdline_options);
-  global_cmdline_options = NULL;
-
-  config_free_lines(global_cmdline_only_options);
-  global_cmdline_only_options = NULL;
+  parsed_cmdline_free(global_cmdline);
 
   if (configured_ports) {
     SMARTLIST_FOREACH(configured_ports,
@@ -1065,15 +1060,12 @@ config_free_all(void)
 
   tor_free(torrc_fname);
   tor_free(torrc_defaults_fname);
-  tor_free(global_dirfrontpagecontents);
-
-  tor_free(the_short_tor_version);
-  tor_free(the_tor_version);
 
   cleanup_protocol_warning_severity_level();
 
-  have_parsed_cmdline = 0;
-  libevent_initialized = 0;
+  have_set_startup_options = false;
+
+  config_mgr_free(options_mgr);
 }
 
 /** Make <b>address</b> -- a piece of information related to our operation as
@@ -1184,24 +1176,13 @@ init_protocol_warning_severity_level(void)
 static void
 cleanup_protocol_warning_severity_level(void)
 {
-   atomic_counter_destroy(&protocol_warning_severity_level);
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
+  atomic_counter_destroy(&protocol_warning_severity_level);
 }
 
-/** List of default directory authorities */
-
-static const char *default_authorities[] = {
-#include "auth_dirs.inc"
-  NULL
-};
-
-/** List of fallback directory authorities. The list is generated by opt-in of
- * relays that meet certain stability criteria.
- */
-static const char *default_fallbacks[] = {
-#include "fallback_dirs.inc"
-  NULL
-};
-
 /** Add the default directory authorities directly into the trusted dir list,
  * but only add them insofar as they share bits with <b>type</b>.
  * Each authority's bits are restricted to the bits shared with <b>type</b>.
@@ -1237,7 +1218,8 @@ add_default_fallback_dir_servers,(void))
  * user if we changed any dangerous ones.
  */
 static int
-validate_dir_servers(or_options_t *options, or_options_t *old_options)
+validate_dir_servers(const or_options_t *options,
+                     const or_options_t *old_options)
 {
   config_line_t *cl;
 
@@ -1418,32 +1400,29 @@ create_keys_directory(const or_options_t *options)
 /* Helps determine flags to pass to switch_id. */
 static int have_low_ports = -1;
 
-/** Fetch the active option list, and take actions based on it. All of the
- * things we do should survive being done repeatedly.  If present,
- * <b>old_options</b> contains the previous value of the options.
- *
- * Return 0 if all goes well, return -1 if things went badly.
- */
+/** Take case of initial startup tasks that must occur before any of the
+ * transactional option-related changes are allowed. */
 static int
-options_act_reversible(const or_options_t *old_options, char **msg)
+options_act_once_on_startup(char **msg_out)
 {
-  smartlist_t *new_listeners = smartlist_new();
-  or_options_t *options = get_options_mutable();
-  int running_tor = options->command == CMD_RUN_TOR;
-  int set_conn_limit = 0;
-  int r = -1;
-  int logs_marked = 0, logs_initialized = 0;
-  int old_min_log_level = get_min_log_level();
+  if (have_set_startup_options)
+    return 0;
+
+  const or_options_t *options = get_options();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
+  if (!running_tor)
+    return 0;
 
   /* Daemonize _first_, since we only want to open most of this stuff in
    * the subprocess.  Libevent bases can't be reliably inherited across
    * processes. */
-  if (running_tor && options->RunAsDaemon) {
+  if (options->RunAsDaemon) {
     if (! start_daemon_has_been_called())
-      crypto_prefork();
+      subsystems_prefork();
     /* No need to roll back, since you can't change the value. */
     if (start_daemon())
-      crypto_postfork();
+      subsystems_postfork();
   }
 
 #ifdef HAVE_SYSTEMD
@@ -1451,105 +1430,43 @@ options_act_reversible(const or_options_t *old_options, char **msg)
   sd_notifyf(0, "MAINPID=%ld\n", (long int)getpid());
 #endif
 
-#ifndef HAVE_SYS_UN_H
-  if (options->ControlSocket || options->ControlSocketsGroupWritable) {
-    *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported "
-                      "on this OS/with this build.");
-    goto rollback;
-  }
-#else /* !(!defined(HAVE_SYS_UN_H)) */
-  if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
-    *msg = tor_strdup("Setting ControlSocketGroupWritable without setting"
-                      "a ControlSocket makes no sense.");
-    goto rollback;
-  }
-#endif /* !defined(HAVE_SYS_UN_H) */
-
-  if (running_tor) {
-    int n_ports=0;
-    /* We need to set the connection limit before we can open the listeners. */
-    if (! sandbox_is_active()) {
-      if (set_max_file_descriptors((unsigned)options->ConnLimit,
-                                   &options->ConnLimit_) < 0) {
-        *msg = tor_strdup("Problem with ConnLimit value. "
-                          "See logs for details.");
-        goto rollback;
-      }
-      set_conn_limit = 1;
-    } else {
-      tor_assert(old_options);
-      options->ConnLimit_ = old_options->ConnLimit_;
-    }
-
-    /* Set up libevent.  (We need to do this before we can register the
-     * listeners as listeners.) */
-    if (running_tor && !libevent_initialized) {
-      init_libevent(options);
-      libevent_initialized = 1;
-
-      /* This has to come up after libevent is initialized. */
-      control_initialize_event_queue();
-
-      /*
-       * Initialize the scheduler - this has to come after
-       * options_init_from_torrc() sets up libevent - why yes, that seems
-       * completely sensible to hide the libevent setup in the option parsing
-       * code!  It also needs to happen before init_keys(), so it needs to
-       * happen here too.  How yucky. */
-      scheduler_init();
-    }
-
-    /* Adjust the port configuration so we can launch listeners. */
-    if (parse_ports(options, 0, msg, &n_ports, NULL)) {
-      if (!*msg)
-        *msg = tor_strdup("Unexpected problem parsing port config");
-      goto rollback;
-    }
-
-    /* Set the hibernation state appropriately.*/
-    consider_hibernation(time(NULL));
-
-    /* Launch the listeners.  (We do this before we setuid, so we can bind to
-     * ports under 1024.)  We don't want to rebind if we're hibernating or
-     * shutting down. If networking is disabled, this will close all but the
-     * control listeners, but disable those. */
-    if (!we_are_hibernating()) {
-      if (retry_all_listeners(new_listeners, options->DisableNetwork) < 0) {
-        *msg = tor_strdup("Failed to bind one of the listener ports.");
-        goto rollback;
-      }
-    }
-    if (options->DisableNetwork) {
-      /* Aggressively close non-controller stuff, NOW */
-      log_notice(LD_NET, "DisableNetwork is set. Tor will not make or accept "
-                 "non-control network connections. Shutting down all existing "
-                 "connections.");
-      connection_mark_all_noncontrol_connections();
-      /* We can't complete circuits until the network is re-enabled. */
-      note_that_we_maybe_cant_complete_circuits();
-    }
-  }
+  /* Set up libevent.  (We need to do this before we can register the
+   * listeners as listeners.) */
+  init_libevent(options);
 
-#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
-  /* Open /dev/pf before dropping privileges. */
-  if (options->TransPort_set &&
-      options->TransProxyType_parsed == TPT_DEFAULT) {
-    if (get_pf_socket() < 0) {
-      *msg = tor_strdup("Unable to open /dev/pf for transparent proxy.");
-      goto rollback;
-    }
-  }
-#endif /* defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) */
+  /* This has to come up after libevent is initialized. */
+  control_initialize_event_queue();
 
-  /* Attempt to lock all current and future memory with mlockall() only once */
+  /*
+   * Initialize the scheduler - this has to come after
+   * options_init_from_torrc() sets up libevent - why yes, that seems
+   * completely sensible to hide the libevent setup in the option parsing
+   * code!  It also needs to happen before init_keys(), so it needs to
+   * happen here too.  How yucky. */
+  scheduler_init();
+
+  /* Attempt to lock all current and future memory with mlockall() only once.
+   * This must happen before setuid. */
   if (options->DisableAllSwap) {
     if (tor_mlockall() == -1) {
-      *msg = tor_strdup("DisableAllSwap failure. Do you have proper "
+      *msg_out = tor_strdup("DisableAllSwap failure. Do you have proper "
                         "permissions?");
-      goto done;
+      return -1;
     }
   }
 
+  have_set_startup_options = true;
+  return 0;
+}
+
+/**
+ * Change our user ID if we're configured to do so.
+ **/
+static int
+options_switch_id(char **msg_out)
+{
+  const or_options_t *options = get_options();
+
   /* Setuid/setgid as appropriate */
   if (options->User) {
     tor_assert(have_low_ports != -1);
@@ -1563,11 +1480,52 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     }
     if (switch_id(options->User, switch_id_flags) != 0) {
       /* No need to roll back, since you can't change the value. */
-      *msg = tor_strdup("Problem with User value. See logs for details.");
-      goto done;
+      *msg_out = tor_strdup("Problem with User value. See logs for details.");
+      return -1;
     }
   }
 
+  return 0;
+}
+
+/**
+ * Helper. Given a data directory (<b>datadir</b>) and another directory
+ * (<b>subdir</b>) with respective group-writable permissions
+ * <b>datadir_gr</b> and <b>subdir_gr</b>, compute whether the subdir should
+ * be group-writeable.
+ **/
+static int
+compute_group_readable_flag(const char *datadir,
+                            const char *subdir,
+                            int datadir_gr,
+                            int subdir_gr)
+{
+  if (subdir_gr != -1) {
+    /* The user specified a default for "subdir", so we always obey it. */
+    return subdir_gr;
+  }
+
+  /* The user left the subdir_gr option on "auto." */
+  if (0 == strcmp(subdir, datadir)) {
+    /* The directories are the same, so we use the group-readable flag from
+     * the datadirectory */
+    return datadir_gr;
+  } else {
+    /* The directores are different, so we default to "not group-readable" */
+    return 0;
+  }
+}
+
+/**
+ * Create our DataDirectory, CacheDirectory, and KeyDirectory, and
+ * set their permissions correctly.
+ */
+STATIC int
+options_create_directories(char **msg_out)
+{
+  const or_options_t *options = get_options();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
   /* Ensure data directory is private; create if possible. */
   /* It's okay to do this in "options_act_reversible()" even though it isn't
    * actually reversible, since you can't change the DataDirectory while
@@ -1576,58 +1534,288 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                                       options->DataDirectory,
                                       options->DataDirectoryGroupReadable,
                                       options->User,
-                                      msg) < 0) {
-    goto done;
+                                      msg_out) < 0) {
+    return -1;
   }
+
+  /* We need to handle the group-readable flag for the cache directory and key
+   * directory specially, since they may be the same as the data directory */
+  const int key_dir_group_readable = compute_group_readable_flag(
+                                        options->DataDirectory,
+                                        options->KeyDirectory,
+                                        options->DataDirectoryGroupReadable,
+                                        options->KeyDirectoryGroupReadable);
+
   if (check_and_create_data_directory(running_tor /* create */,
                                       options->KeyDirectory,
-                                      options->KeyDirectoryGroupReadable,
+                                      key_dir_group_readable,
                                       options->User,
-                                      msg) < 0) {
-    goto done;
+                                      msg_out) < 0) {
+    return -1;
   }
 
-  /* We need to handle the group-readable flag for the cache directory
-   * specially, since the directory defaults to being the same as the
-   * DataDirectory. */
-  int cache_dir_group_readable;
-  if (options->CacheDirectoryGroupReadable != -1) {
-    /* If the user specified a value, use their setting */
-    cache_dir_group_readable = options->CacheDirectoryGroupReadable;
-  } else if (!strcmp(options->CacheDirectory, options->DataDirectory)) {
-    /* If the user left the value as "auto", and the cache is the same as the
-     * datadirectory, use the datadirectory setting.
-     */
-    cache_dir_group_readable = options->DataDirectoryGroupReadable;
-  } else {
-    /* Otherwise, "auto" means "not group readable". */
-    cache_dir_group_readable = 0;
-  }
+  const int cache_dir_group_readable = compute_group_readable_flag(
+                                        options->DataDirectory,
+                                        options->CacheDirectory,
+                                        options->DataDirectoryGroupReadable,
+                                        options->CacheDirectoryGroupReadable);
+
   if (check_and_create_data_directory(running_tor /* create */,
                                       options->CacheDirectory,
                                       cache_dir_group_readable,
                                       options->User,
-                                      msg) < 0) {
-    goto done;
+                                      msg_out) < 0) {
+    return -1;
   }
 
-  /* Bail out at this point if we're not going to be a client or server:
-   * we don't run Tor itself. */
-  if (!running_tor)
-    goto commit;
+  return 0;
+}
+
+/** Structure to represent an incomplete configuration of a set of
+ * listeners.
+ *
+ * This structure is generated by options_start_listener_transaction(), and is
+ * either committed by options_commit_listener_transaction() or rolled back by
+ * options_rollback_listener_transaction(). */
+typedef struct listener_transaction_t {
+  bool set_conn_limit; /**< True if we've set the connection limit */
+  unsigned old_conn_limit; /**< If nonzero, previous connlimit value. */
+  smartlist_t *new_listeners; /**< List of new listeners that we opened. */
+} listener_transaction_t;
+
+/**
+ * Start configuring our listeners based on the current value of
+ * get_options().
+ *
+ * The value <b>old_options</b> holds either the previous options object,
+ * or NULL if we're starting for the first time.
+ *
+ * On success, return a listener_transaction_t that we can either roll back or
+ * commit.
+ *
+ * On failure return NULL and write a message into a newly allocated string in
+ * *<b>msg_out</b>.
+ **/
+static listener_transaction_t *
+options_start_listener_transaction(const or_options_t *old_options,
+                                   char **msg_out)
+{
+  listener_transaction_t *xn = tor_malloc_zero(sizeof(listener_transaction_t));
+  xn->new_listeners = smartlist_new();
+  or_options_t *options = get_options_mutable();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
+  if (! running_tor) {
+    return xn;
+  }
+
+  int n_ports=0;
+  /* We need to set the connection limit before we can open the listeners. */
+  if (! sandbox_is_active()) {
+    if (set_max_file_descriptors((unsigned)options->ConnLimit,
+                                 &options->ConnLimit_) < 0) {
+      *msg_out = tor_strdup("Problem with ConnLimit value. "
+                            "See logs for details.");
+      goto rollback;
+    }
+    xn->set_conn_limit = true;
+    if (old_options)
+      xn->old_conn_limit = (unsigned)old_options->ConnLimit;
+  } else {
+    tor_assert(old_options);
+    options->ConnLimit_ = old_options->ConnLimit_;
+  }
+
+  /* Adjust the port configuration so we can launch listeners. */
+  /* 31851: some ports are relay-only */
+  if (parse_ports(options, 0, msg_out, &n_ports, NULL)) {
+    if (!*msg_out)
+      *msg_out = tor_strdup("Unexpected problem parsing port config");
+    goto rollback;
+  }
+
+  /* Set the hibernation state appropriately.*/
+  consider_hibernation(time(NULL));
+
+  /* Launch the listeners.  (We do this before we setuid, so we can bind to
+   * ports under 1024.)  We don't want to rebind if we're hibernating or
+   * shutting down. If networking is disabled, this will close all but the
+   * control listeners, but disable those. */
+  /* 31851: some listeners are relay-only */
+  if (!we_are_hibernating()) {
+    if (retry_all_listeners(xn->new_listeners,
+                            options->DisableNetwork) < 0) {
+      *msg_out = tor_strdup("Failed to bind one of the listener ports.");
+      goto rollback;
+    }
+  }
+  if (options->DisableNetwork) {
+    /* Aggressively close non-controller stuff, NOW */
+    log_notice(LD_NET, "DisableNetwork is set. Tor will not make or accept "
+               "non-control network connections. Shutting down all existing "
+               "connections.");
+    connection_mark_all_noncontrol_connections();
+    /* We can't complete circuits until the network is re-enabled. */
+    note_that_we_maybe_cant_complete_circuits();
+  }
+
+#if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
+  /* Open /dev/pf before (possibly) dropping privileges. */
+  if (options->TransPort_set &&
+      options->TransProxyType_parsed == TPT_DEFAULT) {
+    if (get_pf_socket() < 0) {
+      *msg_out = tor_strdup("Unable to open /dev/pf for transparent proxy.");
+      goto rollback;
+    }
+  }
+#endif /* defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H) */
+
+  return xn;
+
+ rollback:
+  options_rollback_listener_transaction(xn);
+  return NULL;
+}
+
+/**
+ * Finish configuring the listeners that started to get configured with
+ * <b>xn</b>.  Frees <b>xn</b>.
+ **/
+static void
+options_commit_listener_transaction(listener_transaction_t *xn)
+{
+  tor_assert(xn);
+  if (xn->set_conn_limit) {
+    or_options_t *options = get_options_mutable();
+    /*
+     * If we adjusted the conn limit, recompute the OOS threshold too
+     *
+     * How many possible sockets to keep in reserve?  If we have lots of
+     * possible sockets, keep this below a limit and set ConnLimit_high_thresh
+     * very close to ConnLimit_, but if ConnLimit_ is low, shrink it in
+     * proportion.
+     *
+     * Somewhat arbitrarily, set socks_in_reserve to 5% of ConnLimit_, but
+     * cap it at 64.
+     */
+    int socks_in_reserve = options->ConnLimit_ / 20;
+    if (socks_in_reserve > 64) socks_in_reserve = 64;
+
+    options->ConnLimit_high_thresh = options->ConnLimit_ - socks_in_reserve;
+    options->ConnLimit_low_thresh = (options->ConnLimit_ / 4) * 3;
+    log_info(LD_GENERAL,
+             "Recomputed OOS thresholds: ConnLimit %d, ConnLimit_ %d, "
+             "ConnLimit_high_thresh %d, ConnLimit_low_thresh %d",
+             options->ConnLimit, options->ConnLimit_,
+             options->ConnLimit_high_thresh,
+             options->ConnLimit_low_thresh);
+
+    /* Give the OOS handler a chance with the new thresholds */
+    connection_check_oos(get_n_open_sockets(), 0);
+  }
+
+  smartlist_free(xn->new_listeners);
+  tor_free(xn);
+}
+
+/**
+ * Revert the listener configuration changes that that started to get
+ * configured with <b>xn</b>.  Frees <b>xn</b>.
+ **/
+static void
+options_rollback_listener_transaction(listener_transaction_t *xn)
+{
+  if (! xn)
+    return;
+
+  or_options_t *options = get_options_mutable();
+
+  if (xn->set_conn_limit && xn->old_conn_limit)
+    set_max_file_descriptors(xn->old_conn_limit, &options->ConnLimit_);
+
+  SMARTLIST_FOREACH(xn->new_listeners, connection_t *, conn,
+  {
+    log_notice(LD_NET, "Closing partially-constructed %s on %s:%d",
+               conn_type_to_string(conn->type), conn->address, conn->port);
+    connection_close_immediate(conn);
+    connection_mark_for_close(conn);
+  });
+
+  smartlist_free(xn->new_listeners);
+  tor_free(xn);
+}
+
+/** Structure to represent an incomplete configuration of a set of logs.
+ *
+ * This structure is generated by options_start_log_transaction(), and is
+ * either committed by options_commit_log_transaction() or rolled back by
+ * options_rollback_log_transaction(). */
+typedef struct log_transaction_t {
+  /** Previous lowest severity of any configured log. */
+  int old_min_log_level;
+  /** True if we have marked the previous logs to be closed */
+  bool logs_marked;
+  /** True if we initialized the new set of logs */
+  bool logs_initialized;
+  /** True if our safelogging configuration is different from what it was
+   * previously (or if we are starting for the first time). */
+  bool safelogging_changed;
+} log_transaction_t;
+
+/**
+ * Start configuring our logs based on the current value of get_options().
+ *
+ * The value <b>old_options</b> holds either the previous options object,
+ * or NULL if we're starting for the first time.
+ *
+ * On success, return a log_transaction_t that we can either roll back or
+ * commit.
+ *
+ * On failure return NULL and write a message into a newly allocated string in
+ * *<b>msg_out</b>.
+ **/
+STATIC log_transaction_t *
+options_start_log_transaction(const or_options_t *old_options,
+                              char **msg_out)
+{
+  const or_options_t *options = get_options();
+  const bool running_tor = options->command == CMD_RUN_TOR;
+
+  log_transaction_t *xn = tor_malloc_zero(sizeof(log_transaction_t));
+  xn->old_min_log_level = get_min_log_level();
+  xn->safelogging_changed = !old_options ||
+    old_options->SafeLogging_ != options->SafeLogging_;
+
+  if (! running_tor)
+    goto done;
 
   mark_logs_temp(); /* Close current logs once new logs are open. */
-  logs_marked = 1;
+  xn->logs_marked = true;
   /* Configure the tor_log(s) */
   if (options_init_logs(old_options, options, 0)<0) {
-    *msg = tor_strdup("Failed to init Log options. See logs for details.");
-    goto rollback;
+    *msg_out = tor_strdup("Failed to init Log options. See logs for details.");
+    options_rollback_log_transaction(xn);
+    xn = NULL;
+    goto done;
   }
-  logs_initialized = 1;
 
- commit:
-  r = 0;
-  if (logs_marked) {
+  xn->logs_initialized = true;
+
+ done:
+  return xn;
+}
+
+/**
+ * Finish configuring the logs that started to get configured with <b>xn</b>.
+ * Frees <b>xn</b>.
+ **/
+STATIC void
+options_commit_log_transaction(log_transaction_t *xn)
+{
+  const or_options_t *options = get_options();
+  tor_assert(xn);
+
+  if (xn->logs_marked) {
     log_severity_list_t *severity =
       tor_malloc_zero(sizeof(log_severity_list_t));
     close_temp_logs();
@@ -1637,7 +1825,8 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     tor_free(severity);
     tor_log_update_sigsafe_err_fds();
   }
-  if (logs_initialized) {
+
+  if (xn->logs_initialized) {
     flush_log_messages_from_startup();
   }
 
@@ -1646,12 +1835,12 @@ options_act_reversible(const or_options_t *old_options, char **msg)
     int bad_safelog = 0, bad_severity = 0, new_badness = 0;
     if (options->SafeLogging_ != SAFELOG_SCRUB_ALL) {
       bad_safelog = 1;
-      if (!old_options || old_options->SafeLogging_ != options->SafeLogging_)
+      if (xn->safelogging_changed)
         new_badness = 1;
     }
     if (get_min_log_level() >= LOG_INFO) {
       bad_severity = 1;
-      if (get_min_log_level() != old_min_log_level)
+      if (get_min_log_level() != xn->old_min_log_level)
         new_badness = 1;
     }
     if (bad_safelog && bad_severity)
@@ -1667,59 +1856,105 @@ options_act_reversible(const or_options_t *old_options, char **msg)
                "Overwrite the log afterwards.", badness);
   }
 
-  if (set_conn_limit) {
-    /*
-     * If we adjusted the conn limit, recompute the OOS threshold too
-     *
-     * How many possible sockets to keep in reserve?  If we have lots of
-     * possible sockets, keep this below a limit and set ConnLimit_high_thresh
-     * very close to ConnLimit_, but if ConnLimit_ is low, shrink it in
-     * proportion.
-     *
-     * Somewhat arbitrarily, set socks_in_reserve to 5% of ConnLimit_, but
-     * cap it at 64.
-     */
-    int socks_in_reserve = options->ConnLimit_ / 20;
-    if (socks_in_reserve > 64) socks_in_reserve = 64;
+  tor_free(xn);
+}
 
-    options->ConnLimit_high_thresh = options->ConnLimit_ - socks_in_reserve;
-    options->ConnLimit_low_thresh = (options->ConnLimit_ / 4) * 3;
-    log_info(LD_GENERAL,
-             "Recomputed OOS thresholds: ConnLimit %d, ConnLimit_ %d, "
-             "ConnLimit_high_thresh %d, ConnLimit_low_thresh %d",
-             options->ConnLimit, options->ConnLimit_,
-             options->ConnLimit_high_thresh,
-             options->ConnLimit_low_thresh);
+/**
+ * Revert the log configuration changes that that started to get configured
+ * with <b>xn</b>.  Frees <b>xn</b>.
+ **/
+STATIC void
+options_rollback_log_transaction(log_transaction_t *xn)
+{
+  if (!xn)
+    return;
 
-    /* Give the OOS handler a chance with the new thresholds */
-    connection_check_oos(get_n_open_sockets(), 0);
+  if (xn->logs_marked) {
+    rollback_log_changes();
+    control_adjust_event_log_severity();
   }
 
+  tor_free(xn);
+}
+
+/**
+ * Fetch the active option list, and take actions based on it. All of
+ * the things we do in this function should survive being done
+ * repeatedly, OR be done only once when starting Tor.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * This function is only truly "reversible" _after_ the first time it
+ * is run.  The first time that it runs, it performs some irreversible
+ * tasks in the correct sequence between the reversible option changes.
+ *
+ * Option changes should only be marked as "reversible" if they cannot
+ * be validated before switching them, but they can be switched back if
+ * some other validation fails.
+ *
+ * Return 0 if all goes well, return -1 if things went badly.
+ */
+MOCK_IMPL(STATIC int,
+options_act_reversible,(const or_options_t *old_options, char **msg))
+{
+  const bool first_time = ! have_set_startup_options;
+  log_transaction_t *log_transaction = NULL;
+  listener_transaction_t *listener_transaction = NULL;
+  int r = -1;
+
+  /* The ordering of actions in this function is not free, sadly.
+   *
+   * First of all, we _must_ daemonize before we take all kinds of
+   * initialization actions, since they need to happen in the
+   * subprocess.
+   */
+  if (options_act_once_on_startup(msg) < 0)
+    goto rollback;
+
+  /* Once we've handled most of once-off initialization, we need to
+   * open our listeners before we switch IDs.  (If we open listeners first,
+   * we might not be able to bind to low ports.)
+   */
+  listener_transaction = options_start_listener_transaction(old_options, msg);
+  if (listener_transaction == NULL)
+    goto rollback;
+
+  if (first_time) {
+    if (options_switch_id(msg) < 0)
+      goto rollback;
+  }
+
+  /* On the other hand, we need to touch the file system _after_ we
+   * switch IDs: otherwise, we'll be making directories and opening files
+   * with the wrong permissions.
+   */
+  if (first_time) {
+    if (options_create_directories(msg) < 0)
+      goto rollback;
+  }
+
+  /* Bail out at this point if we're not going to be a client or server:
+   * we don't run Tor itself. */
+  log_transaction = options_start_log_transaction(old_options, msg);
+  if (log_transaction == NULL)
+    goto rollback;
+
+  // Commit!
+  r = 0;
+
+  options_commit_log_transaction(log_transaction);
+
+  options_commit_listener_transaction(listener_transaction);
+
   goto done;
 
  rollback:
   r = -1;
   tor_assert(*msg);
 
-  if (logs_marked) {
-    rollback_log_changes();
-    control_adjust_event_log_severity();
-  }
-
-  if (set_conn_limit && old_options)
-    set_max_file_descriptors((unsigned)old_options->ConnLimit,
-                             &options->ConnLimit_);
-
-  SMARTLIST_FOREACH(new_listeners, connection_t *, conn,
-  {
-    log_notice(LD_NET, "Closing partially-constructed %s on %s:%d",
-               conn_type_to_string(conn->type), conn->address, conn->port);
-    connection_close_immediate(conn);
-    connection_mark_for_close(conn);
-  });
+  options_rollback_log_transaction(log_transaction);
+  options_rollback_listener_transaction(listener_transaction);
 
  done:
-  smartlist_free(new_listeners);
   return r;
 }
 
@@ -1732,6 +1967,7 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   int routerset_usage =
     routerset_needs_geoip(options->EntryNodes) ||
     routerset_needs_geoip(options->ExitNodes) ||
+    routerset_needs_geoip(options->MiddleNodes) ||
     routerset_needs_geoip(options->ExcludeExitNodes) ||
     routerset_needs_geoip(options->ExcludeNodes) ||
     routerset_needs_geoip(options->HSLayer2Nodes) ||
@@ -1749,32 +1985,6 @@ options_need_geoip_info(const or_options_t *options, const char **reason_out)
   return bridge_usage || routerset_usage;
 }
 
-/** Return the bandwidthrate that we are going to report to the authorities
- * based on the config options. */
-uint32_t
-get_effective_bwrate(const or_options_t *options)
-{
-  uint64_t bw = options->BandwidthRate;
-  if (bw > options->MaxAdvertisedBandwidth)
-    bw = options->MaxAdvertisedBandwidth;
-  if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate)
-    bw = options->RelayBandwidthRate;
-  /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */
-  return (uint32_t)bw;
-}
-
-/** Return the bandwidthburst that we are going to report to the authorities
- * based on the config options. */
-uint32_t
-get_effective_bwburst(const or_options_t *options)
-{
-  uint64_t bw = options->BandwidthBurst;
-  if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst)
-    bw = options->RelayBandwidthBurst;
-  /* ensure_bandwidth_cap() makes sure that this cast can't overflow. */
-  return (uint32_t)bw;
-}
-
 /* Used in the various options_transition_affects* functions. */
 #define YES_IF_CHANGED_BOOL(opt) \
   if (!CFG_EQ_BOOL(old_options, new_options, opt)) return 1;
@@ -1817,32 +2027,6 @@ options_transition_affects_guards(const or_options_t *old_options,
   return 0;
 }
 
-/**
- * Return true if changing the configuration from <b>old</b> to <b>new</b>
- * affects the timing of the voting subsystem
- */
-static int
-options_transition_affects_dirauth_timing(const or_options_t *old_options,
-                                          const or_options_t *new_options)
-{
-  tor_assert(old_options);
-  tor_assert(new_options);
-
-  if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options))
-    return 1;
-  if (! authdir_mode_v3(new_options))
-    return 0;
-  YES_IF_CHANGED_INT(V3AuthVotingInterval);
-  YES_IF_CHANGED_INT(V3AuthVoteDelay);
-  YES_IF_CHANGED_INT(V3AuthDistDelay);
-  YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval);
-  YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay);
-  YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay);
-  YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset);
-
-  return 0;
-}
-
 /** Fetch the active option list, and take actions based on it. All of the
  * things we do should survive being done repeatedly.  If present,
  * <b>old_options</b> contains the previous value of the options.
@@ -1850,17 +2034,16 @@ options_transition_affects_dirauth_timing(const or_options_t *old_options,
  * Return 0 if all goes well, return -1 if it's time to die.
  *
  * Note: We haven't moved all the "act on new configuration" logic
- * here yet.  Some is still in do_hup() and other places.
+ * the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
  */
-STATIC int
-options_act(const or_options_t *old_options)
+MOCK_IMPL(STATIC int,
+options_act,(const or_options_t *old_options))
 {
   config_line_t *cl;
   or_options_t *options = get_options_mutable();
   int running_tor = options->command == CMD_RUN_TOR;
   char *msg=NULL;
-  const int transition_affects_workers =
-    old_options && options_transition_affects_workers(old_options, options);
   const int transition_affects_guards =
     old_options && options_transition_affects_guards(old_options, options);
 
@@ -1918,19 +2101,6 @@ options_act(const or_options_t *old_options)
              "in a non-anonymous mode. It will provide NO ANONYMITY.");
   }
 
-  /* If we are a bridge with a pluggable transport proxy but no
-     Extended ORPort, inform the user that they are missing out. */
-  if (server_mode(options) && options->ServerTransportPlugin &&
-      !options->ExtORPort_lines) {
-    log_notice(LD_CONFIG, "We use pluggable transports but the Extended "
-               "ORPort is disabled. Tor and your pluggable transports proxy "
-               "communicate with each other via the Extended ORPort so it "
-               "is suggested you enable it: it will also allow your Bridge "
-               "to collect statistics about its clients that use pluggable "
-               "transports. Please enable it using the ExtORPort torrc option "
-               "(e.g. set 'ExtORPort auto').");
-  }
-
   if (options->Bridges) {
     mark_bridge_list();
     for (cl = options->Bridges; cl; cl = cl->next) {
@@ -1980,22 +2150,17 @@ options_act(const or_options_t *old_options)
   if (! or_state_loaded() && running_tor) {
     if (or_state_load())
       return -1;
-    rep_hist_load_mtbf_data(time(NULL));
-  }
-
-  /* If we have an ExtORPort, initialize its auth cookie. */
-  if (running_tor &&
-      init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) {
-    log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file.");
-    return -1;
+    if (options_act_dirauth_mtbf(options) < 0)
+      return -1;
   }
 
+  /* 31851: some of the code in these functions is relay-only */
   mark_transport_list();
   pt_prepare_proxy_list_for_config_read();
   if (!options->DisableNetwork) {
     if (options->ClientTransportPlugin) {
       for (cl = options->ClientTransportPlugin; cl; cl = cl->next) {
-        if (parse_transport_line(options, cl->value, 0, 0) < 0) {
+        if (pt_parse_transport_line(options, cl->value, 0, 0) < 0) {
           // LCOV_EXCL_START
           log_warn(LD_BUG,
                    "Previously validated ClientTransportPlugin line "
@@ -2005,20 +2170,11 @@ options_act(const or_options_t *old_options)
         }
       }
     }
-
-    if (options->ServerTransportPlugin && server_mode(options)) {
-      for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
-        if (parse_transport_line(options, cl->value, 0, 1) < 0) {
-          // LCOV_EXCL_START
-          log_warn(LD_BUG,
-                   "Previously validated ServerTransportPlugin line "
-                   "could not be added!");
-          return -1;
-          // LCOV_EXCL_STOP
-        }
-      }
-    }
   }
+
+  if (options_act_server_transport(old_options) < 0)
+    return -1;
+
   sweep_transport_list();
   sweep_proxy_list();
 
@@ -2039,19 +2195,8 @@ options_act(const or_options_t *old_options)
     finish_daemon(options->DataDirectory);
   }
 
-  /* See whether we need to enable/disable our once-a-second timer. */
-  reschedule_per_second_timer();
-
-  /* We want to reinit keys as needed before we do much of anything else:
-     keys are important, and other things can depend on them. */
-  if (transition_affects_workers ||
-      (options->V3AuthoritativeDir && (!old_options ||
-                                       !old_options->V3AuthoritativeDir))) {
-    if (init_keys() < 0) {
-      log_warn(LD_BUG,"Error initializing keys; exiting");
-      return -1;
-    }
-  }
+  if (options_act_relay(old_options) < 0)
+    return -1;
 
   /* Write our PID to the PID file. If we do not have write permissions we
    * will log a warning and exit. */
@@ -2075,15 +2220,6 @@ options_act(const or_options_t *old_options)
     return -1;
   }
 
-  if (server_mode(options)) {
-    static int cdm_initialized = 0;
-    if (cdm_initialized == 0) {
-      cdm_initialized = 1;
-      consdiffmgr_configure(NULL);
-      consdiffmgr_validate();
-    }
-  }
-
   if (init_control_cookie_authentication(options->CookieAuthentication) < 0) {
     log_warn(LD_CONFIG,"Error creating control cookie authentication file.");
     return -1;
@@ -2101,15 +2237,8 @@ options_act(const or_options_t *old_options)
    * might be a change of scheduler or parameter. */
   scheduler_conf_changed();
 
-  /* Set up accounting */
-  if (accounting_parse_options(options, 0)<0) {
-    // LCOV_EXCL_START
-    log_warn(LD_BUG,"Error in previously validated accounting options");
+  if (options_act_relay_accounting(old_options) < 0)
     return -1;
-    // LCOV_EXCL_STOP
-  }
-  if (accounting_is_enabled(options))
-    configure_accounting(time(NULL));
 
   /* Change the cell EWMA settings */
   cmux_ewma_set_options(options, networkstatus_get_latest_consensus());
@@ -2133,6 +2262,7 @@ options_act(const or_options_t *old_options)
     tor_free(http_authenticator);
   }
 
+  /* 31851: OutboundBindAddressExit is relay-only */
   if (parse_outbound_addresses(options, 0, &msg) < 0) {
     // LCOV_EXCL_START
     log_warn(LD_BUG, "Failed parsing previously validated outbound "
@@ -2174,6 +2304,7 @@ options_act(const or_options_t *old_options)
                          options->HSLayer2Nodes) ||
         !routerset_equal(old_options->HSLayer3Nodes,
                          options->HSLayer3Nodes) ||
+        !routerset_equal(old_options->MiddleNodes, options->MiddleNodes) ||
         options->StrictNodes != old_options->StrictNodes) {
       log_info(LD_CIRC,
                "Changed to using entry guards or bridges, or changed "
@@ -2218,65 +2349,17 @@ options_act(const or_options_t *old_options)
     if (revise_automap_entries)
       addressmap_clear_invalid_automaps(options);
 
-/* How long should we delay counting bridge stats after becoming a bridge?
- * We use this so we don't count clients who used our bridge thinking it is
- * a relay. If you change this, don't forget to change the log message
- * below. It's 4 hours (the time it takes to stop being used by clients)
- * plus some extra time for clock skew. */
-#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60)
-
-    if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) {
-      int was_relay = 0;
-      if (options->BridgeRelay) {
-        time_t int_start = time(NULL);
-        if (config_lines_eq(old_options->ORPort_lines,options->ORPort_lines)) {
-          int_start += RELAY_BRIDGE_STATS_DELAY;
-          was_relay = 1;
-        }
-        geoip_bridge_stats_init(int_start);
-        log_info(LD_CONFIG, "We are acting as a bridge now.  Starting new "
-                 "GeoIP stats interval%s.", was_relay ? " in 6 "
-                 "hours from now" : "");
-      } else {
-        geoip_bridge_stats_term();
-        log_info(LD_GENERAL, "We are no longer acting as a bridge.  "
-                 "Forgetting GeoIP stats.");
-      }
-    }
-
-    if (transition_affects_workers) {
-      log_info(LD_GENERAL,
-               "Worker-related options changed. Rotating workers.");
-      const int server_mode_turned_on =
-        server_mode(options) && !server_mode(old_options);
-      const int dir_server_mode_turned_on =
-        dir_server_mode(options) && !dir_server_mode(old_options);
-
-      if (server_mode_turned_on || dir_server_mode_turned_on) {
-        cpu_init();
-      }
+    if (options_act_bridge_stats(old_options) < 0)
+      return -1;
 
-      if (server_mode_turned_on) {
-        ip_address_changed(0);
-        if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
-          inform_testing_reachability();
-      }
-      cpuworkers_rotate_keyinfo();
-      if (dns_reset())
-        return -1;
-    } else {
-      if (dns_reset())
-        return -1;
-    }
+    if (dns_reset())
+      return -1;
 
-    if (options->PerConnBWRate != old_options->PerConnBWRate ||
-        options->PerConnBWBurst != old_options->PerConnBWBurst)
-      connection_or_update_token_buckets(get_connection_array(), options);
+    if (options_act_relay_bandwidth(old_options) < 0)
+      return -1;
 
     if (options->BandwidthRate != old_options->BandwidthRate ||
-        options->BandwidthBurst != old_options->BandwidthBurst ||
-        options->RelayBandwidthRate != old_options->RelayBandwidthRate ||
-        options->RelayBandwidthBurst != old_options->RelayBandwidthBurst)
+        options->BandwidthBurst != old_options->BandwidthBurst)
       connection_bucket_adjust(options);
 
     if (options->MainloopStats != old_options->MainloopStats) {
@@ -2284,132 +2367,43 @@ options_act(const or_options_t *old_options)
     }
   }
 
+  /* 31851: These options are relay-only, but we need to disable them if we
+   * are in client mode. In 29211, we will disable all relay options in
+   * client mode. */
   /* Only collect directory-request statistics on relays and bridges. */
   options->DirReqStatistics = options->DirReqStatistics_option &&
     server_mode(options);
   options->HiddenServiceStatistics =
     options->HiddenServiceStatistics_option && server_mode(options);
 
-  if (options->CellStatistics || options->DirReqStatistics ||
-      options->EntryStatistics || options->ExitPortStatistics ||
-      options->ConnDirectionStatistics ||
-      options->HiddenServiceStatistics ||
-      options->BridgeAuthoritativeDir) {
-    time_t now = time(NULL);
-    int print_notice = 0;
-
-    /* Only collect other relay-only statistics on relays. */
-    if (!public_server_mode(options)) {
-      options->CellStatistics = 0;
-      options->EntryStatistics = 0;
-      options->ConnDirectionStatistics = 0;
-      options->ExitPortStatistics = 0;
-    }
-
-    if ((!old_options || !old_options->CellStatistics) &&
-        options->CellStatistics) {
-      rep_hist_buffer_stats_init(now);
-      print_notice = 1;
-    }
-    if ((!old_options || !old_options->DirReqStatistics) &&
-        options->DirReqStatistics) {
-      if (geoip_is_loaded(AF_INET)) {
-        geoip_dirreq_stats_init(now);
-        print_notice = 1;
-      } else {
-        /* disable statistics collection since we have no geoip file */
-        options->DirReqStatistics = 0;
-        if (options->ORPort_set)
-          log_notice(LD_CONFIG, "Configured to measure directory request "
-                                "statistics, but no GeoIP database found. "
-                                "Please specify a GeoIP database using the "
-                                "GeoIPFile option.");
-      }
-    }
-    if ((!old_options || !old_options->EntryStatistics) &&
-        options->EntryStatistics && !should_record_bridge_info(options)) {
-      /* If we get here, we've started recording bridge info when we didn't
-       * do so before.  Note that "should_record_bridge_info()" will
-       * always be false at this point, because of the earlier block
-       * that cleared EntryStatistics when public_server_mode() was false.
-       * We're leaving it in as defensive programming. */
-      if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) {
-        geoip_entry_stats_init(now);
-        print_notice = 1;
-      } else {
-        options->EntryStatistics = 0;
-        log_notice(LD_CONFIG, "Configured to measure entry node "
-                              "statistics, but no GeoIP database found. "
-                              "Please specify a GeoIP database using the "
-                              "GeoIPFile option.");
-      }
-    }
-    if ((!old_options || !old_options->ExitPortStatistics) &&
-        options->ExitPortStatistics) {
-      rep_hist_exit_stats_init(now);
-      print_notice = 1;
-    }
-    if ((!old_options || !old_options->ConnDirectionStatistics) &&
-        options->ConnDirectionStatistics) {
-      rep_hist_conn_stats_init(now);
-    }
-    if ((!old_options || !old_options->HiddenServiceStatistics) &&
-        options->HiddenServiceStatistics) {
-      log_info(LD_CONFIG, "Configured to measure hidden service statistics.");
-      rep_hist_hs_stats_init(now);
-    }
-    if ((!old_options || !old_options->BridgeAuthoritativeDir) &&
-        options->BridgeAuthoritativeDir) {
-      rep_hist_desc_stats_init(now);
-      print_notice = 1;
-    }
-    if (print_notice)
-        log_notice(LD_CONFIG, "Configured to measure statistics. Look for "
-                "the *-stats files that will first be written to the "
-                 "data directory in 24 hours from now.");
-  }
-
-  /* If we used to have statistics enabled but we just disabled them,
-     stop gathering them.  */
-  if (old_options && old_options->CellStatistics &&
-      !options->CellStatistics)
-    rep_hist_buffer_stats_term();
-  if (old_options && old_options->DirReqStatistics &&
-      !options->DirReqStatistics)
-    geoip_dirreq_stats_term();
-  if (old_options && old_options->EntryStatistics &&
-      !options->EntryStatistics)
-    geoip_entry_stats_term();
-  if (old_options && old_options->HiddenServiceStatistics &&
-      !options->HiddenServiceStatistics)
-    rep_hist_hs_stats_term();
-  if (old_options && old_options->ExitPortStatistics &&
-      !options->ExitPortStatistics)
-    rep_hist_exit_stats_term();
-  if (old_options && old_options->ConnDirectionStatistics &&
-      !options->ConnDirectionStatistics)
-    rep_hist_conn_stats_term();
-  if (old_options && old_options->BridgeAuthoritativeDir &&
-      !options->BridgeAuthoritativeDir)
-    rep_hist_desc_stats_term();
-
-  /* Since our options changed, we might need to regenerate and upload our
-   * server descriptor.
-   */
-  if (!old_options ||
-      options_transition_affects_descriptor(old_options, options))
-    mark_my_descriptor_dirty("config change");
+  /* Only collect other relay-only statistics on relays. */
+  if (!public_server_mode(options)) {
+    options->CellStatistics = 0;
+    options->EntryStatistics = 0;
+    options->ConnDirectionStatistics = 0;
+    options->ExitPortStatistics = 0;
+  }
+
+  bool print_notice = 0;
+  if (options_act_relay_stats(old_options, &print_notice) < 0)
+    return -1;
+  if (options_act_dirauth_stats(old_options, &print_notice) < 0)
+    return -1;
+  if (print_notice)
+    options_act_relay_stats_msg();
+
+  if (options_act_relay_desc(old_options) < 0)
+    return -1;
+
+  if (options_act_dirauth(old_options) < 0)
+    return -1;
 
   /* We may need to reschedule some directory stuff if our status changed. */
   if (old_options) {
-    if (options_transition_affects_dirauth_timing(old_options, options)) {
-      voting_schedule_recalculate_timing(options, time(NULL));
-      reschedule_dirvote(options);
-    }
-    if (!bool_eq(directory_fetches_dir_info_early(options),
-                 directory_fetches_dir_info_early(old_options)) ||
-        !bool_eq(directory_fetches_dir_info_later(options),
-                 directory_fetches_dir_info_later(old_options)) ||
+    if (!bool_eq(dirclient_fetches_dir_info_early(options),
+                 dirclient_fetches_dir_info_early(old_options)) ||
+        !bool_eq(dirclient_fetches_dir_info_later(options),
+                 dirclient_fetches_dir_info_later(old_options)) ||
         !config_lines_eq(old_options->Bridges, options->Bridges)) {
       /* Make sure update_router_have_minimum_dir_info() gets called. */
       router_dir_info_changed();
@@ -2419,87 +2413,113 @@ options_act(const or_options_t *old_options)
     }
   }
 
-  /* DoS mitigation subsystem only applies to public relay. */
-  if (public_server_mode(options)) {
-    /* If we are configured as a relay, initialize the subsystem. Even on HUP,
-     * this is safe to call as it will load data from the current options
-     * or/and the consensus. */
-    dos_init();
-  } else if (old_options && public_server_mode(old_options)) {
-    /* Going from relay to non relay, clean it up. */
-    dos_free_all();
-  }
-
-  /* Load the webpage we're going to serve every time someone asks for '/' on
-     our DirPort. */
-  tor_free(global_dirfrontpagecontents);
-  if (options->DirPortFrontPage) {
-    global_dirfrontpagecontents =
-      read_file_to_str(options->DirPortFrontPage, 0, NULL);
-    if (!global_dirfrontpagecontents) {
-      log_warn(LD_CONFIG,
-               "DirPortFrontPage file '%s' not found. Continuing anyway.",
-               options->DirPortFrontPage);
-    }
-  }
+  if (options_act_relay_dos(old_options) < 0)
+    return -1;
+  if (options_act_relay_dir(old_options) < 0)
+    return -1;
 
   return 0;
 }
 
+/**
+ * Enumeration to describe the syntax for a command-line option.
+ **/
 typedef enum {
-  TAKES_NO_ARGUMENT = 0,
+  /** Describe an option that does not take an argument. */
+  ARGUMENT_NONE = 0,
+  /** Describes an option that takes a single argument. */
   ARGUMENT_NECESSARY = 1,
+  /** Describes an option that takes a single optional argument. */
   ARGUMENT_OPTIONAL = 2
 } takes_argument_t;
 
+/** Table describing arguments that Tor accepts on the command line,
+ * other than those that are the same as in torrc. */
 static const struct {
+  /** The string that the user has to provide. */
   const char *name;
+  /** Does this option accept an argument? */
   takes_argument_t takes_argument;
+  /** If not CMD_RUN_TOR, what should Tor do when it starts? */
+  tor_cmdline_mode_t command;
+  /** If nonzero, set the quiet level to this. 1 is "hush", 2 is "quiet" */
+  int quiet;
 } CMDLINE_ONLY_OPTIONS[] = {
-  { "-f",                     ARGUMENT_NECESSARY },
-  { "--allow-missing-torrc",  TAKES_NO_ARGUMENT },
-  { "--defaults-torrc",       ARGUMENT_NECESSARY },
-  { "--hash-password",        ARGUMENT_NECESSARY },
-  { "--dump-config",          ARGUMENT_OPTIONAL },
-  { "--list-fingerprint",     TAKES_NO_ARGUMENT },
-  { "--keygen",               TAKES_NO_ARGUMENT },
-  { "--key-expiration",       ARGUMENT_OPTIONAL },
-  { "--newpass",              TAKES_NO_ARGUMENT },
-  { "--no-passphrase",        TAKES_NO_ARGUMENT },
-  { "--passphrase-fd",        ARGUMENT_NECESSARY },
-  { "--verify-config",        TAKES_NO_ARGUMENT },
-  { "--ignore-missing-torrc", TAKES_NO_ARGUMENT },
-  { "--quiet",                TAKES_NO_ARGUMENT },
-  { "--hush",                 TAKES_NO_ARGUMENT },
-  { "--version",              TAKES_NO_ARGUMENT },
-  { "--list-modules",         TAKES_NO_ARGUMENT },
-  { "--library-versions",     TAKES_NO_ARGUMENT },
-  { "-h",                     TAKES_NO_ARGUMENT },
-  { "--help",                 TAKES_NO_ARGUMENT },
-  { "--list-torrc-options",   TAKES_NO_ARGUMENT },
-  { "--list-deprecated-options",TAKES_NO_ARGUMENT },
-  { "--nt-service",           TAKES_NO_ARGUMENT },
-  { "-nt-service",            TAKES_NO_ARGUMENT },
-  { NULL, 0 },
+  { .name="-f",
+    .takes_argument=ARGUMENT_NECESSARY },
+  { .name="--allow-missing-torrc" },
+  { .name="--defaults-torrc",
+    .takes_argument=ARGUMENT_NECESSARY },
+  { .name="--hash-password",
+    .takes_argument=ARGUMENT_NECESSARY,
+    .command=CMD_HASH_PASSWORD,
+    .quiet=QUIET_HUSH  },
+  { .name="--dump-config",
+    .takes_argument=ARGUMENT_OPTIONAL,
+    .command=CMD_DUMP_CONFIG,
+    .quiet=QUIET_SILENT },
+  { .name="--list-fingerprint",
+    .command=CMD_LIST_FINGERPRINT },
+  { .name="--keygen",
+    .command=CMD_KEYGEN },
+  { .name="--key-expiration",
+    .takes_argument=ARGUMENT_OPTIONAL,
+    .command=CMD_KEY_EXPIRATION },
+  { .name="--newpass" },
+  { .name="--no-passphrase" },
+  { .name="--passphrase-fd",
+    .takes_argument=ARGUMENT_NECESSARY },
+  { .name="--verify-config",
+    .command=CMD_VERIFY_CONFIG },
+  { .name="--ignore-missing-torrc" },
+  { .name="--quiet",
+    .quiet=QUIET_SILENT },
+  { .name="--hush",
+    .quiet=QUIET_HUSH },
+  { .name="--version",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--list-modules",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--library-versions",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="-h",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--help",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH  },
+  { .name="--list-torrc-options",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name="--list-deprecated-options",
+    .command=CMD_IMMEDIATE },
+  { .name="--nt-service" },
+  { .name="-nt-service" },
+  { .name="--dbg-dump-subsystem-list",
+    .command=CMD_IMMEDIATE,
+    .quiet=QUIET_HUSH },
+  { .name=NULL },
 };
 
 /** Helper: Read a list of configuration options from the command line.  If
- * successful, or if ignore_errors is set, put them in *<b>result</b>, put the
- * commandline-only options in *<b>cmdline_result</b>, and return 0;
- * otherwise, return -1 and leave *<b>result</b> and <b>cmdline_result</b>
- * alone. */
-int
-config_parse_commandline(int argc, char **argv, int ignore_errors,
-                         config_line_t **result,
-                         config_line_t **cmdline_result)
+ * successful, return a newly allocated parsed_cmdline_t; otherwise return
+ * NULL.
+ *
+ * If <b>ignore_errors</b> is set, try to recover from all recoverable
+ * errors and return the best command line we can.
+ */
+parsed_cmdline_t *
+config_parse_commandline(int argc, char **argv, int ignore_errors)
 {
+  parsed_cmdline_t *result = tor_malloc_zero(sizeof(parsed_cmdline_t));
+  result->command = CMD_RUN_TOR;
   config_line_t *param = NULL;
 
-  config_line_t *front = NULL;
-  config_line_t **new = &front;
-
-  config_line_t *front_cmdline = NULL;
-  config_line_t **new_cmdline = &front_cmdline;
+  config_line_t **new_cmdline = &result->cmdline_opts;
+  config_line_t **new = &result->other_opts;
 
   char *s, *arg;
   int i = 1;
@@ -2509,11 +2529,19 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
     takes_argument_t want_arg = ARGUMENT_NECESSARY;
     int is_cmdline = 0;
     int j;
+    bool is_a_command = false;
 
     for (j = 0; CMDLINE_ONLY_OPTIONS[j].name != NULL; ++j) {
       if (!strcmp(argv[i], CMDLINE_ONLY_OPTIONS[j].name)) {
         is_cmdline = 1;
         want_arg = CMDLINE_ONLY_OPTIONS[j].takes_argument;
+        if (CMDLINE_ONLY_OPTIONS[j].command != CMD_RUN_TOR) {
+          is_a_command = true;
+          result->command = CMDLINE_ONLY_OPTIONS[j].command;
+        }
+        quiet_level_t quiet = CMDLINE_ONLY_OPTIONS[j].quiet;
+        if (quiet > result->quiet_level)
+          result->quiet_level = quiet;
         break;
       }
     }
@@ -2544,26 +2572,29 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
       } else {
         log_warn(LD_CONFIG,"Command-line option '%s' with no value. Failing.",
             argv[i]);
-        config_free_lines(front);
-        config_free_lines(front_cmdline);
-        return -1;
+        parsed_cmdline_free(result);
+        return NULL;
       }
     } else if (want_arg == ARGUMENT_OPTIONAL && is_last) {
       arg = tor_strdup("");
     } else {
-      arg = (want_arg != TAKES_NO_ARGUMENT) ? tor_strdup(argv[i+1]) :
+      arg = (want_arg != ARGUMENT_NONE) ? tor_strdup(argv[i+1]) :
                                               tor_strdup("");
     }
 
     param = tor_malloc_zero(sizeof(config_line_t));
     param->key = is_cmdline ? tor_strdup(argv[i]) :
-                   tor_strdup(config_expand_abbrev(&options_format, s, 1, 1));
+                 tor_strdup(config_expand_abbrev(get_options_mgr(), s, 1, 1));
     param->value = arg;
     param->command = command;
     param->next = NULL;
     log_debug(LD_CONFIG, "command line: parsed keyword '%s', value '%s'",
         param->key, param->value);
 
+    if (is_a_command) {
+      result->command_arg = param->value;
+    }
+
     if (is_cmdline) {
       *new_cmdline = param;
       new_cmdline = &((*new_cmdline)->next);
@@ -2574,17 +2605,26 @@ config_parse_commandline(int argc, char **argv, int ignore_errors,
 
     i += want_arg ? 2 : 1;
   }
-  *cmdline_result = front_cmdline;
-  *result = front;
-  return 0;
+
+  return result;
+}
+
+/** Release all storage held by <b>cmdline</b>. */
+void
+parsed_cmdline_free_(parsed_cmdline_t *cmdline)
+{
+  if (!cmdline)
+    return;
+  config_free_lines(cmdline->cmdline_opts);
+  config_free_lines(cmdline->other_opts);
+  tor_free(cmdline);
 }
 
 /** Return true iff key is a valid configuration option. */
 int
 option_is_recognized(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return (var != NULL);
+  return config_find_option_name(get_options_mgr(), key) != NULL;
 }
 
 /** Return the canonical name of a configuration option, or NULL
@@ -2592,8 +2632,7 @@ option_is_recognized(const char *key)
 const char *
 option_get_canonical_name(const char *key)
 {
-  const config_var_t *var = config_find_option(&options_format, key);
-  return var ? var->name : NULL;
+  return config_find_option_name(get_options_mgr(), key);
 }
 
 /** Return a canonical list of the options assigned for key.
@@ -2601,7 +2640,7 @@ option_get_canonical_name(const char *key)
 config_line_t *
 option_get_assignment(const or_options_t *options, const char *key)
 {
-  return config_get_assigned_option(&options_format, options, key, 1);
+  return config_get_assigned_option(get_options_mgr(), options, key, 1);
 }
 
 /** Try assigning <b>list</b> to the global options. You do this by duping
@@ -2617,44 +2656,16 @@ setopt_err_t
 options_trial_assign(config_line_t *list, unsigned flags, char **msg)
 {
   int r;
-  or_options_t *trial_options = config_dup(&options_format, get_options());
+  or_options_t *trial_options = config_dup(get_options_mgr(), get_options());
 
-  if ((r=config_assign(&options_format, trial_options,
+  if ((r=config_assign(get_options_mgr(), trial_options,
                        list, flags, msg)) < 0) {
     or_options_free(trial_options);
     return r;
   }
+  const or_options_t *cur_options = get_options();
 
-  setopt_err_t rv;
-  or_options_t *cur_options = get_options_mutable();
-
-  in_option_validation = 1;
-
-  if (options_validate(cur_options, trial_options,
-                       global_default_options, 1, msg) < 0) {
-    or_options_free(trial_options);
-    rv = SETOPT_ERR_PARSE; /*XXX make this a separate return value. */
-    goto done;
-  }
-
-  if (options_transition_allowed(cur_options, trial_options, msg) < 0) {
-    or_options_free(trial_options);
-    rv = SETOPT_ERR_TRANSITION;
-    goto done;
-  }
-  in_option_validation = 0;
-
-  if (set_options(trial_options, msg)<0) {
-    or_options_free(trial_options);
-    rv = SETOPT_ERR_SETTING;
-    goto done;
-  }
-
-  /* we liked it. put it in place. */
-  rv = SETOPT_OK;
- done:
-  in_option_validation = 0;
-  return rv;
+  return options_validate_and_set(cur_options, trial_options, msg);
 }
 
 /** Print a usage message for tor. */
@@ -2664,7 +2675,7 @@ print_usage(void)
   printf(
 "Copyright (c) 2001-2004, Roger Dingledine\n"
 "Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson\n"
-"Copyright (c) 2007-2019, The Tor Project, Inc.\n\n"
+"Copyright (c) 2007-2020, The Tor Project, Inc.\n\n"
 "tor -f <torrc> [args]\n"
 "See man page for options, or https://www.torproject.org/ for "
 "documentation.\n");
@@ -2674,48 +2685,41 @@ print_usage(void)
 static void
 list_torrc_options(void)
 {
-  int i;
-  for (i = 0; option_vars_[i].name; ++i) {
-    const config_var_t *var = &option_vars_[i];
-    if (var->type == CONFIG_TYPE_OBSOLETE ||
-        var->type == CONFIG_TYPE_LINELIST_V)
+  smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+  SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+    /* Possibly this should check listable, rather than (or in addition to)
+     * settable. See ticket 31654.
+     */
+    if (! config_var_is_settable(var)) {
+      /* This variable cannot be set, or cannot be set by this name. */
       continue;
-    printf("%s\n", var->name);
-  }
+    }
+    printf("%s\n", var->member.name);
+  } SMARTLIST_FOREACH_END(var);
+  smartlist_free(vars);
 }
 
 /** Print all deprecated but non-obsolete torrc options. */
 static void
 list_deprecated_options(void)
 {
-  const config_deprecation_t *d;
-  for (d = option_deprecation_notes_; d->name; ++d) {
-    printf("%s\n", d->name);
-  }
+  smartlist_t *deps = config_mgr_list_deprecated_vars(get_options_mgr());
+  /* Possibly this should check whether the variables are listable,
+   * but currently it does not.  See ticket 31654. */
+  SMARTLIST_FOREACH(deps, const char *, name,
+                    printf("%s\n", name));
+  smartlist_free(deps);
 }
 
 /** Print all compile-time modules and their enabled/disabled status. */
 static void
 list_enabled_modules(void)
 {
+  printf("%s: %s\n", "relay", have_module_relay() ? "yes" : "no");
   printf("%s: %s\n", "dirauth", have_module_dirauth() ? "yes" : "no");
-}
-
-/** Last value actually set by resolve_my_address. */
-static uint32_t last_resolved_addr = 0;
-
-/** Accessor for last_resolved_addr from outside this file. */
-uint32_t
-get_last_resolved_addr(void)
-{
-  return last_resolved_addr;
-}
-
-/** Reset last_resolved_addr from outside this file. */
-void
-reset_last_resolved_addr(void)
-{
-  last_resolved_addr = 0;
+  // We don't list dircache, because it cannot be enabled or disabled
+  // independently from relay.  Listing it here would proliferate
+  // test variants in test_parseconf.sh to no useful purpose.
 }
 
 /* Return true if <b>options</b> is using the default authorities, and false
@@ -2726,282 +2730,13 @@ using_default_dir_authorities(const or_options_t *options)
   return (!options->DirAuthorities && !options->AlternateDirAuthority);
 }
 
-/**
- * Attempt getting our non-local (as judged by tor_addr_is_internal()
- * function) IP address using following techniques, listed in
- * order from best (most desirable, try first) to worst (least
- * desirable, try if everything else fails).
- *
- * First, attempt using <b>options-\>Address</b> to get our
- * non-local IP address.
- *
- * If <b>options-\>Address</b> represents a non-local IP address,
- * consider it ours.
- *
- * If <b>options-\>Address</b> is a DNS name that resolves to
- * a non-local IP address, consider this IP address ours.
- *
- * If <b>options-\>Address</b> is NULL, fall back to getting local
- * hostname and using it in above-described ways to try and
- * get our IP address.
- *
- * In case local hostname cannot be resolved to a non-local IP
- * address, try getting an IP address of network interface
- * in hopes it will be non-local one.
- *
- * Fail if one or more of the following is true:
- *   - DNS name in <b>options-\>Address</b> cannot be resolved.
- *   - <b>options-\>Address</b> is a local host address.
- *   - Attempt at getting local hostname fails.
- *   - Attempt at getting network interface address fails.
- *
- * Return 0 if all is well, or -1 if we can't find a suitable
- * public IP address.
- *
- * If we are returning 0:
- *   - Put our public IP address (in host order) into *<b>addr_out</b>.
- *   - If <b>method_out</b> is non-NULL, set *<b>method_out</b> to a static
- *     string describing how we arrived at our answer.
- *      - "CONFIGURED" - parsed from IP address string in
- *        <b>options-\>Address</b>
- *      - "RESOLVED" - resolved from DNS name in <b>options-\>Address</b>
- *      - "GETHOSTNAME" - resolved from a local hostname.
- *      - "INTERFACE" - retrieved from a network interface.
- *   - If <b>hostname_out</b> is non-NULL, and we resolved a hostname to
- *     get our address, set *<b>hostname_out</b> to a newly allocated string
- *     holding that hostname. (If we didn't get our address by resolving a
- *     hostname, set *<b>hostname_out</b> to NULL.)
- *
- * XXXX ipv6
- */
-int
-resolve_my_address(int warn_severity, const or_options_t *options,
-                   uint32_t *addr_out,
-                   const char **method_out, char **hostname_out)
-{
-  struct in_addr in;
-  uint32_t addr; /* host order */
-  char hostname[256];
-  const char *method_used;
-  const char *hostname_used;
-  int explicit_ip=1;
-  int explicit_hostname=1;
-  int from_interface=0;
-  char *addr_string = NULL;
-  const char *address = options->Address;
-  int notice_severity = warn_severity <= LOG_NOTICE ?
-                          LOG_NOTICE : warn_severity;
-
-  tor_addr_t myaddr;
-  tor_assert(addr_out);
-
-  /*
-   * Step one: Fill in 'hostname' to be our best guess.
-   */
-
-  if (address && *address) {
-    strlcpy(hostname, address, sizeof(hostname));
-  } else { /* then we need to guess our address */
-    explicit_ip = 0; /* it's implicit */
-    explicit_hostname = 0; /* it's implicit */
-
-    if (tor_gethostname(hostname, sizeof(hostname)) < 0) {
-      log_fn(warn_severity, LD_NET,"Error obtaining local hostname");
-      return -1;
-    }
-    log_debug(LD_CONFIG, "Guessed local host name as '%s'", hostname);
-  }
-
-  /*
-   * Step two: Now that we know 'hostname', parse it or resolve it. If
-   * it doesn't parse or resolve, look at the interface address. Set 'addr'
-   * to be our (host-order) 32-bit answer.
-   */
-
-  if (tor_inet_aton(hostname, &in) == 0) {
-    /* then we have to resolve it */
-    explicit_ip = 0;
-    if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */
-      uint32_t interface_ip; /* host order */
-
-      if (explicit_hostname) {
-        log_fn(warn_severity, LD_CONFIG,
-               "Could not resolve local Address '%s'. Failing.", hostname);
-        return -1;
-      }
-      log_fn(notice_severity, LD_CONFIG,
-             "Could not resolve guessed local hostname '%s'. "
-             "Trying something else.", hostname);
-      if (get_interface_address(warn_severity, &interface_ip)) {
-        log_fn(warn_severity, LD_CONFIG,
-               "Could not get local interface IP address. Failing.");
-        return -1;
-      }
-      from_interface = 1;
-      addr = interface_ip;
-      log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for "
-             "local interface. Using that.", fmt_addr32(addr));
-      strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
-    } else { /* resolved hostname into addr */
-      tor_addr_from_ipv4h(&myaddr, addr);
-
-      if (!explicit_hostname &&
-          tor_addr_is_internal(&myaddr, 0)) {
-        tor_addr_t interface_ip;
-
-        log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' "
-               "resolves to a private IP address (%s). Trying something "
-               "else.", hostname, fmt_addr32(addr));
-
-        if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) {
-          log_fn(warn_severity, LD_CONFIG,
-                 "Could not get local interface IP address. Too bad.");
-        } else if (tor_addr_is_internal(&interface_ip, 0)) {
-          log_fn(notice_severity, LD_CONFIG,
-                 "Interface IP address '%s' is a private address too. "
-                 "Ignoring.", fmt_addr(&interface_ip));
-        } else {
-          from_interface = 1;
-          addr = tor_addr_to_ipv4h(&interface_ip);
-          log_fn(notice_severity, LD_CONFIG,
-                 "Learned IP address '%s' for local interface."
-                 " Using that.", fmt_addr32(addr));
-          strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
-        }
-      }
-    }
-  } else {
-    addr = ntohl(in.s_addr); /* set addr so that addr_string is not
-                              * illformed */
-  }
-
-  /*
-   * Step three: Check whether 'addr' is an internal IP address, and error
-   * out if it is and we don't want that.
-   */
-
-  tor_addr_from_ipv4h(&myaddr,addr);
-
-  addr_string = tor_dup_ip(addr);
-  if (tor_addr_is_internal(&myaddr, 0)) {
-    /* make sure we're ok with publishing an internal IP */
-    if (using_default_dir_authorities(options)) {
-      /* if they are using the default authorities, disallow internal IPs
-       * always. */
-      log_fn(warn_severity, LD_CONFIG,
-             "Address '%s' resolves to private IP address '%s'. "
-             "Tor servers that use the default DirAuthorities must have "
-             "public IP addresses.", hostname, addr_string);
-      tor_free(addr_string);
-      return -1;
-    }
-    if (!explicit_ip) {
-      /* even if they've set their own authorities, require an explicit IP if
-       * they're using an internal address. */
-      log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private "
-             "IP address '%s'. Please set the Address config option to be "
-             "the IP address you want to use.", hostname, addr_string);
-      tor_free(addr_string);
-      return -1;
-    }
-  }
-
-  /*
-   * Step four: We have a winner! 'addr' is our answer for sure, and
-   * 'addr_string' is its string form. Fill out the various fields to
-   * say how we decided it.
-   */
-
-  log_debug(LD_CONFIG, "Resolved Address to '%s'.", addr_string);
-
-  if (explicit_ip) {
-    method_used = "CONFIGURED";
-    hostname_used = NULL;
-  } else if (explicit_hostname) {
-    method_used = "RESOLVED";
-    hostname_used = hostname;
-  } else if (from_interface) {
-    method_used = "INTERFACE";
-    hostname_used = NULL;
-  } else {
-    method_used = "GETHOSTNAME";
-    hostname_used = hostname;
-  }
-
-  *addr_out = addr;
-  if (method_out)
-    *method_out = method_used;
-  if (hostname_out)
-    *hostname_out = hostname_used ? tor_strdup(hostname_used) : NULL;
-
-  /*
-   * Step five: Check if the answer has changed since last time (or if
-   * there was no last time), and if so call various functions to keep
-   * us up-to-date.
-   */
-
-  if (last_resolved_addr && last_resolved_addr != *addr_out) {
-    /* Leave this as a notice, regardless of the requested severity,
-     * at least until dynamic IP address support becomes bulletproof. */
-    log_notice(LD_NET,
-               "Your IP address seems to have changed to %s "
-               "(METHOD=%s%s%s). Updating.",
-               addr_string, method_used,
-               hostname_used ? " HOSTNAME=" : "",
-               hostname_used ? hostname_used : "");
-    ip_address_changed(0);
-  }
-
-  if (last_resolved_addr != *addr_out) {
-    control_event_server_status(LOG_NOTICE,
-                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=%s%s%s",
-                                addr_string, method_used,
-                                hostname_used ? " HOSTNAME=" : "",
-                                hostname_used ? hostname_used : "");
-  }
-  last_resolved_addr = *addr_out;
-
-  /*
-   * And finally, clean up and return success.
-   */
-
-  tor_free(addr_string);
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is judged to be on the same network as us, or
- * on a private network.
- */
-MOCK_IMPL(int,
-is_local_addr, (const tor_addr_t *addr))
-{
-  if (tor_addr_is_internal(addr, 0))
-    return 1;
-  /* Check whether ip is on the same /24 as we are. */
-  if (get_options()->EnforceDistinctSubnets == 0)
-    return 0;
-  if (tor_addr_family(addr) == AF_INET) {
-    uint32_t ip = tor_addr_to_ipv4h(addr);
-
-    /* It's possible that this next check will hit before the first time
-     * resolve_my_address actually succeeds.  (For clients, it is likely that
-     * resolve_my_address will never be called at all).  In those cases,
-     * last_resolved_addr will be 0, and so checking to see whether ip is on
-     * the same /24 as last_resolved_addr will be the same as checking whether
-     * it was on net 0, which is already done by tor_addr_is_internal.
-     */
-    if ((last_resolved_addr & (uint32_t)0xffffff00ul)
-        == (ip & (uint32_t)0xffffff00ul))
-      return 1;
-  }
-  return 0;
-}
-
 /** Return a new empty or_options_t.  Used for testing. */
 or_options_t *
 options_new(void)
 {
-  return config_new(&options_format);
+  or_options_t *options = config_new(get_options_mgr());
+  options->command = CMD_RUN_TOR;
+  return options;
 }
 
 /** Set <b>options</b> to hold reasonable defaults for most options.
@@ -3009,7 +2744,17 @@ options_new(void)
 void
 options_init(or_options_t *options)
 {
-  config_init(&options_format, options);
+  config_init(get_options_mgr(), options);
+  config_line_t *dflts = get_options_defaults();
+  char *msg=NULL;
+  if (config_assign(get_options_mgr(), options, dflts,
+                    CAL_WARN_DEPRECATIONS, &msg)<0) {
+    log_err(LD_BUG, "Unable to set default options: %s", msg);
+    tor_free(msg);
+    tor_assert_unreached();
+  }
+  config_free_lines(dflts);
+  tor_free(msg);
 }
 
 /** Return a string containing a possible configuration file that would give
@@ -3039,7 +2784,7 @@ options_dump(const or_options_t *options, int how_to_dump)
       return NULL;
   }
 
-  return config_dump(&options_format, use_defaults, options, minimal, 0);
+  return config_dump(get_options_mgr(), use_defaults, options, minimal, 0);
 }
 
 /** Return 0 if every element of sl is a string holding a decimal
@@ -3069,8 +2814,8 @@ validate_ports_csv(smartlist_t *sl, const char *name, char **msg)
  * a complaint into *<b>msg</b> using string <b>desc</b>, and return -1.
  * Else return 0.
  */
-static int
-ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
+int
+config_ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
 {
   if (*value > ROUTER_MAX_DECLARED_BANDWIDTH) {
     /* This handles an understandable special case where somebody says "2gb"
@@ -3086,48 +2831,6 @@ ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg)
   return 0;
 }
 
-/** Parse an authority type from <b>options</b>-\>PublishServerDescriptor
- * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1"
- * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge".
- * Treat "0" as "".
- * Return 0 on success or -1 if not a recognized authority type (in which
- * case the value of PublishServerDescriptor_ is undefined). */
-static int
-compute_publishserverdescriptor(or_options_t *options)
-{
-  smartlist_t *list = options->PublishServerDescriptor;
-  dirinfo_type_t *auth = &options->PublishServerDescriptor_;
-  *auth = NO_DIRINFO;
-  if (!list) /* empty list, answer is none */
-    return 0;
-  SMARTLIST_FOREACH_BEGIN(list, const char *, string) {
-    if (!strcasecmp(string, "v1"))
-      log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because "
-                          "there are no v1 directory authorities anymore.");
-    else if (!strcmp(string, "1"))
-      if (options->BridgeRelay)
-        *auth |= BRIDGE_DIRINFO;
-      else
-        *auth |= V3_DIRINFO;
-    else if (!strcasecmp(string, "v2"))
-      log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because "
-                          "there are no v2 directory authorities anymore.");
-    else if (!strcasecmp(string, "v3"))
-      *auth |= V3_DIRINFO;
-    else if (!strcasecmp(string, "bridge"))
-      *auth |= BRIDGE_DIRINFO;
-    else if (!strcasecmp(string, "hidserv"))
-      log_warn(LD_CONFIG,
-               "PublishServerDescriptor hidserv is invalid. See "
-               "PublishHidServDescriptors.");
-    else if (!strcasecmp(string, "") || !strcmp(string, "0"))
-      /* no authority */;
-    else
-      return -1;
-  } SMARTLIST_FOREACH_END(string);
-  return 0;
-}
-
 /** Lowest allowable value for RendPostPeriod; if this is too low, hidden
  * services can overload the directory system. */
 #define MIN_REND_POST_PERIOD (10*60)
@@ -3160,23 +2863,67 @@ compute_publishserverdescriptor(or_options_t *options)
  * */
 #define RECOMMENDED_MIN_CIRCUIT_BUILD_TIMEOUT (10)
 
-static int
-options_validate_cb(void *old_options, void *options, void *default_options,
-                    int from_setconf, char **msg)
+/**
+ * Validate <b>new_options</b>.  If it is valid, and it is a reasonable
+ * replacement for <b>old_options</b>, replace the previous value of the
+ * global options, and return return SETOPT_OK.
+ *
+ * If it is not valid, then free <b>new_options</b>, set *<b>msg_out</b> to a
+ * newly allocated error message, and return an error code.
+ */
+static setopt_err_t
+options_validate_and_set(const or_options_t *old_options,
+                         or_options_t *new_options,
+                         char **msg_out)
 {
+  setopt_err_t rv;
+  validation_status_t vs;
+
   in_option_validation = 1;
-  int rv = options_validate(old_options, options, default_options,
-                          from_setconf, msg);
+  vs = config_validate(get_options_mgr(), old_options, new_options, msg_out);
+
+  if (vs == VSTAT_TRANSITION_ERR) {
+    rv = SETOPT_ERR_TRANSITION;
+    goto err;
+  } else if (vs < 0) {
+    rv = SETOPT_ERR_PARSE;
+    goto err;
+  }
+  in_option_validation = 0;
+
+  if (set_options(new_options, msg_out)) {
+    rv = SETOPT_ERR_SETTING;
+    goto err;
+  }
+
+  rv = SETOPT_OK;
+  new_options = NULL; /* prevent free */
+ err:
   in_option_validation = 0;
+  tor_assert(new_options == NULL || rv != SETOPT_OK);
+  or_options_free(new_options);
   return rv;
 }
 
-/** Callback to free an or_options_t */
-static void
-options_free_cb(void *options)
+#ifdef TOR_UNIT_TESTS
+/**
+ * Return 0 if every setting in <b>options</b> is reasonable, is a
+ * permissible transition from <b>old_options</b>, and none of the
+ * testing-only settings differ from <b>default_options</b> unless in
+ * testing mode.  Else return -1.  Should have no side effects, except for
+ * normalizing the contents of <b>options</b>.
+ *
+ * On error, tor_strdup an error explanation into *<b>msg</b>.
+ */
+int
+options_validate(const or_options_t *old_options, or_options_t *options,
+                 char **msg)
 {
-  or_options_free_(options);
+  validation_status_t vs;
+  vs = config_validate(get_options_mgr(), old_options, options, msg);
+  return vs < 0 ? -1 : 0;
 }
+#endif /* defined(TOR_UNIT_TESTS) */
 
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
@@ -3198,7 +2945,7 @@ options_free_cb(void *options)
  */
 static int
 warn_if_option_path_is_relative(const char *option,
-                                char *filepath)
+                                const char *filepath)
 {
   if (filepath && path_is_relative(filepath)) {
     char *abs_path = make_path_absolute(filepath);
@@ -3211,34 +2958,29 @@ warn_if_option_path_is_relative(const char *option,
 }
 
 /** Scan <b>options</b> for occurrences of relative file/directory
- * path and log a warning whenever it is found.
+ * paths and log a warning whenever one is found.
  *
  * Return 1 if there were relative paths; 0 otherwise.
  */
 static int
-warn_about_relative_paths(or_options_t *options)
+warn_about_relative_paths(const or_options_t *options)
 {
   tor_assert(options);
   int n = 0;
+  const config_mgr_t *mgr = get_options_mgr();
 
-  n += warn_if_option_path_is_relative("CookieAuthFile",
-                                       options->CookieAuthFile);
-  n += warn_if_option_path_is_relative("ExtORPortCookieAuthFile",
-                                       options->ExtORPortCookieAuthFile);
-  n += warn_if_option_path_is_relative("DirPortFrontPage",
-                                       options->DirPortFrontPage);
-  n += warn_if_option_path_is_relative("V3BandwidthsFile",
-                                       options->V3BandwidthsFile);
-  n += warn_if_option_path_is_relative("ControlPortWriteToFile",
-                                       options->ControlPortWriteToFile);
-  n += warn_if_option_path_is_relative("GeoIPFile",options->GeoIPFile);
-  n += warn_if_option_path_is_relative("GeoIPv6File",options->GeoIPv6File);
-  n += warn_if_option_path_is_relative("Log",options->DebugLogFile);
-  n += warn_if_option_path_is_relative("AccelDir",options->AccelDir);
-  n += warn_if_option_path_is_relative("DataDirectory",options->DataDirectory);
-  n += warn_if_option_path_is_relative("PidFile",options->PidFile);
-  n += warn_if_option_path_is_relative("ClientOnionAuthDir",
-                                        options->ClientOnionAuthDir);
+  smartlist_t *vars = config_mgr_list_vars(mgr);
+  SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, cv) {
+    config_line_t *line;
+    if (cv->member.type != CONFIG_TYPE_FILENAME)
+      continue;
+    const char *name = cv->member.name;
+    line = config_get_assigned_option(mgr, options, name, 0);
+    if (line)
+      n += warn_if_option_path_is_relative(name, line->value);
+    config_free_lines(line);
+  } SMARTLIST_FOREACH_END(cv);
+  smartlist_free(vars);
 
   for (config_line_t *hs_line = options->RendConfigLines; hs_line;
        hs_line = hs_line->next) {
@@ -3265,6 +3007,10 @@ options_validate_scheduler(or_options_t *options, char **msg)
            "can be used or set at least one value.");
   }
   /* Ok, we do have scheduler types, validate them. */
+  if (options->SchedulerTypes_) {
+    SMARTLIST_FOREACH(options->SchedulerTypes_, int *, iptr, tor_free(iptr));
+    smartlist_free(options->SchedulerTypes_);
+  }
   options->SchedulerTypes_ = smartlist_new();
   SMARTLIST_FOREACH_BEGIN(options->Schedulers, const char *, type) {
     int *sched_type;
@@ -3357,25 +3103,20 @@ options_validate_single_onion(or_options_t *options, char **msg)
   return 0;
 }
 
-/** Return 0 if every setting in <b>options</b> is reasonable, is a
- * permissible transition from <b>old_options</b>, and none of the
- * testing-only settings differ from <b>default_options</b> unless in
- * testing mode.  Else return -1.  Should have no side effects, except for
- * normalizing the contents of <b>options</b>.
- *
- * On error, tor_strdup an error explanation into *<b>msg</b>.
- *
- * XXX
- * If <b>from_setconf</b>, we were called by the controller, and our
- * Log line should stay empty. If it's 0, then give us a default log
- * if there are no logs defined.
+/**
+ * Legacy validation/normalization callback for or_options_t.  See
+ * legacy_validate_fn_t for more information.
  */
-STATIC int
-options_validate(or_options_t *old_options, or_options_t *options,
-                 or_options_t *default_options, int from_setconf, char **msg)
+static int
+options_validate_cb(const void *old_options_, void *options_, char **msg)
 {
+  if (old_options_)
+    CHECK_OPTIONS_MAGIC(old_options_);
+  CHECK_OPTIONS_MAGIC(options_);
+  const or_options_t *old_options = old_options_;
+  or_options_t *options = options_;
+
   config_line_t *cl;
-  const char *uname = get_uname();
   int n_ports=0;
   int world_writable_control_socket=0;
 
@@ -3386,22 +3127,30 @@ options_validate(or_options_t *old_options, or_options_t *options,
                   &world_writable_control_socket) < 0)
     return -1;
 
+#ifndef HAVE_SYS_UN_H
+  if (options->ControlSocket || options->ControlSocketsGroupWritable) {
+    *msg = tor_strdup("Unix domain sockets (ControlSocket) not supported "
+                      "on this OS/with this build.");
+    return -1;
+  }
+#else /* defined(HAVE_SYS_UN_H) */
+  if (options->ControlSocketsGroupWritable && !options->ControlSocket) {
+    *msg = tor_strdup("Setting ControlSocketGroupWritable without setting "
+                      "a ControlSocket makes no sense.");
+    return -1;
+  }
+#endif /* !defined(HAVE_SYS_UN_H) */
+
   /* Set UseEntryGuards from the configured value, before we check it below.
    * We change UseEntryGuards when it's incompatible with other options,
    * but leave UseEntryGuards_option with the original value.
    * Always use the value of UseEntryGuards, not UseEntryGuards_option. */
   options->UseEntryGuards = options->UseEntryGuards_option;
 
-  if (server_mode(options) &&
-      (!strcmpstart(uname, "Windows 95") ||
-       !strcmpstart(uname, "Windows 98") ||
-       !strcmpstart(uname, "Windows Me"))) {
-    log_warn(LD_CONFIG, "Tor is running as a server, but you are "
-        "running %s; this probably won't work. See "
-        "https://www.torproject.org/docs/faq.html#BestOSForRelay "
-        "for details.", uname);
-  }
+  if (options_validate_relay_os(old_options, options, msg) < 0)
+    return -1;
 
+  /* 31851: OutboundBindAddressExit is unused in client mode */
   if (parse_outbound_addresses(options, 1, msg) < 0)
     return -1;
 
@@ -3416,59 +3165,16 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "with relative paths.");
   }
 
-  if (options->Nickname == NULL) {
-    if (server_mode(options)) {
-      options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
-    }
-  } else {
-    if (!is_legal_nickname(options->Nickname)) {
-      tor_asprintf(msg,
-          "Nickname '%s', nicknames must be between 1 and 19 characters "
-          "inclusive, and must contain only the characters [a-zA-Z0-9].",
-          options->Nickname);
-      return -1;
-    }
-  }
-
-  if (server_mode(options) && !options->ContactInfo) {
-    log_warn(LD_CONFIG,
-             "Your ContactInfo config option is not set. Please strongly "
-             "consider setting it, so we can contact you if your relay is "
-             "misconfigured, end-of-life, or something else goes wrong. "
-             "It is also possible that your relay might get rejected from "
-             "the network due to a missing valid contact address.");
-  }
-
-  const char *ContactInfo = options->ContactInfo;
-  if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
-    REJECT("ContactInfo config option must be UTF-8.");
+  if (options_validate_relay_info(old_options, options, msg) < 0)
+    return -1;
 
-  /* Special case on first boot if no Log options are given. */
-  if (!options->Logs && !options->RunAsDaemon && !from_setconf) {
-    if (quiet_level == 0)
-      config_line_append(&options->Logs, "Log", "notice stdout");
-    else if (quiet_level == 1)
-      config_line_append(&options->Logs, "Log", "warn stdout");
-  }
+  /* 31851: this function is currently a no-op in client mode */
+  check_network_configuration(server_mode(options));
 
   /* Validate the tor_log(s) */
   if (options_init_logs(old_options, options, 1)<0)
     REJECT("Failed to validate Log options. See logs for details.");
 
-  if (authdir_mode(options)) {
-    /* confirm that our address isn't broken, so we can complain now */
-    uint32_t tmp;
-    if (resolve_my_address(LOG_WARN, options, &tmp, NULL, NULL) < 0)
-      REJECT("Failed to resolve/guess local address. See logs for details.");
-  }
-
-  if (server_mode(options) && options->RendConfigLines)
-    log_warn(LD_CONFIG,
-        "Tor is currently configured as a relay and a hidden service. "
-        "That's not very secure: you should probably run your hidden service "
-        "in a separate Tor process, at least -- see "
-        "https://trac.torproject.org/8742");
-
   /* XXXX require that the only port not be DirPort? */
   /* XXXX require that at least one port be listened-upon. */
   if (n_ports == 0 && !options->RendConfigLines)
@@ -3483,13 +3189,13 @@ options_validate(or_options_t *old_options, or_options_t *options,
     if (!strcasecmp(options->TransProxyType, "default")) {
       options->TransProxyType_parsed = TPT_DEFAULT;
     } else if (!strcasecmp(options->TransProxyType, "pf-divert")) {
-#if !defined(OpenBSD) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined(DARWIN)
       /* Later versions of OS X have pf */
       REJECT("pf-divert is a OpenBSD-specific "
              "and OS X/Darwin-specific feature.");
 #else
       options->TransProxyType_parsed = TPT_PF_DIVERT;
-#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
+#endif /* !defined(OpenBSD) && !defined(DARWIN) */
     } else if (!strcasecmp(options->TransProxyType, "tproxy")) {
 #if !defined(__linux__)
       REJECT("TPROXY is a Linux-specific feature.");
@@ -3513,7 +3219,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
       REJECT("Cannot use TransProxyType without any valid TransPort.");
     }
   }
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   if (options->TransPort_set)
     REJECT("TransPort is disabled in this build.");
 #endif /* defined(USE_TRANSPARENT) */
@@ -3547,69 +3253,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "features to be broken in unpredictable ways.");
   }
 
-  for (cl = options->RecommendedPackages; cl; cl = cl->next) {
-    if (! validate_recommended_package_line(cl->value)) {
-      log_warn(LD_CONFIG, "Invalid RecommendedPackage line %s will be ignored",
-               escaped(cl->value));
-    }
-  }
-
-  if (options->AuthoritativeDir) {
-    if (!options->ContactInfo && !options->TestingTorNetwork)
-      REJECT("Authoritative directory servers must set ContactInfo");
-    if (!options->RecommendedClientVersions)
-      options->RecommendedClientVersions =
-        config_lines_dup(options->RecommendedVersions);
-    if (!options->RecommendedServerVersions)
-      options->RecommendedServerVersions =
-        config_lines_dup(options->RecommendedVersions);
-    if (options->VersioningAuthoritativeDir &&
-        (!options->RecommendedClientVersions ||
-         !options->RecommendedServerVersions))
-      REJECT("Versioning authoritative dir servers must set "
-             "Recommended*Versions.");
-
-#ifdef HAVE_MODULE_DIRAUTH
-    char *t;
-    /* Call these functions to produce warnings only. */
-    t = format_recommended_version_list(options->RecommendedClientVersions, 1);
-    tor_free(t);
-    t = format_recommended_version_list(options->RecommendedServerVersions, 1);
-    tor_free(t);
-#endif
-
-    if (options->UseEntryGuards) {
-      log_info(LD_CONFIG, "Authoritative directory servers can't set "
-               "UseEntryGuards. Disabling.");
-      options->UseEntryGuards = 0;
-    }
-    if (!options->DownloadExtraInfo && authdir_mode_v3(options)) {
-      log_info(LD_CONFIG, "Authoritative directories always try to download "
-               "extra-info documents. Setting DownloadExtraInfo.");
-      options->DownloadExtraInfo = 1;
-    }
-    if (!(options->BridgeAuthoritativeDir ||
-          options->V3AuthoritativeDir))
-      REJECT("AuthoritativeDir is set, but none of "
-             "(Bridge/V3)AuthoritativeDir is set.");
-    /* If we have a v3bandwidthsfile and it's broken, complain on startup */
-    if (options->V3BandwidthsFile && !old_options) {
-      dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
-    }
-    /* same for guardfraction file */
-    if (options->GuardfractionFile && !old_options) {
-      dirserv_read_guardfraction_file(options->GuardfractionFile, NULL);
-    }
-  }
-
-  if (options->AuthoritativeDir && !options->DirPort_set)
-    REJECT("Running as authoritative directory, but no DirPort set.");
-
-  if (options->AuthoritativeDir && !options->ORPort_set)
-    REJECT("Running as authoritative directory, but no ORPort set.");
-
-  if (options->AuthoritativeDir && options->ClientOnly)
-    REJECT("Running as authoritative directory, but ClientOnly also set.");
+  if (options_validate_dirauth_mode(old_options, options, msg) < 0)
+    return -1;
 
   if (options->FetchDirInfoExtraEarly && !options->FetchDirInfoEarly)
     REJECT("FetchDirInfoExtraEarly requires that you also set "
@@ -3748,49 +3393,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
     return -1;
   }
 
-  if (compute_publishserverdescriptor(options) < 0) {
-    tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor");
+  if (options_validate_publish_server(old_options, options, msg) < 0)
     return -1;
-  }
-
-  if ((options->BridgeRelay
-        || options->PublishServerDescriptor_ & BRIDGE_DIRINFO)
-      && (options->PublishServerDescriptor_ & V3_DIRINFO)) {
-    REJECT("Bridges are not supposed to publish router descriptors to the "
-           "directory authorities. Please correct your "
-           "PublishServerDescriptor line.");
-  }
-
-  if (options->BridgeRelay && options->DirPort_set) {
-    log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling "
-             "DirPort");
-    config_free_lines(options->DirPort_lines);
-    options->DirPort_lines = NULL;
-    options->DirPort_set = 0;
-  }
 
-  if (server_mode(options) && options->ConnectionPadding != -1) {
-    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
-  }
-
-  if (server_mode(options) && options->ReducedConnectionPadding != 0) {
-    REJECT("Relays cannot set ReducedConnectionPadding. ");
-  }
-
-  if (options->BridgeDistribution) {
-    if (!options->BridgeRelay) {
-      REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
-    }
-    if (check_bridge_distribution_setting(options->BridgeDistribution) < 0) {
-      REJECT("Invalid BridgeDistribution value.");
-    }
-  }
-
-  if (options->MinUptimeHidServDirectoryV2 < 0) {
-    log_warn(LD_CONFIG, "MinUptimeHidServDirectoryV2 option must be at "
-                        "least 0 seconds. Changing to 0.");
-    options->MinUptimeHidServDirectoryV2 = 0;
-  }
+  if (options_validate_relay_padding(old_options, options, msg) < 0)
+    return -1;
 
   const int min_rendpostperiod =
     options->TestingTorNetwork ?
@@ -3829,7 +3436,7 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "UseEntryGuards is disabled, but you have configured one or more "
              "hidden services on this Tor instance.  Your hidden services "
              "will be very easy to locate using a well-known attack -- see "
-             "http://freehaven.net/anonbib/#hs-attack06 for details.");
+             "https://freehaven.net/anonbib/#hs-attack06 for details.");
   }
 
   if (options->NumPrimaryGuards && options->NumEntryGuards &&
@@ -3899,6 +3506,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
            "default.");
   }
 
+  if (options->DormantClientTimeout < 10*60 && !options->TestingTorNetwork) {
+    REJECT("DormantClientTimeout is too low. It must be at least 10 minutes.");
+  }
+
   if (options->PathBiasNoticeRate > 1.0) {
     tor_asprintf(msg,
               "PathBiasNoticeRate is too high. "
@@ -3950,7 +3561,8 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   if (options->HeartbeatPeriod &&
-      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD) {
+      options->HeartbeatPeriod < MIN_HEARTBEAT_PERIOD &&
+      !options->TestingTorNetwork) {
     log_warn(LD_CONFIG, "HeartbeatPeriod option is too short; "
              "raising to %d seconds.", MIN_HEARTBEAT_PERIOD);
     options->HeartbeatPeriod = MIN_HEARTBEAT_PERIOD;
@@ -3959,134 +3571,24 @@ options_validate(or_options_t *old_options, or_options_t *options,
   if (options->KeepalivePeriod < 1)
     REJECT("KeepalivePeriod option must be positive.");
 
-  if (ensure_bandwidth_cap(&options->BandwidthRate,
+  if (config_ensure_bandwidth_cap(&options->BandwidthRate,
                            "BandwidthRate", msg) < 0)
     return -1;
-  if (ensure_bandwidth_cap(&options->BandwidthBurst,
+  if (config_ensure_bandwidth_cap(&options->BandwidthBurst,
                            "BandwidthBurst", msg) < 0)
     return -1;
-  if (ensure_bandwidth_cap(&options->MaxAdvertisedBandwidth,
-                           "MaxAdvertisedBandwidth", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->RelayBandwidthRate,
-                           "RelayBandwidthRate", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->RelayBandwidthBurst,
-                           "RelayBandwidthBurst", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->PerConnBWRate,
-                           "PerConnBWRate", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->PerConnBWBurst,
-                           "PerConnBWBurst", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->AuthDirFastGuarantee,
-                           "AuthDirFastGuarantee", msg) < 0)
-    return -1;
-  if (ensure_bandwidth_cap(&options->AuthDirGuardBWGuarantee,
-                           "AuthDirGuardBWGuarantee", msg) < 0)
-    return -1;
 
-  if (options->RelayBandwidthRate && !options->RelayBandwidthBurst)
-    options->RelayBandwidthBurst = options->RelayBandwidthRate;
-  if (options->RelayBandwidthBurst && !options->RelayBandwidthRate)
-    options->RelayBandwidthRate = options->RelayBandwidthBurst;
-
-  if (server_mode(options)) {
-    const unsigned required_min_bw =
-      public_server_mode(options) ?
-       RELAY_REQUIRED_MIN_BANDWIDTH : BRIDGE_REQUIRED_MIN_BANDWIDTH;
-    const char * const optbridge =
-      public_server_mode(options) ? "" : "bridge ";
-    if (options->BandwidthRate < required_min_bw) {
-      tor_asprintf(msg,
-                       "BandwidthRate is set to %d bytes/second. "
-                       "For %sservers, it must be at least %u.",
-                       (int)options->BandwidthRate, optbridge,
-                       required_min_bw);
-      return -1;
-    } else if (options->MaxAdvertisedBandwidth <
-               required_min_bw/2) {
-      tor_asprintf(msg,
-                       "MaxAdvertisedBandwidth is set to %d bytes/second. "
-                       "For %sservers, it must be at least %u.",
-                       (int)options->MaxAdvertisedBandwidth, optbridge,
-                       required_min_bw/2);
-      return -1;
-    }
-    if (options->RelayBandwidthRate &&
-      options->RelayBandwidthRate < required_min_bw) {
-      tor_asprintf(msg,
-                       "RelayBandwidthRate is set to %d bytes/second. "
-                       "For %sservers, it must be at least %u.",
-                       (int)options->RelayBandwidthRate, optbridge,
-                       required_min_bw);
-      return -1;
-    }
-  }
-
-  if (options->RelayBandwidthRate > options->RelayBandwidthBurst)
-    REJECT("RelayBandwidthBurst must be at least equal "
-           "to RelayBandwidthRate.");
+  if (options_validate_relay_bandwidth(old_options, options, msg) < 0)
+    return -1;
 
   if (options->BandwidthRate > options->BandwidthBurst)
     REJECT("BandwidthBurst must be at least equal to BandwidthRate.");
 
-  /* if they set relaybandwidth* really high but left bandwidth*
-   * at the default, raise the defaults. */
-  if (options->RelayBandwidthRate > options->BandwidthRate)
-    options->BandwidthRate = options->RelayBandwidthRate;
-  if (options->RelayBandwidthBurst > options->BandwidthBurst)
-    options->BandwidthBurst = options->RelayBandwidthBurst;
-
-  if (accounting_parse_options(options, 1)<0)
-    REJECT("Failed to parse accounting options. See logs for details.");
-
-  if (options->AccountingMax) {
-    if (options->RendConfigLines && server_mode(options)) {
-      log_warn(LD_CONFIG, "Using accounting with a hidden service and an "
-               "ORPort is risky: your hidden service(s) and your public "
-               "address will all turn off at the same time, which may alert "
-               "observers that they are being run by the same party.");
-    } else if (config_count_key(options->RendConfigLines,
-                                "HiddenServiceDir") > 1) {
-      log_warn(LD_CONFIG, "Using accounting with multiple hidden services is "
-               "risky: they will all turn off at the same time, which may "
-               "alert observers that they are being run by the same party.");
-    }
-  }
-
-  options->AccountingRule = ACCT_MAX;
-  if (options->AccountingRule_option) {
-    if (!strcmp(options->AccountingRule_option, "sum"))
-      options->AccountingRule = ACCT_SUM;
-    else if (!strcmp(options->AccountingRule_option, "max"))
-      options->AccountingRule = ACCT_MAX;
-    else if (!strcmp(options->AccountingRule_option, "in"))
-      options->AccountingRule = ACCT_IN;
-    else if (!strcmp(options->AccountingRule_option, "out"))
-      options->AccountingRule = ACCT_OUT;
-    else
-      REJECT("AccountingRule must be 'sum', 'max', 'in', or 'out'");
-  }
-
-  if (options->DirPort_set && !options->DirCache) {
-    REJECT("DirPort configured but DirCache disabled. DirPort requires "
-           "DirCache.");
-  }
-
-  if (options->BridgeRelay && !options->DirCache) {
-    REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires "
-           "DirCache.");
-  }
+  if (options_validate_relay_accounting(old_options, options, msg) < 0)
+    return -1;
 
-  if (server_mode(options)) {
-    char *dircache_msg = NULL;
-    if (have_enough_mem_for_dircache(options, 0, &dircache_msg)) {
-      log_warn(LD_CONFIG, "%s", dircache_msg);
-      tor_free(dircache_msg);
-    }
-  }
+  if (options_validate_relay_mode(old_options, options, msg) < 0)
+    return -1;
 
   if (options->HTTPProxy) { /* parse it now */
     if (tor_addr_port_lookup(options->HTTPProxy,
@@ -4136,19 +3638,28 @@ options_validate(or_options_t *old_options, or_options_t *options,
     }
   }
 
+  if (options->TCPProxy) {
+    int res = parse_tcp_proxy_line(options->TCPProxy, options, msg);
+    if (res < 0) {
+      return res;
+    }
+  }
+
   /* Check if more than one exclusive proxy type has been enabled. */
   if (!!options->Socks4Proxy + !!options->Socks5Proxy +
-      !!options->HTTPSProxy > 1)
+      !!options->HTTPSProxy + !!options->TCPProxy > 1)
     REJECT("You have configured more than one proxy type. "
-           "(Socks4Proxy|Socks5Proxy|HTTPSProxy)");
+           "(Socks4Proxy|Socks5Proxy|HTTPSProxy|TCPProxy)");
 
   /* Check if the proxies will give surprising behavior. */
   if (options->HTTPProxy && !(options->Socks4Proxy ||
                               options->Socks5Proxy ||
-                              options->HTTPSProxy)) {
-    log_warn(LD_CONFIG, "HTTPProxy configured, but no SOCKS proxy or "
-             "HTTPS proxy configured. Watch out: this configuration will "
-             "proxy unencrypted directory connections only.");
+                              options->HTTPSProxy ||
+                              options->TCPProxy)) {
+    log_warn(LD_CONFIG, "HTTPProxy configured, but no SOCKS proxy, "
+             "HTTPS proxy, or any other TCP proxy configured. Watch out: "
+             "this configuration will proxy unencrypted directory "
+             "connections only.");
   }
 
   if (options->Socks5ProxyUsername) {
@@ -4216,15 +3727,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
              "have it group-readable.");
   }
 
-  if (options->MyFamily_lines && options->BridgeRelay) {
-    log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
-             "supported: it can reveal bridge fingerprints to censors. "
-             "You should also make sure you aren't listing this bridge's "
-             "fingerprint in any other MyFamily.");
-  }
-  if (normalize_nickname_list(&options->MyFamily,
-                              options->MyFamily_lines, "MyFamily", msg))
-    return -1;
   for (cl = options->NodeFamilies; cl; cl = cl->next) {
     routerset_t *rs = routerset_new();
     if (routerset_parse(rs, cl->value, cl->key)) {
@@ -4259,50 +3761,12 @@ options_validate(or_options_t *old_options, or_options_t *options,
   }
 
   for (cl = options->ClientTransportPlugin; cl; cl = cl->next) {
-    if (parse_transport_line(options, cl->value, 1, 0) < 0)
+    if (pt_parse_transport_line(options, cl->value, 1, 0) < 0)
       REJECT("Invalid client transport line. See logs for details.");
   }
 
-  for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
-    if (parse_transport_line(options, cl->value, 1, 1) < 0)
-      REJECT("Invalid server transport line. See logs for details.");
-  }
-
-  if (options->ServerTransportPlugin && !server_mode(options)) {
-    log_notice(LD_GENERAL, "Tor is not configured as a relay but you specified"
-               " a ServerTransportPlugin line (%s). The ServerTransportPlugin "
-               "line will be ignored.",
-               escaped(options->ServerTransportPlugin->value));
-  }
-
-  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
-    /** If get_bindaddr_from_transport_listen_line() fails with
-        'transport' being NULL, it means that something went wrong
-        while parsing the ServerTransportListenAddr line. */
-    char *bindaddr = get_bindaddr_from_transport_listen_line(cl->value, NULL);
-    if (!bindaddr)
-      REJECT("ServerTransportListenAddr did not parse. See logs for details.");
-    tor_free(bindaddr);
-  }
-
-  if (options->ServerTransportListenAddr && !options->ServerTransportPlugin) {
-    log_notice(LD_GENERAL, "You need at least a single managed-proxy to "
-               "specify a transport listen address. The "
-               "ServerTransportListenAddr line will be ignored.");
-  }
-
-  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
-    /** If get_options_from_transport_options_line() fails with
-        'transport' being NULL, it means that something went wrong
-        while parsing the ServerTransportOptions line. */
-    smartlist_t *options_sl =
-      get_options_from_transport_options_line(cl->value, NULL);
-    if (!options_sl)
-      REJECT("ServerTransportOptions did not parse. See logs for details.");
-
-    SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp));
-    smartlist_free(options_sl);
-  }
+  if (options_validate_server_transport(old_options, options, msg) < 0)
+    return -1;
 
   if (options->ConstrainedSockets) {
     /* If the user wants to constrain socket buffer use, make sure the desired
@@ -4316,85 +3780,10 @@ options_validate(or_options_t *old_options, or_options_t *options,
           MIN_CONSTRAINED_TCP_BUFFER, MAX_CONSTRAINED_TCP_BUFFER);
       return -1;
     }
-    if (options->DirPort_set) {
-      /* Providing cached directory entries while system TCP buffers are scarce
-       * will exacerbate the socket errors.  Suggest that this be disabled. */
-      COMPLAIN("You have requested constrained socket buffers while also "
-               "serving directory entries via DirPort.  It is strongly "
-               "suggested that you disable serving directory requests when "
-               "system TCP buffer resources are scarce.");
-    }
   }
 
-  if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
-      options->V3AuthVotingInterval/2) {
-    /*
-    This doesn't work, but it seems like it should:
-     what code is preventing the interval being less than twice the lead-up?
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
-          options->V3AuthVotingInterval) {
-        REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than "
-               "V3AuthVotingInterval");
-      } else {
-        COMPLAIN("V3AuthVoteDelay plus V3AuthDistDelay is more than half "
-                 "V3AuthVotingInterval. This may lead to "
-                 "consensus instability, particularly if clocks drift.");
-      }
-    } else {
-     */
-      REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
-             "V3AuthVotingInterval");
-    /*
-    }
-     */
-  }
-
-  if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) {
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) {
-        REJECT("V3AuthVoteDelay is way too low.");
-      } else {
-        COMPLAIN("V3AuthVoteDelay is very low. "
-                 "This may lead to failure to vote for a consensus.");
-      }
-    } else {
-      REJECT("V3AuthVoteDelay is way too low.");
-    }
-  }
-
-  if (options->V3AuthDistDelay < MIN_DIST_SECONDS) {
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) {
-        REJECT("V3AuthDistDelay is way too low.");
-      } else {
-        COMPLAIN("V3AuthDistDelay is very low. "
-                 "This may lead to missing votes in a consensus.");
-      }
-    } else {
-      REJECT("V3AuthDistDelay is way too low.");
-    }
-  }
-
-  if (options->V3AuthNIntervalsValid < 2)
-    REJECT("V3AuthNIntervalsValid must be at least 2.");
-
-  if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) {
-    if (options->TestingTorNetwork) {
-      if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) {
-        REJECT("V3AuthVotingInterval is insanely low.");
-      } else {
-        COMPLAIN("V3AuthVotingInterval is very low. "
-                 "This may lead to failure to synchronise for a consensus.");
-      }
-    } else {
-      REJECT("V3AuthVotingInterval is insanely low.");
-    }
-  } else if (options->V3AuthVotingInterval > 24*60*60) {
-    REJECT("V3AuthVotingInterval is insanely high.");
-  } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) {
-    COMPLAIN("V3AuthVotingInterval does not divide evenly into 24 hours.");
-  }
+  if (options_validate_dirauth_schedule(old_options, options, msg) < 0)
+    return -1;
 
   if (hs_config_service_all(options, 1) < 0)
     REJECT("Failed to configure rendezvous options. See logs for details.");
@@ -4422,88 +3811,51 @@ options_validate(or_options_t *old_options, or_options_t *options,
 
 #define CHECK_DEFAULT(arg)                                              \
   STMT_BEGIN                                                            \
-    if (!options->TestingTorNetwork &&                                  \
-        !options->UsingTestNetworkDefaults_ &&                          \
-        !config_is_same(&options_format,options,                        \
-                        default_options,#arg)) {                        \
+    if (!config_is_same(get_options_mgr(),options,                      \
+                        dflt_options,#arg)) {                           \
+      or_options_free(dflt_options);                                    \
       REJECT(#arg " may only be changed in testing Tor "                \
              "networks!");                                              \
-    } STMT_END
-  CHECK_DEFAULT(TestingV3AuthInitialVotingInterval);
-  CHECK_DEFAULT(TestingV3AuthInitialVoteDelay);
-  CHECK_DEFAULT(TestingV3AuthInitialDistDelay);
-  CHECK_DEFAULT(TestingV3AuthVotingStartOffset);
-  CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability);
-  CHECK_DEFAULT(TestingEstimatedDescriptorPropagationTime);
-  CHECK_DEFAULT(TestingServerDownloadInitialDelay);
-  CHECK_DEFAULT(TestingClientDownloadInitialDelay);
-  CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay);
-  CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay);
-  CHECK_DEFAULT(TestingBridgeDownloadInitialDelay);
-  CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay);
-  CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest);
-  CHECK_DEFAULT(TestingDirConnectionMaxStall);
-  CHECK_DEFAULT(TestingAuthKeyLifetime);
-  CHECK_DEFAULT(TestingLinkCertLifetime);
-  CHECK_DEFAULT(TestingSigningKeySlop);
-  CHECK_DEFAULT(TestingAuthKeySlop);
-  CHECK_DEFAULT(TestingLinkKeySlop);
+    }                                                                   \
+  STMT_END
+
+  /* Check for options that can only be changed from the defaults in testing
+     networks.  */
+  if (! options->TestingTorNetwork && !options->UsingTestNetworkDefaults_) {
+    or_options_t *dflt_options = options_new();
+    options_init(dflt_options);
+    /* 31851: some of these options are dirauth or relay only */
+    CHECK_DEFAULT(TestingV3AuthInitialVotingInterval);
+    CHECK_DEFAULT(TestingV3AuthInitialVoteDelay);
+    CHECK_DEFAULT(TestingV3AuthInitialDistDelay);
+    CHECK_DEFAULT(TestingV3AuthVotingStartOffset);
+    CHECK_DEFAULT(TestingAuthDirTimeToLearnReachability);
+    CHECK_DEFAULT(TestingServerDownloadInitialDelay);
+    CHECK_DEFAULT(TestingClientDownloadInitialDelay);
+    CHECK_DEFAULT(TestingServerConsensusDownloadInitialDelay);
+    CHECK_DEFAULT(TestingClientConsensusDownloadInitialDelay);
+    CHECK_DEFAULT(TestingBridgeDownloadInitialDelay);
+    CHECK_DEFAULT(TestingBridgeBootstrapDownloadInitialDelay);
+    CHECK_DEFAULT(TestingClientMaxIntervalWithoutRequest);
+    CHECK_DEFAULT(TestingDirConnectionMaxStall);
+    CHECK_DEFAULT(TestingAuthKeyLifetime);
+    CHECK_DEFAULT(TestingLinkCertLifetime);
+    CHECK_DEFAULT(TestingSigningKeySlop);
+    CHECK_DEFAULT(TestingAuthKeySlop);
+    CHECK_DEFAULT(TestingLinkKeySlop);
+    or_options_free(dflt_options);
+  }
 #undef CHECK_DEFAULT
 
   if (!options->ClientDNSRejectInternalAddresses &&
       !(options->DirAuthorities ||
         (options->AlternateDirAuthority && options->AlternateBridgeAuthority)))
     REJECT("ClientDNSRejectInternalAddresses used for default network.");
-  if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2)
-    REJECT("SigningKeyLifetime is too short.");
-  if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2)
-    REJECT("LinkCertLifetime is too short.");
-  if (options->TestingAuthKeyLifetime < options->TestingLinkKeySlop*2)
-    REJECT("TestingAuthKeyLifetime is too short.");
-
-  if (options->TestingV3AuthInitialVotingInterval
-      < MIN_VOTE_INTERVAL_TESTING_INITIAL) {
-    REJECT("TestingV3AuthInitialVotingInterval is insanely low.");
-  } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) {
-    REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into "
-           "30 minutes.");
-  }
-
-  if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) {
-    REJECT("TestingV3AuthInitialVoteDelay is way too low.");
-  }
-
-  if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) {
-    REJECT("TestingV3AuthInitialDistDelay is way too low.");
-  }
 
-  if (options->TestingV3AuthInitialVoteDelay +
-      options->TestingV3AuthInitialDistDelay >=
-      options->TestingV3AuthInitialVotingInterval) {
-    REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay "
-           "must be less than TestingV3AuthInitialVotingInterval");
-  }
-
-  if (options->TestingV3AuthVotingStartOffset >
-      MIN(options->TestingV3AuthInitialVotingInterval,
-          options->V3AuthVotingInterval)) {
-    REJECT("TestingV3AuthVotingStartOffset is higher than the voting "
-           "interval.");
-  } else if (options->TestingV3AuthVotingStartOffset < 0) {
-    REJECT("TestingV3AuthVotingStartOffset must be non-negative.");
-  }
-
-  if (options->TestingAuthDirTimeToLearnReachability < 0) {
-    REJECT("TestingAuthDirTimeToLearnReachability must be non-negative.");
-  } else if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) {
-    COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high.");
-  }
-
-  if (options->TestingEstimatedDescriptorPropagationTime < 0) {
-    REJECT("TestingEstimatedDescriptorPropagationTime must be non-negative.");
-  } else if (options->TestingEstimatedDescriptorPropagationTime > 60*60) {
-    COMPLAIN("TestingEstimatedDescriptorPropagationTime is insanely high.");
-  }
+  if (options_validate_relay_testing(old_options, options, msg) < 0)
+    return -1;
+  if (options_validate_dirauth_testing(old_options, options, msg) < 0)
+    return -1;
 
   if (options->TestingClientMaxIntervalWithoutRequest < 1) {
     REJECT("TestingClientMaxIntervalWithoutRequest is way too low.");
@@ -4545,27 +3897,6 @@ options_validate(or_options_t *old_options, or_options_t *options,
                         "testing Tor network!");
   }
 
-  if (options->AccelName && !options->HardwareAccel)
-    options->HardwareAccel = 1;
-  if (options->AccelDir && !options->AccelName)
-    REJECT("Can't use hardware crypto accelerator dir without engine name.");
-
-  if (options->PublishServerDescriptor)
-    SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, {
-      if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0"))
-        if (smartlist_len(options->PublishServerDescriptor) > 1) {
-          COMPLAIN("You have passed a list of multiple arguments to the "
-                   "PublishServerDescriptor option that includes 0 or 1. "
-                   "0 or 1 should only be used as the sole argument. "
-                   "This configuration will be rejected in a future release.");
-          break;
-        }
-    });
-
-  if (options->BridgeRelay == 1 && ! options->ORPort_set)
-      REJECT("BridgeRelay is 1, ORPort is not set. This is an invalid "
-             "combination.");
-
   if (options_validate_scheduler(options, msg) < 0) {
     return -1;
   }
@@ -4580,8 +3911,11 @@ options_validate(or_options_t *old_options, or_options_t *options,
  * actual maximum value.  We clip this value if it's too low, and autodetect
  * it if it's set to 0. */
 STATIC uint64_t
-compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
+compute_real_max_mem_in_queues(const uint64_t val, bool is_server)
 {
+#define MIN_SERVER_MB 64
+#define MIN_UNWARNED_SERVER_MB 256
+#define MIN_UNWARNED_CLIENT_MB 64
   uint64_t result;
 
   if (val == 0) {
@@ -4610,7 +3944,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
 #else
 /* On a 32-bit platform, we can't have 8GB of ram. */
 #define RAM_IS_VERY_LARGE(x) (0)
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
       if (RAM_IS_VERY_LARGE(ram)) {
         /* If we have 8 GB, or more, RAM available, we set the MaxMemInQueues
@@ -4639,7 +3973,7 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
         result = avail;
       }
     }
-    if (log_guess && ! notice_sent) {
+    if (is_server && ! notice_sent) {
       log_notice(LD_CONFIG, "%sMaxMemInQueues is set to %"PRIu64" MB. "
                  "You can override this by setting MaxMemInQueues by hand.",
                  ram ? "Based on detected system memory, " : "",
@@ -4647,60 +3981,30 @@ compute_real_max_mem_in_queues(const uint64_t val, int log_guess)
       notice_sent = 1;
     }
     return result;
-  } else if (val < ONE_GIGABYTE / 4) {
-    log_warn(LD_CONFIG, "MaxMemInQueues must be at least 256 MB for now. "
-             "Ideally, have it as large as you can afford.");
-    return ONE_GIGABYTE / 4;
+  } else if (is_server && val < ONE_MEGABYTE * MIN_SERVER_MB) {
+    /* We can't configure less than this much on a server.  */
+    log_warn(LD_CONFIG, "MaxMemInQueues must be at least %d MB on servers "
+             "for now. Ideally, have it as large as you can afford.",
+             MIN_SERVER_MB);
+    return MIN_SERVER_MB * ONE_MEGABYTE;
+  } else if (is_server && val < ONE_MEGABYTE * MIN_UNWARNED_SERVER_MB) {
+    /* On a server, if it's less than this much, we warn that things
+     * may go badly. */
+    log_warn(LD_CONFIG, "MaxMemInQueues is set to a low value; if your "
+             "relay doesn't work, this may be the reason why.");
+    return val;
+  } else if (! is_server && val < ONE_MEGABYTE * MIN_UNWARNED_CLIENT_MB) {
+    /* On a client, if it's less than this much, we warn that things
+     * may go badly. */
+    log_warn(LD_CONFIG, "MaxMemInQueues is set to a low value; if your "
+             "client doesn't work, this may be the reason why.");
+    return val;
   } else {
     /* The value was fine all along */
     return val;
   }
 }
 
-/* If we have less than 300 MB suggest disabling dircache */
-#define DIRCACHE_MIN_MEM_MB 300
-#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
-#define STRINGIFY(val) #val
-
-/** Create a warning message for emitting if we are a dircache but may not have
- * enough system memory, or if we are not a dircache but probably should be.
- * Return -1 when a message is returned in *msg*, else return 0. */
-STATIC int
-have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
-                             char **msg)
-{
-  *msg = NULL;
-  /* XXX We should possibly be looking at MaxMemInQueues here
-   * unconditionally.  Or we should believe total_mem unconditionally. */
-  if (total_mem == 0) {
-    if (get_total_system_memory(&total_mem) < 0) {
-      total_mem = options->MaxMemInQueues >= SIZE_MAX ?
-        SIZE_MAX : (size_t)options->MaxMemInQueues;
-    }
-  }
-  if (options->DirCache) {
-    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
-      if (options->BridgeRelay) {
-        tor_asprintf(msg, "Running a Bridge with less than %d MB of memory "
-                       "is not recommended.", DIRCACHE_MIN_MEM_MB);
-      } else {
-        tor_asprintf(msg, "Being a directory cache (default) with less than "
-                       "%d MB of memory is not recommended and may consume "
-                       "most of the available resources. Consider disabling "
-                       "this functionality by setting the DirCache option "
-                       "to 0.", DIRCACHE_MIN_MEM_MB);
-      }
-    }
-  } else {
-    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
-      *msg = tor_strdup("DirCache is disabled and we are configured as a "
-               "relay. We will not become a Guard.");
-    }
-  }
-  return *msg == NULL ? 0 : -1;
-}
-#undef STRINGIFY
-
 /** Helper: return true iff s1 and s2 are both NULL, or both non-NULL
  * equal strings. */
 static int
@@ -4709,13 +4013,19 @@ opt_streq(const char *s1, const char *s2)
   return 0 == strcmp_opt(s1, s2);
 }
 
-/** Check if any of the previous options have changed but aren't allowed to. */
+/** Check if any config options have changed but aren't allowed to. */
 static int
-options_transition_allowed(const or_options_t *old,
-                           const or_options_t *new_val,
-                           char **msg)
+options_check_transition_cb(const void *old_,
+                            const void *new_val_,
+                            char **msg)
 {
-  if (!old)
+  CHECK_OPTIONS_MAGIC(old_);
+  CHECK_OPTIONS_MAGIC(new_val_);
+
+  const or_options_t *old = old_;
+  const or_options_t *new_val = new_val_;
+
+  if (BUG(!old))
     return 0;
 
 #define BAD_CHANGE_TO(opt, how) do {                                    \
@@ -4724,36 +4034,6 @@ options_transition_allowed(const or_options_t *old,
     return -1;                                                          \
   } while (0)
 
-#define NO_CHANGE_BOOL(opt) \
-  if (! CFG_EQ_BOOL(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
-#define NO_CHANGE_INT(opt) \
-  if (! CFG_EQ_INT(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
-#define NO_CHANGE_STRING(opt) \
-  if (! CFG_EQ_STRING(old, new_val, opt)) BAD_CHANGE_TO(opt,"")
-
-  NO_CHANGE_STRING(PidFile);
-  NO_CHANGE_BOOL(RunAsDaemon);
-  NO_CHANGE_BOOL(Sandbox);
-  NO_CHANGE_STRING(DataDirectory);
-  NO_CHANGE_STRING(KeyDirectory);
-  NO_CHANGE_STRING(CacheDirectory);
-  NO_CHANGE_STRING(User);
-  NO_CHANGE_BOOL(KeepBindCapabilities);
-  NO_CHANGE_STRING(SyslogIdentityTag);
-  NO_CHANGE_STRING(AndroidIdentityTag);
-  NO_CHANGE_BOOL(HardwareAccel);
-  NO_CHANGE_STRING(AccelName);
-  NO_CHANGE_STRING(AccelDir);
-  NO_CHANGE_BOOL(TestingTorNetwork);
-  NO_CHANGE_BOOL(DisableAllSwap);
-  NO_CHANGE_INT(TokenBucketRefillInterval);
-  NO_CHANGE_BOOL(HiddenServiceSingleHopMode);
-  NO_CHANGE_BOOL(HiddenServiceNonAnonymousMode);
-  NO_CHANGE_BOOL(DisableDebuggerAttachment);
-  NO_CHANGE_BOOL(NoExec);
-  NO_CHANGE_INT(OwningControllerFD);
-  NO_CHANGE_BOOL(DisableSignalHandlers);
-
   if (sandbox_is_active()) {
 #define SB_NOCHANGE_STR(opt)                      \
     if (! CFG_EQ_STRING(old, new_val, opt))       \
@@ -4790,71 +4070,6 @@ options_transition_allowed(const or_options_t *old,
   return 0;
 }
 
-/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
- * will require us to rotate the CPU and DNS workers; else return 0. */
-static int
-options_transition_affects_workers(const or_options_t *old_options,
-                                   const or_options_t *new_options)
-{
-  YES_IF_CHANGED_STRING(DataDirectory);
-  YES_IF_CHANGED_INT(NumCPUs);
-  YES_IF_CHANGED_LINELIST(ORPort_lines);
-  YES_IF_CHANGED_BOOL(ServerDNSSearchDomains);
-  YES_IF_CHANGED_BOOL(SafeLogging_);
-  YES_IF_CHANGED_BOOL(ClientOnly);
-  YES_IF_CHANGED_BOOL(LogMessageDomains);
-  YES_IF_CHANGED_LINELIST(Logs);
-
-  if (server_mode(old_options) != server_mode(new_options) ||
-      public_server_mode(old_options) != public_server_mode(new_options) ||
-      dir_server_mode(old_options) != dir_server_mode(new_options))
-    return 1;
-
-  /* Nothing that changed matters. */
-  return 0;
-}
-
-/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
- * will require us to generate a new descriptor; else return 0. */
-static int
-options_transition_affects_descriptor(const or_options_t *old_options,
-                                      const or_options_t *new_options)
-{
-  /* XXX We can be smarter here. If your DirPort isn't being
-   * published and you just turned it off, no need to republish. Etc. */
-
-  YES_IF_CHANGED_STRING(DataDirectory);
-  YES_IF_CHANGED_STRING(Nickname);
-  YES_IF_CHANGED_STRING(Address);
-  YES_IF_CHANGED_LINELIST(ExitPolicy);
-  YES_IF_CHANGED_BOOL(ExitRelay);
-  YES_IF_CHANGED_BOOL(ExitPolicyRejectPrivate);
-  YES_IF_CHANGED_BOOL(ExitPolicyRejectLocalInterfaces);
-  YES_IF_CHANGED_BOOL(IPv6Exit);
-  YES_IF_CHANGED_LINELIST(ORPort_lines);
-  YES_IF_CHANGED_LINELIST(DirPort_lines);
-  YES_IF_CHANGED_LINELIST(DirPort_lines);
-  YES_IF_CHANGED_BOOL(ClientOnly);
-  YES_IF_CHANGED_BOOL(DisableNetwork);
-  YES_IF_CHANGED_BOOL(PublishServerDescriptor_);
-  YES_IF_CHANGED_STRING(ContactInfo);
-  YES_IF_CHANGED_STRING(BridgeDistribution);
-  YES_IF_CHANGED_LINELIST(MyFamily);
-  YES_IF_CHANGED_STRING(AccountingStart);
-  YES_IF_CHANGED_INT(AccountingMax);
-  YES_IF_CHANGED_INT(AccountingRule);
-  YES_IF_CHANGED_BOOL(DirCache);
-  YES_IF_CHANGED_BOOL(AssumeReachable);
-
-  if (get_effective_bwrate(old_options) != get_effective_bwrate(new_options) ||
-      get_effective_bwburst(old_options) !=
-        get_effective_bwburst(new_options) ||
-      public_server_mode(old_options) != public_server_mode(new_options))
-    return 1;
-
-  return 0;
-}
-
 #ifdef _WIN32
 /** Return the directory on windows where we expect to find our application
  * data. */
@@ -4939,85 +4154,6 @@ get_default_conf_file(int defaults_file)
 #endif /* defined(DISABLE_SYSTEM_TORRC) || ... */
 }
 
-/** Verify whether lst is a list of strings containing valid-looking
- * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
- * to any nickname or fingerprint that needs it. Also splits comma-separated
- * list elements into multiple elements. Return 0 on success.
- * Warn and return -1 on failure.
- */
-static int
-normalize_nickname_list(config_line_t **normalized_out,
-                        const config_line_t *lst, const char *name,
-                        char **msg)
-{
-  if (!lst)
-    return 0;
-
-  config_line_t *new_nicknames = NULL;
-  config_line_t **new_nicknames_next = &new_nicknames;
-
-  const config_line_t *cl;
-  for (cl = lst; cl; cl = cl->next) {
-    const char *line = cl->value;
-    if (!line)
-      continue;
-
-    int valid_line = 1;
-    smartlist_t *sl = smartlist_new();
-    smartlist_split_string(sl, line, ",",
-      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
-    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
-    {
-      char *normalized = NULL;
-      if (!is_legal_nickname_or_hexdigest(s)) {
-        // check if first char is dollar
-        if (s[0] != '$') {
-          // Try again but with a dollar symbol prepended
-          char *prepended;
-          tor_asprintf(&prepended, "$%s", s);
-
-          if (is_legal_nickname_or_hexdigest(prepended)) {
-            // The nickname is valid when it's prepended, set it as the
-            // normalized version
-            normalized = prepended;
-          } else {
-            // Still not valid, free and fallback to error message
-            tor_free(prepended);
-          }
-        }
-
-        if (!normalized) {
-          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
-          valid_line = 0;
-          break;
-        }
-      } else {
-        normalized = tor_strdup(s);
-      }
-
-      config_line_t *next = tor_malloc_zero(sizeof(*next));
-      next->key = tor_strdup(cl->key);
-      next->value = normalized;
-      next->next = NULL;
-
-      *new_nicknames_next = next;
-      new_nicknames_next = &next->next;
-    } SMARTLIST_FOREACH_END(s);
-
-    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
-    smartlist_free(sl);
-
-    if (!valid_line) {
-      config_free_lines(new_nicknames);
-      return -1;
-    }
-  }
-
-  *normalized_out = new_nicknames;
-
-  return 0;
-}
-
 /** Learn config file name from command line arguments, or use the default.
  *
  * If <b>defaults_file</b> is true, we're looking for torrc-defaults;
@@ -5030,12 +4166,12 @@ normalize_nickname_list(config_line_t **normalized_out,
  * filename if it doesn't exist.
  */
 static char *
-find_torrc_filename(config_line_t *cmd_arg,
+find_torrc_filename(const config_line_t *cmd_arg,
                     int defaults_file,
                     int *using_default_fname, int *ignore_missing_torrc)
 {
   char *fname=NULL;
-  config_line_t *p_index;
+  const config_line_t *p_index;
   const char *fname_opt = defaults_file ? "--defaults-torrc" : "-f";
   const char *ignore_opt = defaults_file ? NULL : "--ignore-missing-torrc";
 
@@ -5087,7 +4223,7 @@ find_torrc_filename(config_line_t *cmd_arg,
       } else {
         fname = dflt ? tor_strdup(dflt) : NULL;
       }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
       fname = dflt ? tor_strdup(dflt) : NULL;
 #endif /* !defined(_WIN32) */
     }
@@ -5114,7 +4250,7 @@ load_torrc_from_stdin(void)
  * Return the contents of the file on success, and NULL on failure.
  */
 static char *
-load_torrc_from_disk(config_line_t *cmd_arg, int defaults_file)
+load_torrc_from_disk(const config_line_t *cmd_arg, int defaults_file)
 {
   char *fname=NULL;
   char *cf = NULL;
@@ -5169,24 +4305,20 @@ int
 options_init_from_torrc(int argc, char **argv)
 {
   char *cf=NULL, *cf_defaults=NULL;
-  int command;
   int retval = -1;
-  char *command_arg = NULL;
   char *errmsg=NULL;
-  config_line_t *p_index = NULL;
-  config_line_t *cmdline_only_options = NULL;
+  const config_line_t *cmdline_only_options;
 
   /* Go through command-line variables */
-  if (! have_parsed_cmdline) {
+  if (global_cmdline == NULL) {
     /* Or we could redo the list every time we pass this place.
      * It does not really matter */
-    if (config_parse_commandline(argc, argv, 0, &global_cmdline_options,
-                                 &global_cmdline_only_options) < 0) {
+    global_cmdline = config_parse_commandline(argc, argv, 0);
+    if (global_cmdline == NULL) {
       goto err;
     }
-    have_parsed_cmdline = 1;
   }
-  cmdline_only_options = global_cmdline_only_options;
+  cmdline_only_options = global_cmdline->cmdline_opts;
 
   if (config_line_find(cmdline_only_options, "-h") ||
       config_line_find(cmdline_only_options, "--help")) {
@@ -5203,6 +4335,10 @@ options_init_from_torrc(int argc, char **argv)
     list_deprecated_options();
     return 1;
   }
+  if (config_line_find(cmdline_only_options, "--dbg-dump-subsystem-list")) {
+    subsystems_dump_list();
+    return 1;
+  }
 
   if (config_line_find(cmdline_only_options, "--version")) {
     printf("Tor version %s.\n",get_version());
@@ -5249,25 +4385,10 @@ options_init_from_torrc(int argc, char **argv)
     return 1;
   }
 
-  command = CMD_RUN_TOR;
-  for (p_index = cmdline_only_options; p_index; p_index = p_index->next) {
-    if (!strcmp(p_index->key,"--keygen")) {
-      command = CMD_KEYGEN;
-    } else if (!strcmp(p_index->key, "--key-expiration")) {
-      command = CMD_KEY_EXPIRATION;
-      command_arg = p_index->value;
-    } else if (!strcmp(p_index->key,"--list-fingerprint")) {
-      command = CMD_LIST_FINGERPRINT;
-    } else if (!strcmp(p_index->key, "--hash-password")) {
-      command = CMD_HASH_PASSWORD;
-      command_arg = p_index->value;
-    } else if (!strcmp(p_index->key, "--dump-config")) {
-      command = CMD_DUMP_CONFIG;
-      command_arg = p_index->value;
-    } else if (!strcmp(p_index->key, "--verify-config")) {
-      command = CMD_VERIFY_CONFIG;
-    }
-  }
+  int command = global_cmdline->command;
+  const char *command_arg = global_cmdline->command_arg;
+  /* "immediate" has already been handled by this point. */
+  tor_assert(command != CMD_IMMEDIATE);
 
   if (command == CMD_HASH_PASSWORD) {
     cf_defaults = tor_strdup("");
@@ -5389,6 +4510,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
                          int command, const char *command_arg,
                          char **msg)
 {
+  bool retry = false;
   or_options_t *oldoptions, *newoptions, *newdefaultoptions=NULL;
   config_line_t *cl;
   int retval;
@@ -5399,8 +4521,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
   oldoptions = global_options; /* get_options unfortunately asserts if
                                   this is the first time we run*/
 
-  newoptions = tor_malloc_zero(sizeof(or_options_t));
-  newoptions->magic_ = OR_OPTIONS_MAGIC;
+  newoptions = options_new();
   options_init(newoptions);
   newoptions->command = command;
   newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
@@ -5419,7 +4540,7 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       err = SETOPT_ERR_PARSE;
       goto err;
     }
-    retval = config_assign(&options_format, newoptions, cl,
+    retval = config_assign(get_options_mgr(), newoptions, cl,
                            CAL_WARN_DEPRECATIONS, msg);
     config_free_lines(cl);
     if (retval < 0) {
@@ -5427,16 +4548,23 @@ options_init_from_string(const char *cf_defaults, const char *cf,
       goto err;
     }
     if (i==0)
-      newdefaultoptions = config_dup(&options_format, newoptions);
+      newdefaultoptions = config_dup(get_options_mgr(), newoptions);
   }
 
   if (newdefaultoptions == NULL) {
-    newdefaultoptions = config_dup(&options_format, global_default_options);
+    newdefaultoptions = config_dup(get_options_mgr(), global_default_options);
   }
 
   /* Go through command-line variables too */
-  retval = config_assign(&options_format, newoptions,
-                         global_cmdline_options, CAL_WARN_DEPRECATIONS, msg);
+  {
+    config_line_t *other_opts = NULL;
+    if (global_cmdline) {
+      other_opts = global_cmdline->other_opts;
+    }
+    retval = config_assign(get_options_mgr(), newoptions,
+                           other_opts,
+                           CAL_WARN_DEPRECATIONS, msg);
+  }
   if (retval < 0) {
     err = SETOPT_ERR_PARSE;
     goto err;
@@ -5444,98 +4572,22 @@ options_init_from_string(const char *cf_defaults, const char *cf,
 
   newoptions->IncludeUsed = cf_has_include;
   newoptions->FilesOpenedByIncludes = opened_files;
+  opened_files = NULL; // prevent double-free.
 
   /* If this is a testing network configuration, change defaults
-   * for a list of dependent config options, re-initialize newoptions
-   * with the new defaults, and assign all options to it second time. */
-  if (newoptions->TestingTorNetwork) {
-    /* XXXX this is a bit of a kludge.  perhaps there's a better way to do
-     * this?  We could, for example, make the parsing algorithm do two passes
-     * over the configuration.  If it finds any "suite" options like
-     * TestingTorNetwork, it could change the defaults before its second pass.
-     * Not urgent so long as this seems to work, but at any sign of trouble,
-     * let's clean it up.  -NM */
-
-    /* Change defaults. */
-    for (int i = 0; testing_tor_network_defaults[i].name; ++i) {
-      const config_var_t *new_var = &testing_tor_network_defaults[i];
-      config_var_t *old_var =
-          config_find_option_mutable(&options_format, new_var->name);
-      tor_assert(new_var);
-      tor_assert(old_var);
-      old_var->initvalue = new_var->initvalue;
-
-      if ((config_find_deprecation(&options_format, new_var->name))) {
-        log_warn(LD_GENERAL, "Testing options override the deprecated "
-                 "option %s. Is that intentional?",
-                 new_var->name);
-      }
-    }
-
-    /* Clear newoptions and re-initialize them with new defaults. */
-    or_options_free(newoptions);
-    or_options_free(newdefaultoptions);
-    newdefaultoptions = NULL;
-    newoptions = tor_malloc_zero(sizeof(or_options_t));
-    newoptions->magic_ = OR_OPTIONS_MAGIC;
-    options_init(newoptions);
-    newoptions->command = command;
-    newoptions->command_arg = command_arg ? tor_strdup(command_arg) : NULL;
-
-    /* Assign all options a second time. */
-    opened_files = smartlist_new();
-    for (int i = 0; i < 2; ++i) {
-      const char *body = i==0 ? cf_defaults : cf;
-      if (!body)
-        continue;
-
-      /* get config lines, assign them */
-      retval = config_get_lines_include(body, &cl, 1,
-                                        body == cf ? &cf_has_include : NULL,
-                                        opened_files);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      retval = config_assign(&options_format, newoptions, cl, 0, msg);
-      config_free_lines(cl);
-      if (retval < 0) {
-        err = SETOPT_ERR_PARSE;
-        goto err;
-      }
-      if (i==0)
-        newdefaultoptions = config_dup(&options_format, newoptions);
-    }
-    /* Assign command-line variables a second time too */
-    retval = config_assign(&options_format, newoptions,
-                           global_cmdline_options, 0, msg);
-    if (retval < 0) {
-      err = SETOPT_ERR_PARSE;
-      goto err;
-    }
-  }
-
-  newoptions->IncludeUsed = cf_has_include;
-  in_option_validation = 1;
-  newoptions->FilesOpenedByIncludes = opened_files;
-
-  /* Validate newoptions */
-  if (options_validate(oldoptions, newoptions, newdefaultoptions,
-                       0, msg) < 0) {
-    err = SETOPT_ERR_PARSE; /*XXX make this a separate return value.*/
+   * for a list of dependent config options, and try this function again. */
+  if (newoptions->TestingTorNetwork && ! testing_network_configured) {
+    // retry with the testing defaults.
+    testing_network_configured = true;
+    retry = true;
     goto err;
   }
 
-  if (options_transition_allowed(oldoptions, newoptions, msg) < 0) {
-    err = SETOPT_ERR_TRANSITION;
+  err = options_validate_and_set(oldoptions, newoptions, msg);
+  if (err < 0) {
+    newoptions = NULL; // This was already freed in options_validate_and_set.
     goto err;
   }
-  in_option_validation = 0;
-
-  if (set_options(newoptions, msg)) {
-    err = SETOPT_ERR_SETTING;
-    goto err; /* frees and replaces old options */
-  }
 
   or_options_free(global_default_options);
   global_default_options = newdefaultoptions;
@@ -5548,15 +4600,16 @@ options_init_from_string(const char *cf_defaults, const char *cf,
     SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
     smartlist_free(opened_files);
   }
-  // may have been set to opened_files, avoid double free
-  newoptions->FilesOpenedByIncludes = NULL;
-  or_options_free(newoptions);
   or_options_free(newdefaultoptions);
+  or_options_free(newoptions);
   if (*msg) {
     char *old_msg = *msg;
     tor_asprintf(msg, "Failed to parse/validate config: %s", old_msg);
     tor_free(old_msg);
   }
+  if (retry)
+    return options_init_from_string(cf_defaults, cf, command, command_arg,
+                                    msg);
   return err;
 }
 
@@ -5681,22 +4734,14 @@ open_and_add_file_log(const log_severity_list_t *severity,
 }
 
 /**
- * Initialize the logs based on the configuration file.
- */
+ * Try to set our global log granularity from `options->LogGranularity`,
+ * adjusting it as needed so that we are an even divisor of a second, or an
+ * even multiple of seconds. Return 0 on success, -1 on failure.
+ **/
 static int
-options_init_logs(const or_options_t *old_options, or_options_t *options,
-                  int validate_only)
+options_init_log_granularity(const or_options_t *options,
+                             int validate_only)
 {
-  config_line_t *opt;
-  int ok;
-  smartlist_t *elts;
-  int run_as_daemon =
-#ifdef _WIN32
-               0;
-#else
-               options->RunAsDaemon;
-#endif
-
   if (options->LogTimeGranularity <= 0) {
     log_warn(LD_CONFIG, "Log time granularity '%d' has to be positive.",
              options->LogTimeGranularity);
@@ -5726,9 +4771,38 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
       set_log_time_granularity(options->LogTimeGranularity);
   }
 
+  return 0;
+}
+
+/**
+ * Initialize the logs based on the configuration file.
+ */
+STATIC int
+options_init_logs(const or_options_t *old_options, const or_options_t *options,
+                  int validate_only)
+{
+  config_line_t *opt;
+  int ok;
+  smartlist_t *elts;
+  int run_as_daemon =
+#ifdef _WIN32
+               0;
+#else
+               options->RunAsDaemon;
+#endif
+
+  if (options_init_log_granularity(options, validate_only) < 0)
+    return -1;
+
   ok = 1;
   elts = smartlist_new();
 
+  if (options->Logs == NULL && !run_as_daemon && !validate_only) {
+    /* When no logs are given, the default behavior is to log nothing (if
+       RunAsDaemon is set) or to log based on the quiet level otherwise. */
+    add_default_log_for_quiet_level(quiet_level);
+  }
+
   for (opt = options->Logs; opt; opt = opt->next) {
     log_severity_list_t *severity;
     const char *cfg = opt->value;
@@ -5782,7 +4856,7 @@ options_init_logs(const or_options_t *old_options, or_options_t *options,
 #else
         log_warn(LD_CONFIG, "Android logging is not supported"
                             " on this system. Sorry.");
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
         goto cleanup;
       }
     }
@@ -6002,6 +5076,68 @@ parse_bridge_line(const char *line)
   return bridge_line;
 }
 
+/** Parse the contents of a TCPProxy line from <b>line</b> and put it
+ * in <b>options</b>. Return 0 if the line is well-formed, and -1 if it
+ * isn't.
+ *
+ * This will mutate only options->TCPProxyProtocol, options->TCPProxyAddr,
+ * and options->TCPProxyPort.
+ *
+ * On error, tor_strdup an error explanation into *<b>msg</b>.
+ */
+STATIC int
+parse_tcp_proxy_line(const char *line, or_options_t *options, char **msg)
+{
+  int ret = 0;
+  tor_assert(line);
+  tor_assert(options);
+  tor_assert(msg);
+
+  smartlist_t *sl = smartlist_new();
+  /* Split between the protocol and the address/port. */
+  smartlist_split_string(sl, line, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2);
+
+  /* The address/port is not specified. */
+  if (smartlist_len(sl) < 2) {
+    *msg = tor_strdup("TCPProxy has no address/port. Please fix.");
+    goto err;
+  }
+
+  char *protocol_string = smartlist_get(sl, 0);
+  char *addrport_string = smartlist_get(sl, 1);
+
+  /* The only currently supported protocol is 'haproxy'. */
+  if (strcasecmp(protocol_string, "haproxy")) {
+    *msg = tor_strdup("TCPProxy protocol is not supported. Currently "
+                      "the only supported protocol is 'haproxy'. "
+                      "Please fix.");
+    goto err;
+  } else {
+    /* Otherwise, set the correct protocol. */
+    options->TCPProxyProtocol = TCP_PROXY_PROTOCOL_HAPROXY;
+  }
+
+  /* Parse the address/port. */
+  if (tor_addr_port_lookup(addrport_string, &options->TCPProxyAddr,
+                           &options->TCPProxyPort) < 0) {
+    *msg = tor_strdup("TCPProxy address/port failed to parse or resolve. "
+                      "Please fix.");
+    goto err;
+  }
+
+  /* Success. */
+  ret = 0;
+  goto end;
+
+ err:
+  ret = -1;
+ end:
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return ret;
+}
+
 /** Read the contents of a ClientTransportPlugin or ServerTransportPlugin
  * line from <b>line</b>, depending on the value of <b>server</b>. Return 0
  * if the line is well-formed, and -1 if it isn't.
@@ -6012,9 +5148,8 @@ parse_bridge_line(const char *line)
  * our internal transport list.
  * - If it's a managed proxy line, launch the managed proxy.
  */
-
-STATIC int
-parse_transport_line(const or_options_t *options,
+int
+pt_parse_transport_line(const or_options_t *options,
                      const char *line, int validate_only,
                      int server)
 {
@@ -6150,9 +5285,10 @@ parse_transport_line(const or_options_t *options,
 
     /* ClientTransportPlugins connecting through a proxy is managed only. */
     if (!server && (options->Socks4Proxy || options->Socks5Proxy ||
-                    options->HTTPSProxy)) {
+                    options->HTTPSProxy || options->TCPProxy)) {
       log_warn(LD_CONFIG, "You have configured an external proxy with another "
-                          "proxy type. (Socks4Proxy|Socks5Proxy|HTTPSProxy)");
+                          "proxy type. (Socks4Proxy|Socks5Proxy|HTTPSProxy|"
+                          "TCPProxy)");
       goto err;
     }
 
@@ -6207,157 +5343,6 @@ parse_transport_line(const or_options_t *options,
   return r;
 }
 
-/** Given a ServerTransportListenAddr <b>line</b>, return its
- *  <address:port> string. Return NULL if the line was not
- *  well-formed.
- *
- *  If <b>transport</b> is set, return NULL if the line is not
- *  referring to <b>transport</b>.
- *
- *  The returned string is allocated on the heap and it's the
- *  responsibility of the caller to free it. */
-static char *
-get_bindaddr_from_transport_listen_line(const char *line,const char *transport)
-{
-  smartlist_t *items = NULL;
-  const char *parsed_transport = NULL;
-  char *addrport = NULL;
-  tor_addr_t addr;
-  uint16_t port = 0;
-
-  items = smartlist_new();
-  smartlist_split_string(items, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-
-  if (smartlist_len(items) < 2) {
-    log_warn(LD_CONFIG,"Too few arguments on ServerTransportListenAddr line.");
-    goto err;
-  }
-
-  parsed_transport = smartlist_get(items, 0);
-  addrport = tor_strdup(smartlist_get(items, 1));
-
-  /* If 'transport' is given, check if it matches the one on the line */
-  if (transport && strcmp(transport, parsed_transport))
-    goto err;
-
-  /* Validate addrport */
-  if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) {
-    log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr "
-             "address '%s'", addrport);
-    goto err;
-  }
-
-  goto done;
-
- err:
-  tor_free(addrport);
-  addrport = NULL;
-
- done:
-  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
-  smartlist_free(items);
-
-  return addrport;
-}
-
-/** Given a ServerTransportOptions <b>line</b>, return a smartlist
- *  with the options. Return NULL if the line was not well-formed.
- *
- *  If <b>transport</b> is set, return NULL if the line is not
- *  referring to <b>transport</b>.
- *
- *  The returned smartlist and its strings are allocated on the heap
- *  and it's the responsibility of the caller to free it. */
-smartlist_t *
-get_options_from_transport_options_line(const char *line,const char *transport)
-{
-  smartlist_t *items = smartlist_new();
-  smartlist_t *options = smartlist_new();
-  const char *parsed_transport = NULL;
-
-  smartlist_split_string(items, line, NULL,
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
-
-  if (smartlist_len(items) < 2) {
-    log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line.");
-    goto err;
-  }
-
-  parsed_transport = smartlist_get(items, 0);
-  /* If 'transport' is given, check if it matches the one on the line */
-  if (transport && strcmp(transport, parsed_transport))
-    goto err;
-
-  SMARTLIST_FOREACH_BEGIN(items, const char *, option) {
-    if (option_sl_idx == 0) /* skip the transport field (first field)*/
-      continue;
-
-    /* validate that it's a k=v value */
-    if (!string_is_key_value(LOG_WARN, option)) {
-      log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option));
-      goto err;
-    }
-
-    /* add it to the options smartlist */
-    smartlist_add_strdup(options, option);
-    log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
-  } SMARTLIST_FOREACH_END(option);
-
-  goto done;
-
- err:
-  SMARTLIST_FOREACH(options, char*, s, tor_free(s));
-  smartlist_free(options);
-  options = NULL;
-
- done:
-  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
-  smartlist_free(items);
-
-  return options;
-}
-
-/** Given the name of a pluggable transport in <b>transport</b>, check
- *  the configuration file to see if the user has explicitly asked for
- *  it to listen on a specific port. Return a <address:port> string if
- *  so, otherwise NULL. */
-char *
-get_transport_bindaddr_from_config(const char *transport)
-{
-  config_line_t *cl;
-  const or_options_t *options = get_options();
-
-  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
-    char *bindaddr =
-      get_bindaddr_from_transport_listen_line(cl->value, transport);
-    if (bindaddr)
-      return bindaddr;
-  }
-
-  return NULL;
-}
-
-/** Given the name of a pluggable transport in <b>transport</b>, check
- *  the configuration file to see if the user has asked us to pass any
- *  parameters to the pluggable transport. Return a smartlist
- *  containing the parameters, otherwise NULL. */
-smartlist_t *
-get_options_for_server_transport(const char *transport)
-{
-  config_line_t *cl;
-  const or_options_t *options = get_options();
-
-  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
-    smartlist_t *options_sl =
-      get_options_from_transport_options_line(cl->value, transport);
-    if (options_sl)
-      return options_sl;
-  }
-
-  return NULL;
-}
-
 /** Read the contents of a DirAuthority line from <b>line</b>. If
  * <b>validate_only</b> is 0, and the line is well-formed, and it
  * shares any bits with <b>required_type</b> or <b>required_type</b>
@@ -6628,22 +5613,33 @@ parse_dir_fallback_line(const char *line,
   return r;
 }
 
-/** Allocate and return a new port_cfg_t with reasonable defaults. */
-STATIC port_cfg_t *
+/** Allocate and return a new port_cfg_t with reasonable defaults.
+ *
+ * <b>namelen</b> is the length of the unix socket name
+ * (typically the filesystem path), not including the trailing NUL.
+ * It should be 0 for ports that are not zunix sockets. */
+port_cfg_t *
 port_cfg_new(size_t namelen)
 {
   tor_assert(namelen <= SIZE_T_CEILING - sizeof(port_cfg_t) - 1);
   port_cfg_t *cfg = tor_malloc_zero(sizeof(port_cfg_t) + namelen + 1);
+
+  /* entry_cfg flags */
   cfg->entry_cfg.ipv4_traffic = 1;
   cfg->entry_cfg.ipv6_traffic = 1;
+  cfg->entry_cfg.prefer_ipv6 = 0;
   cfg->entry_cfg.dns_request = 1;
   cfg->entry_cfg.onion_traffic = 1;
   cfg->entry_cfg.prefer_ipv6_virtaddr = 1;
+  cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
+  cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
+
+  /* Other flags default to 0 due to tor_malloc_zero */
   return cfg;
 }
 
 /** Free all storage held in <b>port</b> */
-STATIC void
+void
 port_cfg_free_(port_cfg_t *port)
 {
   tor_free(port);
@@ -6677,27 +5673,6 @@ warn_nonlocal_client_ports(const smartlist_t *ports,
   } SMARTLIST_FOREACH_END(port);
 }
 
-/** Warn for every Extended ORPort port in <b>ports</b> that is on a
- *  publicly routable address. */
-static void
-warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname)
-{
-  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
-    if (port->type != CONN_TYPE_EXT_OR_LISTENER)
-      continue;
-    if (port->is_unix_addr)
-      continue;
-    /* XXX maybe warn even if address is RFC1918? */
-    if (!tor_addr_is_internal(&port->addr, 1)) {
-      log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. "
-               "This is not advised; this address is supposed to only be "
-               "exposed on localhost so that your pluggable transport "
-               "proxies can connect to it.",
-               fmt_addrport(&port->addr, port->port), portname);
-    }
-  } SMARTLIST_FOREACH_END(port);
-}
-
 /** Given a list of port_cfg_t in <b>ports</b>, warn if any controller port
  * there is listening on any non-loopback address.  If <b>forbid_nonlocal</b>
  * is true, then emit a stronger warning and remove the port from the list.
@@ -6768,7 +5743,7 @@ port_cfg_line_extract_addrport(const char *line,
     size_t sz;
     *is_unix_out = 1;
     *addrport_out = NULL;
-    line += strlen(unix_socket_prefix); /*No q: Keep the quote */
+    line += strlen(unix_socket_prefix); /* No 'unix:', but keep the quote */
     *rest_out = unescape_string(line, addrport_out, &sz);
     if (!*rest_out || (*addrport_out && sz != strlen(*addrport_out))) {
       tor_free(*addrport_out);
@@ -6811,55 +5786,6 @@ warn_client_dns_cache(const char *option, int disabling)
 }
 
 /**
- * Validate the configured bridge distribution method from a BridgeDistribution
- * config line.
- *
- * The input <b>bd</b>, is a string taken from the BridgeDistribution config
- * line (if present).  If the option wasn't set, return 0 immediately.  The
- * BridgeDistribution option is then validated.  Currently valid, recognised
- * options are:
- *
- * - "none"
- * - "any"
- * - "https"
- * - "email"
- * - "moat"
- * - "hyphae"
- *
- * If the option string is unrecognised, a warning will be logged and 0 is
- * returned.  If the option string contains an invalid character, -1 is
- * returned.
- **/
-STATIC int
-check_bridge_distribution_setting(const char *bd)
-{
-  if (bd == NULL)
-    return 0;
-
-  const char *RECOGNIZED[] = {
-    "none", "any", "https", "email", "moat", "hyphae"
-  };
-  unsigned i;
-  for (i = 0; i < ARRAY_LENGTH(RECOGNIZED); ++i) {
-    if (!strcasecmp(bd, RECOGNIZED[i]))
-      return 0;
-  }
-
-  const char *cp = bd;
-  //  Method = (KeywordChar | "_") +
-  while (TOR_ISALNUM(*cp) || *cp == '-' || *cp == '_')
-    ++cp;
-
-  if (*cp == 0) {
-    log_warn(LD_CONFIG, "Unrecognized BridgeDistribution value %s. I'll "
-           "assume you know what you are doing...", escaped(bd));
-    return 0; // we reached the end of the string; all is well
-  } else {
-    return -1; // we found a bad character in the string.
-  }
-}
-
-/**
  * Parse port configuration for a single port type.
  *
  * Read entries of the "FooPort" type from the list <b>ports</b>.  Syntax is
@@ -6889,8 +5815,8 @@ check_bridge_distribution_setting(const char *bd)
  * <b>out</b> for every port that the client should listen on.  Return 0
  * on success, -1 on failure.
  */
-STATIC int
-parse_port_config(smartlist_t *out,
+int
+port_parse_config(smartlist_t *out,
                   const config_line_t *ports,
                   const char *portname,
                   int listener_type,
@@ -6912,11 +5838,12 @@ parse_port_config(smartlist_t *out,
   const unsigned is_unix_socket = flags & CL_PORT_IS_UNIXSOCKET;
   int got_zero_port=0, got_nonzero_port=0;
   char *unix_socket_path = NULL;
+  port_cfg_t *cfg = NULL;
 
   /* If there's no FooPort, then maybe make a default one. */
   if (! ports) {
     if (defaultport && defaultaddr && out) {
-      port_cfg_t *cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
+       cfg = port_cfg_new(is_unix_socket ? strlen(defaultaddr) : 0);
        cfg->type = listener_type;
        if (is_unix_socket) {
          tor_addr_make_unspec(&cfg->addr);
@@ -6926,8 +5853,6 @@ parse_port_config(smartlist_t *out,
          cfg->port = defaultport;
          tor_addr_parse(&cfg->addr, defaultaddr);
        }
-       cfg->entry_cfg.session_group = SESSION_GROUP_UNSET;
-       cfg->entry_cfg.isolation_flags = ISO_DEFAULT;
        smartlist_add(out, cfg);
     }
     return 0;
@@ -6941,28 +5866,12 @@ parse_port_config(smartlist_t *out,
   for (; ports; ports = ports->next) {
     tor_addr_t addr;
     tor_addr_make_unspec(&addr);
-
-    int port;
-    int sessiongroup = SESSION_GROUP_UNSET;
-    unsigned isolation = ISO_DEFAULT;
-    int prefer_no_auth = 0;
-    int socks_iso_keep_alive = 0;
-
+    int port, ok,
+        has_used_unix_socket_only_option = 0,
+        is_unix_tagged_addr = 0;
     uint16_t ptmp=0;
-    int ok;
-    /* This must be kept in sync with port_cfg_new's defaults */
-    int no_listen = 0, no_advertise = 0, all_addrs = 0,
-      bind_ipv4_only = 0, bind_ipv6_only = 0,
-      ipv4_traffic = 1, ipv6_traffic = 1, prefer_ipv6 = 0, dns_request = 1,
-      onion_traffic = 1,
-      cache_ipv4 = 0, use_cached_ipv4 = 0,
-      cache_ipv6 = 0, use_cached_ipv6 = 0,
-      prefer_ipv6_automap = 1, world_writable = 0, group_writable = 0,
-      relax_dirmode_check = 0,
-      has_used_unix_socket_only_option = 0;
-
-    int is_unix_tagged_addr = 0;
     const char *rest_of_line = NULL;
+
     if (port_cfg_line_extract_addrport(ports->value,
                           &addrport, &is_unix_tagged_addr, &rest_of_line)<0) {
       log_warn(LD_CONFIG, "Invalid %sPort line with unparsable address",
@@ -7038,17 +5947,20 @@ parse_port_config(smartlist_t *out,
       }
     }
 
+    /* Default port_cfg_t object initialization */
+    cfg = port_cfg_new(unix_socket_path ? strlen(unix_socket_path) : 0);
+
     if (unix_socket_path && default_to_group_writable)
-      group_writable = 1;
+      cfg->is_group_writable = 1;
 
     /* Now parse the rest of the options, if any. */
     if (use_server_options) {
       /* This is a server port; parse advertising options */
       SMARTLIST_FOREACH_BEGIN(elts, char *, elt) {
         if (!strcasecmp(elt, "NoAdvertise")) {
-          no_advertise = 1;
+          cfg->server_cfg.no_advertise = 1;
         } else if (!strcasecmp(elt, "NoListen")) {
-          no_listen = 1;
+          cfg->server_cfg.no_listen = 1;
 #if 0
         /* not implemented yet. */
         } else if (!strcasecmp(elt, "AllAddrs")) {
@@ -7056,33 +5968,36 @@ parse_port_config(smartlist_t *out,
           all_addrs = 1;
 #endif /* 0 */
         } else if (!strcasecmp(elt, "IPv4Only")) {
-          bind_ipv4_only = 1;
+          cfg->server_cfg.bind_ipv4_only = 1;
         } else if (!strcasecmp(elt, "IPv6Only")) {
-          bind_ipv6_only = 1;
+          cfg->server_cfg.bind_ipv6_only = 1;
         } else {
           log_warn(LD_CONFIG, "Unrecognized %sPort option '%s'",
                    portname, escaped(elt));
         }
       } SMARTLIST_FOREACH_END(elt);
 
-      if (no_advertise && no_listen) {
+      if (cfg->server_cfg.no_advertise && cfg->server_cfg.no_listen) {
         log_warn(LD_CONFIG, "Tried to set both NoListen and NoAdvertise "
                  "on %sPort line '%s'",
                  portname, escaped(ports->value));
         goto err;
       }
-      if (bind_ipv4_only && bind_ipv6_only) {
+      if (cfg->server_cfg.bind_ipv4_only &&
+          cfg->server_cfg.bind_ipv6_only) {
         log_warn(LD_CONFIG, "Tried to set both IPv4Only and IPv6Only "
                  "on %sPort line '%s'",
                  portname, escaped(ports->value));
         goto err;
       }
-      if (bind_ipv4_only && tor_addr_family(&addr) != AF_INET) {
+      if (cfg->server_cfg.bind_ipv4_only &&
+          tor_addr_family(&addr) != AF_INET) {
         log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv4",
                  portname);
         goto err;
       }
-      if (bind_ipv6_only && tor_addr_family(&addr) != AF_INET6) {
+      if (cfg->server_cfg.bind_ipv6_only &&
+          tor_addr_family(&addr) != AF_INET6) {
         log_warn(LD_CONFIG, "Could not interpret %sPort address as IPv6",
                  portname);
         goto err;
@@ -7101,12 +6016,12 @@ parse_port_config(smartlist_t *out,
                      portname, escaped(elt));
             goto err;
           }
-          if (sessiongroup >= 0) {
+          if (cfg->entry_cfg.session_group >= 0) {
             log_warn(LD_CONFIG, "Multiple SessionGroup options on %sPort",
                      portname);
             goto err;
           }
-          sessiongroup = group;
+          cfg->entry_cfg.session_group = group;
           continue;
         }
 
@@ -7116,15 +6031,15 @@ parse_port_config(smartlist_t *out,
         }
 
         if (!strcasecmp(elt, "GroupWritable")) {
-          group_writable = !no;
+          cfg->is_group_writable = !no;
           has_used_unix_socket_only_option = 1;
           continue;
         } else if (!strcasecmp(elt, "WorldWritable")) {
-          world_writable = !no;
+          cfg->is_world_writable = !no;
           has_used_unix_socket_only_option = 1;
           continue;
         } else if (!strcasecmp(elt, "RelaxDirModeCheck")) {
-          relax_dirmode_check = !no;
+          cfg->relax_dirmode_check = !no;
           has_used_unix_socket_only_option = 1;
           continue;
         }
@@ -7137,19 +6052,19 @@ parse_port_config(smartlist_t *out,
 
         if (takes_hostnames) {
           if (!strcasecmp(elt, "IPv4Traffic")) {
-            ipv4_traffic = ! no;
+            cfg->entry_cfg.ipv4_traffic = ! no;
             continue;
           } else if (!strcasecmp(elt, "IPv6Traffic")) {
-            ipv6_traffic = ! no;
+            cfg->entry_cfg.ipv6_traffic = ! no;
             continue;
           } else if (!strcasecmp(elt, "PreferIPv6")) {
-            prefer_ipv6 = ! no;
+            cfg->entry_cfg.prefer_ipv6 = ! no;
             continue;
           } else if (!strcasecmp(elt, "DNSRequest")) {
-            dns_request = ! no;
+            cfg->entry_cfg.dns_request = ! no;
             continue;
           } else if (!strcasecmp(elt, "OnionTraffic")) {
-            onion_traffic = ! no;
+            cfg->entry_cfg.onion_traffic = ! no;
             continue;
           } else if (!strcasecmp(elt, "OnionTrafficOnly")) {
             /* Only connect to .onion addresses.  Equivalent to
@@ -7160,43 +6075,50 @@ parse_port_config(smartlist_t *out,
                        "DNSRequest, IPv4Traffic, and/or IPv6Traffic instead.",
                        portname, escaped(elt));
             } else {
-              ipv4_traffic = ipv6_traffic = dns_request = 0;
+              cfg->entry_cfg.ipv4_traffic = 0;
+              cfg->entry_cfg.ipv6_traffic = 0;
+              cfg->entry_cfg.dns_request = 0;
             }
             continue;
           }
         }
         if (!strcasecmp(elt, "CacheIPv4DNS")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          cache_ipv4 = ! no;
+          cfg->entry_cfg.cache_ipv4_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "CacheIPv6DNS")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          cache_ipv6 = ! no;
+          cfg->entry_cfg.cache_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "CacheDNS")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          cache_ipv4 = cache_ipv6 = ! no;
+          cfg->entry_cfg.cache_ipv4_answers = ! no;
+          cfg->entry_cfg.cache_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "UseIPv4Cache")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          use_cached_ipv4 = ! no;
+          cfg->entry_cfg.use_cached_ipv4_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "UseIPv6Cache")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          use_cached_ipv6 = ! no;
+          cfg->entry_cfg.use_cached_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "UseDNSCache")) {
           warn_client_dns_cache(elt, no); // since 0.2.9.2-alpha
-          use_cached_ipv4 = use_cached_ipv6 = ! no;
+          cfg->entry_cfg.use_cached_ipv4_answers = ! no;
+          cfg->entry_cfg.use_cached_ipv6_answers = ! no;
           continue;
         } else if (!strcasecmp(elt, "PreferIPv6Automap")) {
-          prefer_ipv6_automap = ! no;
+          cfg->entry_cfg.prefer_ipv6_virtaddr = ! no;
           continue;
         } else if (!strcasecmp(elt, "PreferSOCKSNoAuth")) {
-          prefer_no_auth = ! no;
+          cfg->entry_cfg.socks_prefer_no_auth = ! no;
           continue;
         } else if (!strcasecmp(elt, "KeepAliveIsolateSOCKSAuth")) {
-          socks_iso_keep_alive = ! no;
+          cfg->entry_cfg.socks_iso_keep_alive = ! no;
+          continue;
+        } else if (!strcasecmp(elt, "ExtendedErrors")) {
+          cfg->entry_cfg.extended_socks5_codes = ! no;
           continue;
         }
 
@@ -7219,9 +6141,9 @@ parse_port_config(smartlist_t *out,
         }
 
         if (no) {
-          isolation &= ~isoflag;
+          cfg->entry_cfg.isolation_flags &= ~isoflag;
         } else {
-          isolation |= isoflag;
+          cfg->entry_cfg.isolation_flags |= isoflag;
         }
       } SMARTLIST_FOREACH_END(elt);
     }
@@ -7231,51 +6153,51 @@ parse_port_config(smartlist_t *out,
     else
       got_zero_port = 1;
 
-    if (dns_request == 0 && listener_type == CONN_TYPE_AP_DNS_LISTENER) {
+    if (cfg->entry_cfg.dns_request == 0 &&
+        listener_type == CONN_TYPE_AP_DNS_LISTENER) {
       log_warn(LD_CONFIG, "You have a %sPort entry with DNS disabled; that "
                "won't work.", portname);
       goto err;
     }
-
-    if (ipv4_traffic == 0 && ipv6_traffic == 0 && onion_traffic == 0
-        && listener_type != CONN_TYPE_AP_DNS_LISTENER) {
+    if (cfg->entry_cfg.ipv4_traffic == 0 &&
+        cfg->entry_cfg.ipv6_traffic == 0 &&
+        cfg->entry_cfg.onion_traffic == 0 &&
+        listener_type != CONN_TYPE_AP_DNS_LISTENER) {
       log_warn(LD_CONFIG, "You have a %sPort entry with all of IPv4 and "
                "IPv6 and .onion disabled; that won't work.", portname);
       goto err;
     }
-
-    if (dns_request == 1 && ipv4_traffic == 0 && ipv6_traffic == 0
-        && listener_type != CONN_TYPE_AP_DNS_LISTENER) {
+    if (cfg->entry_cfg.dns_request == 1 &&
+        cfg->entry_cfg.ipv4_traffic == 0 &&
+        cfg->entry_cfg.ipv6_traffic == 0 &&
+        listener_type != CONN_TYPE_AP_DNS_LISTENER) {
       log_warn(LD_CONFIG, "You have a %sPort entry with DNSRequest enabled, "
                "but IPv4 and IPv6 disabled; DNS-based sites won't work.",
                portname);
       goto err;
     }
-
-    if ( has_used_unix_socket_only_option && ! unix_socket_path) {
+    if (has_used_unix_socket_only_option && !unix_socket_path) {
       log_warn(LD_CONFIG, "You have a %sPort entry with GroupWritable, "
                "WorldWritable, or RelaxDirModeCheck, but it is not a "
                "unix socket.", portname);
       goto err;
     }
-
-    if (!(isolation & ISO_SOCKSAUTH) && socks_iso_keep_alive) {
+    if (!(cfg->entry_cfg.isolation_flags & ISO_SOCKSAUTH) &&
+        cfg->entry_cfg.socks_iso_keep_alive) {
       log_warn(LD_CONFIG, "You have a %sPort entry with both "
                "NoIsolateSOCKSAuth and KeepAliveIsolateSOCKSAuth set.",
                portname);
       goto err;
     }
-
-    if (unix_socket_path && (isolation & ISO_CLIENTADDR)) {
+    if (unix_socket_path &&
+        (cfg->entry_cfg.isolation_flags & ISO_CLIENTADDR)) {
       /* `IsolateClientAddr` is nonsensical in the context of AF_LOCAL.
        * just silently remove the isolation flag.
        */
-      isolation &= ~ISO_CLIENTADDR;
+      cfg->entry_cfg.isolation_flags &= ~ISO_CLIENTADDR;
     }
-
     if (out && port) {
       size_t namelen = unix_socket_path ? strlen(unix_socket_path) : 0;
-      port_cfg_t *cfg = port_cfg_new(namelen);
       if (unix_socket_path) {
         tor_addr_make_unspec(&cfg->addr);
         memcpy(cfg->unix_addr, unix_socket_path, namelen + 1);
@@ -7286,32 +6208,13 @@ parse_port_config(smartlist_t *out,
         cfg->port = port;
       }
       cfg->type = listener_type;
-      cfg->is_world_writable = world_writable;
-      cfg->is_group_writable = group_writable;
-      cfg->relax_dirmode_check = relax_dirmode_check;
-      cfg->entry_cfg.isolation_flags = isolation;
-      cfg->entry_cfg.session_group = sessiongroup;
-      cfg->server_cfg.no_advertise = no_advertise;
-      cfg->server_cfg.no_listen = no_listen;
-      cfg->server_cfg.all_addrs = all_addrs;
-      cfg->server_cfg.bind_ipv4_only = bind_ipv4_only;
-      cfg->server_cfg.bind_ipv6_only = bind_ipv6_only;
-      cfg->entry_cfg.ipv4_traffic = ipv4_traffic;
-      cfg->entry_cfg.ipv6_traffic = ipv6_traffic;
-      cfg->entry_cfg.prefer_ipv6 = prefer_ipv6;
-      cfg->entry_cfg.dns_request = dns_request;
-      cfg->entry_cfg.onion_traffic = onion_traffic;
-      cfg->entry_cfg.cache_ipv4_answers = cache_ipv4;
-      cfg->entry_cfg.cache_ipv6_answers = cache_ipv6;
-      cfg->entry_cfg.use_cached_ipv4_answers = use_cached_ipv4;
-      cfg->entry_cfg.use_cached_ipv6_answers = use_cached_ipv6;
-      cfg->entry_cfg.prefer_ipv6_virtaddr = prefer_ipv6_automap;
-      cfg->entry_cfg.socks_prefer_no_auth = prefer_no_auth;
-      if (! (isolation & ISO_SOCKSAUTH))
+      if (! (cfg->entry_cfg.isolation_flags & ISO_SOCKSAUTH))
         cfg->entry_cfg.socks_prefer_no_auth = 1;
-      cfg->entry_cfg.socks_iso_keep_alive = socks_iso_keep_alive;
-
       smartlist_add(out, cfg);
+      /* out owns cfg now, don't re-use or free it */
+      cfg = NULL;
+    } else {
+      tor_free(cfg);
     }
     SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
     smartlist_clear(elts);
@@ -7323,7 +6226,7 @@ parse_port_config(smartlist_t *out,
     if (is_control)
       warn_nonlocal_controller_ports(out, forbid_nonlocal);
     else if (is_ext_orport)
-      warn_nonlocal_ext_orports(out, portname);
+      port_warn_nonlocal_ext_orports(out, portname);
     else
       warn_nonlocal_client_ports(out, portname, listener_type);
   }
@@ -7337,18 +6240,30 @@ parse_port_config(smartlist_t *out,
 
   retval = 0;
  err:
+  /* There are two ways we can error out:
+   *  1. part way through the loop: cfg needs to be freed;
+   *  2. ending the loop normally: cfg is always NULL.
+   *     In this case, cfg has either been:
+   *     - added to out, then set to NULL, or
+   *     - freed and set to NULL (because out is NULL, or port is 0).
+   */
+  tor_free(cfg);
+
+  /* Free the other variables from the loop.
+   * elts is always non-NULL here, but it may or may not be empty. */
   SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
   smartlist_free(elts);
   tor_free(unix_socket_path);
   tor_free(addrport);
+
   return retval;
 }
 
 /** Return the number of ports which are actually going to listen with type
  * <b>listenertype</b>.  Do not count no_listen ports.  Only count unix
  * sockets if count_sockets is true. */
-static int
-count_real_listeners(const smartlist_t *ports, int listenertype,
+int
+port_count_real_listeners(const smartlist_t *ports, int listenertype,
                      int count_sockets)
 {
   int n = 0;
@@ -7386,7 +6301,7 @@ parse_ports(or_options_t *options, int validate_only,
 
   const unsigned gw_flag = options->UnixSocksGroupWritable ?
     CL_PORT_DFLT_GROUP_WRITABLE : 0;
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
              options->SocksPort_lines,
              "Socks", CONN_TYPE_AP_LISTENER,
              "127.0.0.1", 9050,
@@ -7395,7 +6310,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid SocksPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->DNSPort_lines,
                         "DNS", CONN_TYPE_AP_DNS_LISTENER,
                         "127.0.0.1", 0,
@@ -7403,7 +6318,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid DNSPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->TransPort_lines,
                         "Trans", CONN_TYPE_AP_TRANS_LISTENER,
                         "127.0.0.1", 0,
@@ -7411,7 +6326,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid TransPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->NATDPort_lines,
                         "NATD", CONN_TYPE_AP_NATD_LISTENER,
                         "127.0.0.1", 0,
@@ -7419,7 +6334,7 @@ parse_ports(or_options_t *options, int validate_only,
     *msg = tor_strdup("Invalid NatdPort configuration");
     goto err;
   }
-  if (parse_port_config(ports,
+  if (port_parse_config(ports,
                         options->HTTPTunnelPort_lines,
                         "HTTP Tunnel", CONN_TYPE_AP_HTTP_CONNECT_LISTENER,
                         "127.0.0.1", 0,
@@ -7439,7 +6354,7 @@ parse_ports(or_options_t *options, int validate_only,
     if (options->ControlSocketsGroupWritable)
       control_port_flags |= CL_PORT_DFLT_GROUP_WRITABLE;
 
-    if (parse_port_config(ports,
+    if (port_parse_config(ports,
                           options->ControlPort_lines,
                           "Control", CONN_TYPE_CONTROL_LISTENER,
                           "127.0.0.1", 0,
@@ -7448,7 +6363,7 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
 
-    if (parse_port_config(ports, options->ControlSocket,
+    if (port_parse_config(ports, options->ControlSocket,
                           "ControlSocket",
                           CONN_TYPE_CONTROL_LISTENER, NULL, 0,
                           control_port_flags | CL_PORT_IS_UNIXSOCKET) < 0) {
@@ -7456,40 +6371,9 @@ parse_ports(or_options_t *options, int validate_only,
       goto err;
     }
   }
-  if (! options->ClientOnly) {
-    if (parse_port_config(ports,
-                          options->ORPort_lines,
-                          "OR", CONN_TYPE_OR_LISTENER,
-                          "0.0.0.0", 0,
-                          CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid ORPort configuration");
-      goto err;
-    }
-    if (parse_port_config(ports,
-                          options->ExtORPort_lines,
-                          "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
-                          "127.0.0.1", 0,
-                          CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
-      *msg = tor_strdup("Invalid ExtORPort configuration");
-      goto err;
-    }
-    if (parse_port_config(ports,
-                          options->DirPort_lines,
-                          "Dir", CONN_TYPE_DIR_LISTENER,
-                          "0.0.0.0", 0,
-                          CL_PORT_SERVER_OPTIONS) < 0) {
-      *msg = tor_strdup("Invalid DirPort configuration");
-      goto err;
-    }
-  }
 
-  int n_low_ports = 0;
-  if (check_server_ports(ports, options, &n_low_ports) < 0) {
-    *msg = tor_strdup("Misconfigured server ports");
+  if (port_parse_ports_relay(options, msg, ports, &have_low_ports) < 0)
     goto err;
-  }
-  if (have_low_ports < 0)
-    have_low_ports = (n_low_ports > 0);
 
   *n_ports_out = smartlist_len(ports);
 
@@ -7497,25 +6381,20 @@ parse_ports(or_options_t *options, int validate_only,
 
   /* Update the *Port_set options.  The !! here is to force a boolean out of
      an integer. */
-  options->ORPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_OR_LISTENER, 0);
+  port_update_port_set_relay(options, ports);
   options->SocksPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_LISTENER, 1);
   options->TransPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_TRANS_LISTENER, 1);
   options->NATDPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_NATD_LISTENER, 1);
   options->HTTPTunnelPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_HTTP_CONNECT_LISTENER, 1);
   /* Use options->ControlSocket to test if a control socket is set */
   options->ControlPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0);
-  options->DirPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_DIR_LISTENER, 0);
+    !! port_count_real_listeners(ports, CONN_TYPE_CONTROL_LISTENER, 0);
   options->DNSPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER, 1);
-  options->ExtORPort_set =
-    !! count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER, 0);
+    !! port_count_real_listeners(ports, CONN_TYPE_AP_DNS_LISTENER, 1);
 
   if (world_writable_control_socket) {
     SMARTLIST_FOREACH(ports, port_cfg_t *, p,
@@ -7546,7 +6425,7 @@ parse_ports(or_options_t *options, int validate_only,
 }
 
 /* Does port bind to IPv4? */
-static int
+int
 port_binds_ipv4(const port_cfg_t *port)
 {
   return tor_addr_family(&port->addr) == AF_INET ||
@@ -7555,7 +6434,7 @@ port_binds_ipv4(const port_cfg_t *port)
 }
 
 /* Does port bind to IPv6? */
-static int
+int
 port_binds_ipv6(const port_cfg_t *port)
 {
   return tor_addr_family(&port->addr) == AF_INET6 ||
@@ -7563,94 +6442,6 @@ port_binds_ipv6(const port_cfg_t *port)
           && !port->server_cfg.bind_ipv4_only);
 }
 
-/** Given a list of <b>port_cfg_t</b> in <b>ports</b>, check them for internal
- * consistency and warn as appropriate.  Set *<b>n_low_ports_out</b> to the
- * number of sub-1024 ports we will be binding. */
-static int
-check_server_ports(const smartlist_t *ports,
-                   const or_options_t *options,
-                   int *n_low_ports_out)
-{
-  int n_orport_advertised = 0;
-  int n_orport_advertised_ipv4 = 0;
-  int n_orport_listeners = 0;
-  int n_dirport_advertised = 0;
-  int n_dirport_listeners = 0;
-  int n_low_port = 0;
-  int r = 0;
-
-  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
-    if (port->type == CONN_TYPE_DIR_LISTENER) {
-      if (! port->server_cfg.no_advertise)
-        ++n_dirport_advertised;
-      if (! port->server_cfg.no_listen)
-        ++n_dirport_listeners;
-    } else if (port->type == CONN_TYPE_OR_LISTENER) {
-      if (! port->server_cfg.no_advertise) {
-        ++n_orport_advertised;
-        if (port_binds_ipv4(port))
-          ++n_orport_advertised_ipv4;
-      }
-      if (! port->server_cfg.no_listen)
-        ++n_orport_listeners;
-    } else {
-      continue;
-    }
-#ifndef _WIN32
-    if (!port->server_cfg.no_listen && port->port < 1024)
-      ++n_low_port;
-#endif
-  } SMARTLIST_FOREACH_END(port);
-
-  if (n_orport_advertised && !n_orport_listeners) {
-    log_warn(LD_CONFIG, "We are advertising an ORPort, but not actually "
-             "listening on one.");
-    r = -1;
-  }
-  if (n_orport_listeners && !n_orport_advertised) {
-    log_warn(LD_CONFIG, "We are listening on an ORPort, but not advertising "
-             "any ORPorts. This will keep us from building a %s "
-             "descriptor, and make us impossible to use.",
-             options->BridgeRelay ? "bridge" : "router");
-    r = -1;
-  }
-  if (n_dirport_advertised && !n_dirport_listeners) {
-    log_warn(LD_CONFIG, "We are advertising a DirPort, but not actually "
-             "listening on one.");
-    r = -1;
-  }
-  if (n_dirport_advertised > 1) {
-    log_warn(LD_CONFIG, "Can't advertise more than one DirPort.");
-    r = -1;
-  }
-  if (n_orport_advertised && !n_orport_advertised_ipv4 &&
-      !options->BridgeRelay) {
-    log_warn(LD_CONFIG, "Configured public relay to listen only on an IPv6 "
-             "address. Tor needs to listen on an IPv4 address too.");
-    r = -1;
-  }
-
-  if (n_low_port && options->AccountingMax &&
-      (!have_capability_support() || options->KeepBindCapabilities == 0)) {
-    const char *extra = "";
-    if (options->KeepBindCapabilities == 0 && have_capability_support())
-      extra = ", and you have disabled KeepBindCapabilities.";
-    log_warn(LD_CONFIG,
-          "You have set AccountingMax to use hibernation. You have also "
-          "chosen a low DirPort or OrPort%s."
-          "This combination can make Tor stop "
-          "working when it tries to re-attach the port after a period of "
-          "hibernation. Please choose a different port or turn off "
-          "hibernation unless you know this combination will work on your "
-          "platform.", extra);
-  }
-
-  if (n_low_ports_out)
-    *n_low_ports_out = n_low_port;
-
-  return r;
-}
-
 /** Return a list of port_cfg_t for client ports parsed from the
  * options. */
 MOCK_IMPL(const smartlist_t *,
@@ -7835,7 +6626,7 @@ get_data_directory(const char *val)
   } else {
     return tor_strdup(get_windows_conf_root());
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *d = val;
   if (!d)
     d = "~/.tor";
@@ -8020,7 +6811,7 @@ get_num_cpus(const or_options_t *options)
 static void
 init_libevent(const or_options_t *options)
 {
-  tor_libevent_cfg cfg;
+  tor_libevent_cfg_t cfg;
 
   tor_assert(options);
 
@@ -8129,43 +6920,6 @@ write_to_data_subdir(const char* subdir, const char* fname,
   return return_val;
 }
 
-/** Return a smartlist of ports that must be forwarded by
- *  tor-fw-helper. The smartlist contains the ports in a string format
- *  that is understandable by tor-fw-helper. */
-smartlist_t *
-get_list_of_ports_to_forward(void)
-{
-  smartlist_t *ports_to_forward = smartlist_new();
-  int port = 0;
-
-  /** XXX TODO tor-fw-helper does not support forwarding ports to
-      other hosts than the local one. If the user is binding to a
-      different IP address, tor-fw-helper won't work.  */
-  port = router_get_advertised_or_port(get_options());  /* Get ORPort */
-  if (port)
-    smartlist_add_asprintf(ports_to_forward, "%d:%d", port, port);
-
-  port = router_get_advertised_dir_port(get_options(), 0); /* Get DirPort */
-  if (port)
-    smartlist_add_asprintf(ports_to_forward, "%d:%d", port, port);
-
-  /* Get ports of transport proxies */
-  {
-    smartlist_t *transport_ports = get_transport_proxy_ports();
-    if (transport_ports) {
-      smartlist_add_all(ports_to_forward, transport_ports);
-      smartlist_free(transport_ports);
-    }
-  }
-
-  if (!smartlist_len(ports_to_forward)) {
-    smartlist_free(ports_to_forward);
-    ports_to_forward = NULL;
-  }
-
-  return ports_to_forward;
-}
-
 /** Helper to implement GETINFO functions about configuration variables (not
  * their values).  Given a "config/names" question, set *<b>answer</b> to a
  * new string describing the supported configuration variables and their
@@ -8179,72 +6933,48 @@ getinfo_helper_config(control_connection_t *conn,
   (void) errmsg;
   if (!strcmp(question, "config/names")) {
     smartlist_t *sl = smartlist_new();
-    int i;
-    for (i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
-      const char *type;
-      /* don't tell controller about triple-underscore options */
-      if (!strncmp(option_vars_[i].name, "___", 3))
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
+      /* don't tell controller about invisible options */
+      if (! config_var_is_listable(var))
         continue;
-      switch (var->type) {
-        case CONFIG_TYPE_STRING: type = "String"; break;
-        case CONFIG_TYPE_FILENAME: type = "Filename"; break;
-        case CONFIG_TYPE_UINT: type = "Integer"; break;
-        case CONFIG_TYPE_UINT64: type = "Integer"; break;
-        case CONFIG_TYPE_INT: type = "SignedInteger"; break;
-        case CONFIG_TYPE_PORT: type = "Port"; break;
-        case CONFIG_TYPE_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_MSEC_INTERVAL: type = "TimeMsecInterval"; break;
-        case CONFIG_TYPE_MEMUNIT: type = "DataSize"; break;
-        case CONFIG_TYPE_DOUBLE: type = "Float"; break;
-        case CONFIG_TYPE_BOOL: type = "Boolean"; break;
-        case CONFIG_TYPE_AUTOBOOL: type = "Boolean+Auto"; break;
-        case CONFIG_TYPE_ISOTIME: type = "Time"; break;
-        case CONFIG_TYPE_ROUTERSET: type = "RouterList"; break;
-        case CONFIG_TYPE_CSV: type = "CommaList"; break;
-        /* This type accepts more inputs than TimeInterval, but it ignores
-         * everything after the first entry, so we may as well pretend
-         * it's a TimeInterval. */
-        case CONFIG_TYPE_CSV_INTERVAL: type = "TimeInterval"; break;
-        case CONFIG_TYPE_LINELIST: type = "LineList"; break;
-        case CONFIG_TYPE_LINELIST_S: type = "Dependent"; break;
-        case CONFIG_TYPE_LINELIST_V: type = "Virtual"; break;
-        default:
-        case CONFIG_TYPE_OBSOLETE:
-          type = NULL; break;
-      }
+      const char *type = struct_var_get_typename(&var->member);
       if (!type)
         continue;
-      smartlist_add_asprintf(sl, "%s %s\n",var->name,type);
-    }
+      smartlist_add_asprintf(sl, "%s %s\n",var->member.name,type);
+    } SMARTLIST_FOREACH_END(var);
     *answer = smartlist_join_strings(sl, "", 0, NULL);
     SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
     smartlist_free(sl);
+    smartlist_free(vars);
   } else if (!strcmp(question, "config/defaults")) {
     smartlist_t *sl = smartlist_new();
     int dirauth_lines_seen = 0, fallback_lines_seen = 0;
-    for (int i = 0; option_vars_[i].name; ++i) {
-      const config_var_t *var = &option_vars_[i];
+    /* Possibly this should check whether the variables are listable,
+     * but currently it does not.  See ticket 31654. */
+    smartlist_t *vars = config_mgr_list_vars(get_options_mgr());
+    SMARTLIST_FOREACH_BEGIN(vars, const config_var_t *, var) {
       if (var->initvalue != NULL) {
-        if (strcmp(option_vars_[i].name, "DirAuthority") == 0) {
+        if (strcmp(var->member.name, "DirAuthority") == 0) {
           /*
            * Count dirauth lines we have a default for; we'll use the
            * count later to decide whether to add the defaults manually
            */
           ++dirauth_lines_seen;
         }
-        if (strcmp(option_vars_[i].name, "FallbackDir") == 0) {
+        if (strcmp(var->member.name, "FallbackDir") == 0) {
           /*
-           * Similarly count fallback lines, so that we can decided later
+           * Similarly count fallback lines, so that we can decide later
            * to add the defaults manually.
            */
           ++fallback_lines_seen;
         }
         char *val = esc_for_log(var->initvalue);
-        smartlist_add_asprintf(sl, "%s %s\n",var->name,val);
+        smartlist_add_asprintf(sl, "%s %s\n",var->member.name,val);
         tor_free(val);
       }
-    }
+    } SMARTLIST_FOREACH_END(var);
+    smartlist_free(vars);
 
     if (dirauth_lines_seen == 0) {
       /*
@@ -8405,7 +7135,7 @@ config_load_geoip_file_(sa_family_t family,
   }
   r = geoip_load_file(family, fname, severity);
   tor_free(free_fname);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)default_fname;
   r = geoip_load_file(family, fname, severity);
 #endif /* defined(_WIN32) */
@@ -8493,7 +7223,7 @@ init_cookie_authentication(const char *fname, const char *header,
       log_warn(LD_FS,"Unable to make %s group-readable.", escaped(fname));
     }
   }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void) group_readable;
 #endif /* !defined(_WIN32) */
 
diff --git a/src/app/config/config.h b/src/app/config/config.h
index 6852d352dc..1ba10d1d37 100644
--- a/src/app/config/config.h
+++ b/src/app/config/config.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 
 #include "app/config/or_options_st.h"
 #include "lib/testsupport/testsupport.h"
+#include "app/config/quiet_level.h"
 
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__) || defined(DARWIN)
 #define KERNEL_MAY_SUPPORT_IPFW
@@ -30,7 +31,6 @@
 #define MAX_DEFAULT_MEMORY_QUEUE_SIZE (UINT64_C(2) << 30)
 #endif
 
-MOCK_DECL(const char*, get_dirportfrontpage, (void));
 MOCK_DECL(const or_options_t *, get_options, (void));
 MOCK_DECL(or_options_t *, get_options_mutable, (void));
 int set_options(or_options_t *new_val, char **msg);
@@ -41,8 +41,8 @@ const char *escaped_safe_str_client(const char *address);
 const char *escaped_safe_str(const char *address);
 void init_protocol_warning_severity_level(void);
 int get_protocol_warning_severity_level(void);
-const char *get_version(void);
-const char *get_short_version(void);
+
+#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
 
 /** An error from options_trial_assign() or options_init_from_string(). */
 typedef enum setopt_err_t {
@@ -55,12 +55,6 @@ typedef enum setopt_err_t {
 setopt_err_t options_trial_assign(struct config_line_t *list, unsigned flags,
                                   char **msg);
 
-uint32_t get_last_resolved_addr(void);
-void reset_last_resolved_addr(void);
-int resolve_my_address(int warn_severity, const or_options_t *options,
-                       uint32_t *addr_out,
-                       const char **method_out, char **hostname_out);
-MOCK_DECL(int, is_local_addr, (const tor_addr_t *addr));
 void options_init(or_options_t *options);
 
 #define OPTIONS_DUMP_MINIMAL 1
@@ -164,6 +158,8 @@ int write_to_data_subdir(const char* subdir, const char* fname,
 int get_num_cpus(const or_options_t *options);
 
 MOCK_DECL(const smartlist_t *,get_configured_ports,(void));
+int port_binds_ipv4(const port_cfg_t *port);
+int port_binds_ipv6(const port_cfg_t *port);
 int get_first_advertised_port_by_type_af(int listener_type,
                                          int address_family);
 #define get_primary_or_port() \
@@ -182,26 +178,36 @@ char *get_first_listener_addrport_string(int listener_type);
 int options_need_geoip_info(const or_options_t *options,
                             const char **reason_out);
 
-smartlist_t *get_list_of_ports_to_forward(void);
-
 int getinfo_helper_config(control_connection_t *conn,
                           const char *question, char **answer,
                           const char **errmsg);
 
-uint32_t get_effective_bwrate(const or_options_t *options);
-uint32_t get_effective_bwburst(const or_options_t *options);
-
-char *get_transport_bindaddr_from_config(const char *transport);
-
 int init_cookie_authentication(const char *fname, const char *header,
                                int cookie_len, int group_readable,
                                uint8_t **cookie_out, int *cookie_is_set_out);
 
 or_options_t *options_new(void);
 
-int config_parse_commandline(int argc, char **argv, int ignore_errors,
-                             struct config_line_t **result,
-                             struct config_line_t **cmdline_result);
+/** Options settings parsed from the command-line. */
+typedef struct {
+  /** List of options that can only be set from the command-line */
+  struct config_line_t *cmdline_opts;
+  /** List of other options, to be handled by the general Tor configuration
+      system. */
+  struct config_line_t *other_opts;
+  /** Subcommand that Tor has been told to run */
+  tor_cmdline_mode_t command;
+  /** Argument for the command mode, if any. */
+  const char *command_arg;
+  /** How quiet have we been told to be? */
+  quiet_level_t quiet_level;
+} parsed_cmdline_t;
+
+parsed_cmdline_t *config_parse_commandline(int argc, char **argv,
+                                           int ignore_errors);
+void parsed_cmdline_free_(parsed_cmdline_t *cmdline);
+#define parsed_cmdline_free(c) \
+  FREE_AND_NULL(parsed_cmdline_t, parsed_cmdline_free_, (c))
 
 void config_register_addressmaps(const or_options_t *options);
 /* XXXX move to connection_edge.h */
@@ -230,14 +236,16 @@ void bridge_line_free_(bridge_line_t *bridge_line);
 #define bridge_line_free(line) \
   FREE_AND_NULL(bridge_line_t, bridge_line_free_, (line))
 bridge_line_t *parse_bridge_line(const char *line);
-smartlist_t *get_options_from_transport_options_line(const char *line,
-                                                     const char *transport);
-smartlist_t *get_options_for_server_transport(const char *transport);
 
 /* Port helper functions. */
 int options_any_client_port_set(const or_options_t *options);
-
-#ifdef CONFIG_PRIVATE
+int port_parse_config(smartlist_t *out,
+                      const struct config_line_t *ports,
+                      const char *portname,
+                      int listener_type,
+                      const char *defaultaddr,
+                      int defaultport,
+                      const unsigned flags);
 
 #define CL_PORT_NO_STREAM_OPTIONS (1u<<0)
 #define CL_PORT_WARN_NONLOCAL (1u<<1)
@@ -248,27 +256,34 @@ int options_any_client_port_set(const or_options_t *options);
 #define CL_PORT_IS_UNIXSOCKET (1u<<6)
 #define CL_PORT_DFLT_GROUP_WRITABLE (1u<<7)
 
-STATIC int options_act(const or_options_t *old_options);
-#ifdef TOR_UNIT_TESTS
-extern struct config_format_t options_format;
-#endif
-
-STATIC port_cfg_t *port_cfg_new(size_t namelen);
+port_cfg_t *port_cfg_new(size_t namelen);
 #define port_cfg_free(port) \
   FREE_AND_NULL(port_cfg_t, port_cfg_free_, (port))
-STATIC void port_cfg_free_(port_cfg_t *port);
+void port_cfg_free_(port_cfg_t *port);
+
+int port_count_real_listeners(const smartlist_t *ports,
+                         int listenertype,
+                         int count_sockets);
+int pt_parse_transport_line(const or_options_t *options,
+                         const char *line, int validate_only,
+                         int server);
+int config_ensure_bandwidth_cap(uint64_t *value, const char *desc, char **msg);
+
+#ifdef CONFIG_PRIVATE
+
+MOCK_DECL(STATIC int, options_act,(const or_options_t *old_options));
+MOCK_DECL(STATIC int, options_act_reversible,(const or_options_t *old_options,
+                                             char **msg));
+struct config_mgr_t;
+STATIC const struct config_mgr_t *get_options_mgr(void);
+
 #define or_options_free(opt) \
   FREE_AND_NULL(or_options_t, or_options_free_, (opt))
 STATIC void or_options_free_(or_options_t *options);
 STATIC int options_validate_single_onion(or_options_t *options,
                                          char **msg);
-STATIC int options_validate(or_options_t *old_options,
-                            or_options_t *options,
-                            or_options_t *default_options,
-                            int from_setconf, char **msg);
-STATIC int parse_transport_line(const or_options_t *options,
-                                const char *line, int validate_only,
-                                int server);
+STATIC int parse_tcp_proxy_line(const char *line, or_options_t *options,
+                                char **msg);
 STATIC int consider_adding_dir_servers(const or_options_t *options,
                                        const or_options_t *old_options);
 STATIC void add_default_trusted_dir_authorities(dirinfo_type_t type);
@@ -277,23 +292,28 @@ STATIC int parse_dir_authority_line(const char *line,
                                     dirinfo_type_t required_type,
                                     int validate_only);
 STATIC int parse_dir_fallback_line(const char *line, int validate_only);
-STATIC int have_enough_mem_for_dircache(const or_options_t *options,
-                                        size_t total_mem, char **msg);
-STATIC int parse_port_config(smartlist_t *out,
-                  const struct config_line_t *ports,
-                  const char *portname,
-                  int listener_type,
-                  const char *defaultaddr,
-                  int defaultport,
-                  const unsigned flags);
-
-STATIC int check_bridge_distribution_setting(const char *bd);
 
 STATIC uint64_t compute_real_max_mem_in_queues(const uint64_t val,
-                                               int log_guess);
+                                               bool is_server);
 STATIC int open_and_add_file_log(const log_severity_list_t *severity,
                                  const char *fname,
                                  int truncate_log);
+STATIC int options_init_logs(const or_options_t *old_options,
+                             const or_options_t *options, int validate_only);
+
+STATIC int options_create_directories(char **msg_out);
+struct log_transaction_t;
+STATIC struct log_transaction_t *options_start_log_transaction(
+                              const or_options_t *old_options,
+                              char **msg_out);
+STATIC void options_commit_log_transaction(struct log_transaction_t *xn);
+STATIC void options_rollback_log_transaction(struct log_transaction_t *xn);
+
+#ifdef TOR_UNIT_TESTS
+int options_validate(const or_options_t *old_options,
+                     or_options_t *options,
+                     char **msg);
+#endif
 
 STATIC int parse_ports(or_options_t *options, int validate_only,
                        char **msg, int *n_ports_out,
diff --git a/src/app/config/confparse.c b/src/app/config/confparse.c
deleted file mode 100644
index efa0c19fa6..0000000000
--- a/src/app/config/confparse.c
+++ /dev/null
@@ -1,1207 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.c
- *
- * \brief Back-end for parsing and generating key-value files, used to
- *   implement the torrc file format and the state file.
- *
- * This module is used by config.c to parse and encode torrc
- * configuration files, and by statefile.c to parse and encode the
- * $DATADIR/state file.
- *
- * To use this module, its callers provide an instance of
- * config_format_t to describe the mappings from a set of configuration
- * options to a number of fields in a C structure.  With this mapping,
- * the functions here can convert back and forth between the C structure
- * specified, and a linked list of key-value pairs.
- */
-
-#include "core/or/or.h"
-#include "app/config/confparse.h"
-#include "feature/nodelist/routerset.h"
-
-#include "lib/container/bitarray.h"
-#include "lib/encoding/confline.h"
-
-static uint64_t config_parse_memunit(const char *s, int *ok);
-static int config_parse_msec_interval(const char *s, int *ok);
-static int config_parse_interval(const char *s, int *ok);
-static void config_reset(const config_format_t *fmt, void *options,
-                         const config_var_t *var, int use_defaults);
-
-/** Allocate an empty configuration object of a given format type. */
-void *
-config_new(const config_format_t *fmt)
-{
-  void *opts = tor_malloc_zero(fmt->size);
-  *(uint32_t*)STRUCT_VAR_P(opts, fmt->magic_offset) = fmt->magic;
-  CONFIG_CHECK(fmt, opts);
-  return opts;
-}
-
-/*
- * Functions to parse config options
- */
-
-/** If <b>option</b> is an official abbreviation for a longer option,
- * return the longer option.  Otherwise return <b>option</b>.
- * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
- * apply abbreviations that work for the config file and the command line.
- * If <b>warn_obsolete</b> is set, warn about deprecated names. */
-const char *
-config_expand_abbrev(const config_format_t *fmt, const char *option,
-                     int command_line, int warn_obsolete)
-{
-  int i;
-  if (! fmt->abbrevs)
-    return option;
-  for (i=0; fmt->abbrevs[i].abbreviated; ++i) {
-    /* Abbreviations are case insensitive. */
-    if (!strcasecmp(option,fmt->abbrevs[i].abbreviated) &&
-        (command_line || !fmt->abbrevs[i].commandline_only)) {
-      if (warn_obsolete && fmt->abbrevs[i].warn) {
-        log_warn(LD_CONFIG,
-                 "The configuration option '%s' is deprecated; "
-                 "use '%s' instead.",
-                 fmt->abbrevs[i].abbreviated,
-                 fmt->abbrevs[i].full);
-      }
-      /* Keep going through the list in case we want to rewrite it more.
-       * (We could imagine recursing here, but I don't want to get the
-       * user into an infinite loop if we craft our list wrong.) */
-      option = fmt->abbrevs[i].full;
-    }
-  }
-  return option;
-}
-
-/** If <b>key</b> is a deprecated configuration option, return the message
- * explaining why it is deprecated (which may be an empty string). Return NULL
- * if it is not deprecated. The <b>key</b> field must be fully expanded. */
-const char *
-config_find_deprecation(const config_format_t *fmt, const char *key)
-{
-  if (BUG(fmt == NULL) || BUG(key == NULL))
-    return NULL;
-  if (fmt->deprecations == NULL)
-    return NULL;
-
-  const config_deprecation_t *d;
-  for (d = fmt->deprecations; d->name; ++d) {
-    if (!strcasecmp(d->name, key)) {
-      return d->why_deprecated ? d->why_deprecated : "";
-    }
-  }
-  return NULL;
-}
-
-/** As config_find_option, but return a non-const pointer. */
-config_var_t *
-config_find_option_mutable(config_format_t *fmt, const char *key)
-{
-  int i;
-  size_t keylen = strlen(key);
-  if (!keylen)
-    return NULL; /* if they say "--" on the command line, it's not an option */
-  /* First, check for an exact (case-insensitive) match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strcasecmp(key, fmt->vars[i].name)) {
-      return &fmt->vars[i];
-    }
-  }
-  /* If none, check for an abbreviated match */
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (!strncasecmp(key, fmt->vars[i].name, keylen)) {
-      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
-               "Please use '%s' instead",
-               key, fmt->vars[i].name);
-      return &fmt->vars[i];
-    }
-  }
-  /* Okay, unrecognized option */
-  return NULL;
-}
-
-/** If <b>key</b> is a configuration option, return the corresponding const
- * config_var_t.  Otherwise, if <b>key</b> is a non-standard abbreviation,
- * warn, and return the corresponding const config_var_t.  Otherwise return
- * NULL.
- */
-const config_var_t *
-config_find_option(const config_format_t *fmt, const char *key)
-{
-  return config_find_option_mutable((config_format_t*)fmt, key);
-}
-
-/** Return the number of option entries in <b>fmt</b>. */
-static int
-config_count_options(const config_format_t *fmt)
-{
-  int i;
-  for (i=0; fmt->vars[i].name; ++i)
-    ;
-  return i;
-}
-
-/*
- * Functions to assign config options.
- */
-
-/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
- * with <b>c</b>-\>value and return 0, or return -1 if bad value.
- *
- * Called from config_assign_line() and option_reset().
- */
-static int
-config_assign_value(const config_format_t *fmt, void *options,
-                    config_line_t *c, char **msg)
-{
-  int i, ok;
-  const config_var_t *var;
-  void *lvalue;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  tor_assert(var);
-
-  lvalue = STRUCT_VAR_P(options, var->var_offset);
-
-  switch (var->type) {
-
-  case CONFIG_TYPE_PORT:
-    if (!strcasecmp(c->value, "auto")) {
-      *(int *)lvalue = CFG_AUTO_PORT;
-      break;
-    }
-    FALLTHROUGH;
-  case CONFIG_TYPE_INT:
-  case CONFIG_TYPE_UINT:
-    i = (int)tor_parse_long(c->value, 10,
-                            var->type==CONFIG_TYPE_INT ? INT_MIN : 0,
-                            var->type==CONFIG_TYPE_PORT ? 65535 : INT_MAX,
-                            &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Int keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_UINT64: {
-    uint64_t u64 = tor_parse_uint64(c->value, 10,
-                                    0, UINT64_MAX, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "uint64 keyword '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_CSV_INTERVAL: {
-    /* We used to have entire smartlists here.  But now that all of our
-     * download schedules use exponential backoff, only the first part
-     * matters. */
-    const char *comma = strchr(c->value, ',');
-    const char *val = c->value;
-    char *tmp = NULL;
-    if (comma) {
-      tmp = tor_strndup(c->value, comma - c->value);
-      val = tmp;
-    }
-
-    i = config_parse_interval(val, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      tor_free(tmp);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    tor_free(tmp);
-    break;
-  }
-
-  case CONFIG_TYPE_INTERVAL: {
-    i = config_parse_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MSEC_INTERVAL: {
-    i = config_parse_msec_interval(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Msec interval '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-  }
-
-  case CONFIG_TYPE_MEMUNIT: {
-    uint64_t u64 = config_parse_memunit(c->value, &ok);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Value '%s %s' is malformed or out of bounds.",
-          c->key, c->value);
-      return -1;
-    }
-    *(uint64_t *)lvalue = u64;
-    break;
-  }
-
-  case CONFIG_TYPE_BOOL:
-    i = (int)tor_parse_long(c->value, 10, 0, 1, &ok, NULL);
-    if (!ok) {
-      tor_asprintf(msg,
-          "Boolean '%s %s' expects 0 or 1.",
-          c->key, c->value);
-      return -1;
-    }
-    *(int *)lvalue = i;
-    break;
-
-  case CONFIG_TYPE_AUTOBOOL:
-    if (!strcasecmp(c->value, "auto"))
-      *(int *)lvalue = -1;
-    else if (!strcmp(c->value, "0"))
-      *(int *)lvalue = 0;
-    else if (!strcmp(c->value, "1"))
-      *(int *)lvalue = 1;
-    else {
-      tor_asprintf(msg, "Boolean '%s %s' expects 0, 1, or 'auto'.",
-                   c->key, c->value);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_STRING:
-  case CONFIG_TYPE_FILENAME:
-    tor_free(*(char **)lvalue);
-    *(char **)lvalue = tor_strdup(c->value);
-    break;
-
-  case CONFIG_TYPE_DOUBLE:
-    *(double *)lvalue = atof(c->value);
-    break;
-
-  case CONFIG_TYPE_ISOTIME:
-    if (parse_iso_time(c->value, (time_t *)lvalue)) {
-      tor_asprintf(msg,
-          "Invalid time '%s' for keyword '%s'", c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_ROUTERSET:
-    if (*(routerset_t**)lvalue) {
-      routerset_free(*(routerset_t**)lvalue);
-    }
-    *(routerset_t**)lvalue = routerset_new();
-    if (routerset_parse(*(routerset_t**)lvalue, c->value, c->key)<0) {
-      tor_asprintf(msg, "Invalid exit list '%s' for option '%s'",
-                   c->value, c->key);
-      return -1;
-    }
-    break;
-
-  case CONFIG_TYPE_CSV:
-    if (*(smartlist_t**)lvalue) {
-      SMARTLIST_FOREACH(*(smartlist_t**)lvalue, char *, cp, tor_free(cp));
-      smartlist_clear(*(smartlist_t**)lvalue);
-    } else {
-      *(smartlist_t**)lvalue = smartlist_new();
-    }
-
-    smartlist_split_string(*(smartlist_t**)lvalue, c->value, ",",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    break;
-
-  case CONFIG_TYPE_LINELIST:
-  case CONFIG_TYPE_LINELIST_S:
-    {
-      config_line_t *lastval = *(config_line_t**)lvalue;
-      if (lastval && lastval->fragile) {
-        if (c->command != CONFIG_LINE_APPEND) {
-          config_free_lines(lastval);
-          *(config_line_t**)lvalue = NULL;
-        } else {
-          lastval->fragile = 0;
-        }
-      }
-
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-    }
-    break;
-  case CONFIG_TYPE_OBSOLETE:
-    log_warn(LD_CONFIG, "Skipping obsolete configuration option '%s'", c->key);
-    break;
-  case CONFIG_TYPE_LINELIST_V:
-    tor_asprintf(msg,
-        "You may not provide a value for virtual option '%s'", c->key);
-    return -1;
-  default:
-    tor_assert(0);
-    break;
-  }
-  return 0;
-}
-
-/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
- * to it will replace old ones. */
-static void
-config_mark_lists_fragile(const config_format_t *fmt, void *options)
-{
-  int i;
-  tor_assert(fmt);
-  tor_assert(options);
-
-  for (i = 0; fmt->vars[i].name; ++i) {
-    const config_var_t *var = &fmt->vars[i];
-    config_line_t *list;
-    if (var->type != CONFIG_TYPE_LINELIST &&
-        var->type != CONFIG_TYPE_LINELIST_V)
-      continue;
-
-    list = *(config_line_t **)STRUCT_VAR_P(options, var->var_offset);
-    if (list)
-      list->fragile = 1;
-  }
-}
-
-void
-warn_deprecated_option(const char *what, const char *why)
-{
-  const char *space = (why && strlen(why)) ? " " : "";
-  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
-           "be removed in a future version of Tor.%s%s (If you think this is "
-           "a mistake, please let us know!)",
-           what, space, why);
-}
-
-/** If <b>c</b> is a syntactically valid configuration line, update
- * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
- * key, -2 for bad value.
- *
- * If <b>clear_first</b> is set, clear the value first. Then if
- * <b>use_defaults</b> is set, set the value to the default.
- *
- * Called from config_assign().
- */
-static int
-config_assign_line(const config_format_t *fmt, void *options,
-                   config_line_t *c, unsigned flags,
-                   bitarray_t *options_seen, char **msg)
-{
-  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
-  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
-  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, c->key);
-  if (!var) {
-    if (fmt->extra) {
-      void *lvalue = STRUCT_VAR_P(options, fmt->extra->var_offset);
-      log_info(LD_CONFIG,
-               "Found unrecognized option '%s'; saving it.", c->key);
-      config_line_append((config_line_t**)lvalue, c->key, c->value);
-      return 0;
-    } else {
-      tor_asprintf(msg,
-                "Unknown option '%s'.  Failing.", c->key);
-      return -1;
-    }
-  }
-
-  /* Put keyword into canonical case. */
-  if (strcmp(var->name, c->key)) {
-    tor_free(c->key);
-    c->key = tor_strdup(var->name);
-  }
-
-  const char *deprecation_msg;
-  if (warn_deprecations &&
-      (deprecation_msg = config_find_deprecation(fmt, var->name))) {
-    warn_deprecated_option(var->name, deprecation_msg);
-  }
-
-  if (!strlen(c->value)) {
-    /* reset or clear it, then return */
-    if (!clear_first) {
-      if ((var->type == CONFIG_TYPE_LINELIST ||
-           var->type == CONFIG_TYPE_LINELIST_S) &&
-          c->command != CONFIG_LINE_CLEAR) {
-        /* We got an empty linelist from the torrc or command line.
-           As a special case, call this an error. Warn and ignore. */
-        log_warn(LD_CONFIG,
-                 "Linelist option '%s' has no value. Skipping.", c->key);
-      } else { /* not already cleared */
-        config_reset(fmt, options, var, use_defaults);
-      }
-    }
-    return 0;
-  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
-    config_reset(fmt, options, var, use_defaults);
-  }
-
-  if (options_seen && (var->type != CONFIG_TYPE_LINELIST &&
-                       var->type != CONFIG_TYPE_LINELIST_S)) {
-    /* We're tracking which options we've seen, and this option is not
-     * supposed to occur more than once. */
-    int var_index = (int)(var - fmt->vars);
-    if (bitarray_is_set(options_seen, var_index)) {
-      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
-               "value will be ignored.", var->name);
-    }
-    bitarray_set(options_seen, var_index);
-  }
-
-  if (config_assign_value(fmt, options, c, msg) < 0)
-    return -2;
-  return 0;
-}
-
-/** Restore the option named <b>key</b> in options to its default value.
- * Called from config_assign(). */
-static void
-config_reset_line(const config_format_t *fmt, void *options,
-                  const char *key, int use_defaults)
-{
-  const config_var_t *var;
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var)
-    return; /* give error on next pass. */
-
-  config_reset(fmt, options, var, use_defaults);
-}
-
-/** Return true iff value needs to be quoted and escaped to be used in
- * a configuration file. */
-static int
-config_value_needs_escape(const char *value)
-{
-  if (*value == '\"')
-    return 1;
-  while (*value) {
-    switch (*value)
-    {
-    case '\r':
-    case '\n':
-    case '#':
-      /* Note: quotes and backspaces need special handling when we are using
-       * quotes, not otherwise, so they don't trigger escaping on their
-       * own. */
-      return 1;
-    default:
-      if (!TOR_ISPRINT(*value))
-        return 1;
-    }
-    ++value;
-  }
-  return 0;
-}
-
-/** Return newly allocated line or lines corresponding to <b>key</b> in the
- * configuration <b>options</b>.  If <b>escape_val</b> is true and a
- * value needs to be quoted before it's put in a config file, quote and
- * escape that value. Return NULL if no such key exists. */
-config_line_t *
-config_get_assigned_option(const config_format_t *fmt, const void *options,
-                           const char *key, int escape_val)
-{
-  const config_var_t *var;
-  const void *value;
-  config_line_t *result;
-  tor_assert(options && key);
-
-  CONFIG_CHECK(fmt, options);
-
-  var = config_find_option(fmt, key);
-  if (!var) {
-    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
-    return NULL;
-  }
-  value = STRUCT_VAR_P(options, var->var_offset);
-
-  result = tor_malloc_zero(sizeof(config_line_t));
-  result->key = tor_strdup(var->name);
-  switch (var->type)
-    {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      if (*(char**)value) {
-        result->value = tor_strdup(*(char**)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-        return NULL;
-      }
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      if (*(time_t*)value) {
-        result->value = tor_malloc(ISO_TIME_LEN+1);
-        format_iso_time(result->value, *(time_t*)value);
-      } else {
-        tor_free(result->key);
-        tor_free(result);
-      }
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_PORT:
-      if (*(int*)value == CFG_AUTO_PORT) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-      /* This means every or_options_t uint or bool element
-       * needs to be an int. Not, say, a uint16_t or char. */
-      tor_asprintf(&result->value, "%d", *(int*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_UINT64: FALLTHROUGH;
-    case CONFIG_TYPE_MEMUNIT:
-      tor_asprintf(&result->value, "%"PRIu64,
-                   (*(uint64_t*)value));
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      tor_asprintf(&result->value, "%f", *(double*)value);
-      escape_val = 0; /* Can't need escape. */
-      break;
-
-    case CONFIG_TYPE_AUTOBOOL:
-      if (*(int*)value == -1) {
-        result->value = tor_strdup("auto");
-        escape_val = 0;
-        break;
-      }
-      FALLTHROUGH;
-    case CONFIG_TYPE_BOOL:
-      result->value = tor_strdup(*(int*)value ? "1" : "0");
-      escape_val = 0; /* Can't need escape. */
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      result->value = routerset_to_string(*(routerset_t**)value);
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)value)
-        result->value =
-          smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
-      else
-        result->value = tor_strdup("");
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      log_fn(LOG_INFO, LD_CONFIG,
-             "You asked me for the value of an obsolete config option '%s'.",
-             key);
-      tor_free(result->key);
-      tor_free(result);
-      return NULL;
-    case CONFIG_TYPE_LINELIST_S:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup_and_filter(*(const config_line_t **)value,
-                                           key);
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_V:
-      tor_free(result->key);
-      tor_free(result);
-      result = config_lines_dup(*(const config_line_t**)value);
-      break;
-    default:
-      tor_free(result->key);
-      tor_free(result);
-      log_warn(LD_BUG,"Unknown type %d for known key '%s'",
-               var->type, key);
-      return NULL;
-    }
-
-  if (escape_val) {
-    config_line_t *line;
-    for (line = result; line; line = line->next) {
-      if (line->value && config_value_needs_escape(line->value)) {
-        char *newval = esc_for_log(line->value);
-        tor_free(line->value);
-        line->value = newval;
-      }
-    }
-  }
-
-  return result;
-}
-/** Iterate through the linked list of requested options <b>list</b>.
- * For each item, convert as appropriate and assign to <b>options</b>.
- * If an item is unrecognized, set *msg and return -1 immediately,
- * else return 0 for success.
- *
- * If <b>clear_first</b>, interpret config options as replacing (not
- * extending) their previous values. If <b>clear_first</b> is set,
- * then <b>use_defaults</b> to decide if you set to defaults after
- * clearing, or make the value 0 or NULL.
- *
- * Here are the use cases:
- * 1. A non-empty AllowInvalid line in your torrc. Appends to current
- *    if linelist, replaces current if csv.
- * 2. An empty AllowInvalid line in your torrc. Should clear it.
- * 3. "RESETCONF AllowInvalid" sets it to default.
- * 4. "SETCONF AllowInvalid" makes it NULL.
- * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
- *
- * Use_defaults   Clear_first
- *    0                0       "append"
- *    1                0       undefined, don't use
- *    0                1       "set to null first"
- *    1                1       "set to defaults first"
- * Return 0 on success, -1 on bad key, -2 on bad value.
- *
- * As an additional special case, if a LINELIST config option has
- * no value and clear_first is 0, then warn and ignore it.
- */
-
-/*
-There are three call cases for config_assign() currently.
-
-Case one: Torrc entry
-options_init_from_torrc() calls config_assign(0, 0)
-  calls config_assign_line(0, 0).
-    if value is empty, calls config_reset(0) and returns.
-    calls config_assign_value(), appends.
-
-Case two: setconf
-options_trial_assign() calls config_assign(0, 1)
-  calls config_reset_line(0)
-    calls config_reset(0)
-      calls option_clear().
-  calls config_assign_line(0, 1).
-    if value is empty, returns.
-    calls config_assign_value(), appends.
-
-Case three: resetconf
-options_trial_assign() calls config_assign(1, 1)
-  calls config_reset_line(1)
-    calls config_reset(1)
-      calls option_clear().
-      calls config_assign_value(default)
-  calls config_assign_line(1, 1).
-    returns.
-*/
-int
-config_assign(const config_format_t *fmt, void *options, config_line_t *list,
-              unsigned config_assign_flags, char **msg)
-{
-  config_line_t *p;
-  bitarray_t *options_seen;
-  const int n_options = config_count_options(fmt);
-  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
-  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
-
-  CONFIG_CHECK(fmt, options);
-
-  /* pass 1: normalize keys */
-  for (p = list; p; p = p->next) {
-    const char *full = config_expand_abbrev(fmt, p->key, 0, 1);
-    if (strcmp(full,p->key)) {
-      tor_free(p->key);
-      p->key = tor_strdup(full);
-    }
-  }
-
-  /* pass 2: if we're reading from a resetting source, clear all
-   * mentioned config options, and maybe set to their defaults. */
-  if (clear_first) {
-    for (p = list; p; p = p->next)
-      config_reset_line(fmt, options, p->key, use_defaults);
-  }
-
-  options_seen = bitarray_init_zero(n_options);
-  /* pass 3: assign. */
-  while (list) {
-    int r;
-    if ((r=config_assign_line(fmt, options, list, config_assign_flags,
-                              options_seen, msg))) {
-      bitarray_free(options_seen);
-      return r;
-    }
-    list = list->next;
-  }
-  bitarray_free(options_seen);
-
-  /** Now we're done assigning a group of options to the configuration.
-   * Subsequent group assignments should _replace_ linelists, not extend
-   * them. */
-  config_mark_lists_fragile(fmt, options);
-
-  return 0;
-}
-
-/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
- * Called from config_reset() and config_free(). */
-static void
-config_clear(const config_format_t *fmt, void *options,
-             const config_var_t *var)
-{
-  void *lvalue = STRUCT_VAR_P(options, var->var_offset);
-  (void)fmt; /* unused */
-  switch (var->type) {
-    case CONFIG_TYPE_STRING:
-    case CONFIG_TYPE_FILENAME:
-      tor_free(*(char**)lvalue);
-      break;
-    case CONFIG_TYPE_DOUBLE:
-      *(double*)lvalue = 0.0;
-      break;
-    case CONFIG_TYPE_ISOTIME:
-      *(time_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_CSV_INTERVAL:
-    case CONFIG_TYPE_INTERVAL:
-    case CONFIG_TYPE_MSEC_INTERVAL:
-    case CONFIG_TYPE_UINT:
-    case CONFIG_TYPE_INT:
-    case CONFIG_TYPE_PORT:
-    case CONFIG_TYPE_BOOL:
-      *(int*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_AUTOBOOL:
-      *(int*)lvalue = -1;
-      break;
-    case CONFIG_TYPE_UINT64:
-    case CONFIG_TYPE_MEMUNIT:
-      *(uint64_t*)lvalue = 0;
-      break;
-    case CONFIG_TYPE_ROUTERSET:
-      if (*(routerset_t**)lvalue) {
-        routerset_free(*(routerset_t**)lvalue);
-        *(routerset_t**)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_CSV:
-      if (*(smartlist_t**)lvalue) {
-        SMARTLIST_FOREACH(*(smartlist_t **)lvalue, char *, cp, tor_free(cp));
-        smartlist_free(*(smartlist_t **)lvalue);
-        *(smartlist_t **)lvalue = NULL;
-      }
-      break;
-    case CONFIG_TYPE_LINELIST:
-    case CONFIG_TYPE_LINELIST_S:
-      config_free_lines(*(config_line_t **)lvalue);
-      *(config_line_t **)lvalue = NULL;
-      break;
-    case CONFIG_TYPE_LINELIST_V:
-      /* handled by linelist_s. */
-      break;
-    case CONFIG_TYPE_OBSOLETE:
-      break;
-  }
-}
-
-/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
- * <b>use_defaults</b>, set it to its default value.
- * Called by config_init() and option_reset_line() and option_assign_line(). */
-static void
-config_reset(const config_format_t *fmt, void *options,
-             const config_var_t *var, int use_defaults)
-{
-  config_line_t *c;
-  char *msg = NULL;
-  CONFIG_CHECK(fmt, options);
-  config_clear(fmt, options, var); /* clear it first */
-  if (!use_defaults)
-    return; /* all done */
-  if (var->initvalue) {
-    c = tor_malloc_zero(sizeof(config_line_t));
-    c->key = tor_strdup(var->name);
-    c->value = tor_strdup(var->initvalue);
-    if (config_assign_value(fmt, options, c, &msg) < 0) {
-      log_warn(LD_BUG, "Failed to assign default: %s", msg);
-      tor_free(msg); /* if this happens it's a bug */
-    }
-    config_free_lines(c);
-  }
-}
-
-/** Release storage held by <b>options</b>. */
-void
-config_free_(const config_format_t *fmt, void *options)
-{
-  int i;
-
-  if (!options)
-    return;
-
-  tor_assert(fmt);
-
-  for (i=0; fmt->vars[i].name; ++i)
-    config_clear(fmt, options, &(fmt->vars[i]));
-  if (fmt->extra) {
-    config_line_t **linep = STRUCT_VAR_P(options, fmt->extra->var_offset);
-    config_free_lines(*linep);
-    *linep = NULL;
-  }
-  tor_free(options);
-}
-
-/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
- * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
- */
-int
-config_is_same(const config_format_t *fmt,
-               const void *o1, const void *o2,
-               const char *name)
-{
-  config_line_t *c1, *c2;
-  int r = 1;
-  CONFIG_CHECK(fmt, o1);
-  CONFIG_CHECK(fmt, o2);
-
-  c1 = config_get_assigned_option(fmt, o1, name, 0);
-  c2 = config_get_assigned_option(fmt, o2, name, 0);
-  r = config_lines_eq(c1, c2);
-  config_free_lines(c1);
-  config_free_lines(c2);
-  return r;
-}
-
-/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
-void *
-config_dup(const config_format_t *fmt, const void *old)
-{
-  void *newopts;
-  int i;
-  config_line_t *line;
-
-  newopts = config_new(fmt);
-  for (i=0; fmt->vars[i].name; ++i) {
-    if (fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE)
-      continue;
-    line = config_get_assigned_option(fmt, old, fmt->vars[i].name, 0);
-    if (line) {
-      char *msg = NULL;
-      if (config_assign(fmt, newopts, line, 0, &msg) < 0) {
-        log_err(LD_BUG, "config_get_assigned_option() generated "
-                "something we couldn't config_assign(): %s", msg);
-        tor_free(msg);
-        tor_assert(0);
-      }
-    }
-    config_free_lines(line);
-  }
-  return newopts;
-}
-/** Set all vars in the configuration object <b>options</b> to their default
- * values. */
-void
-config_init(const config_format_t *fmt, void *options)
-{
-  int i;
-  const config_var_t *var;
-  CONFIG_CHECK(fmt, options);
-
-  for (i=0; fmt->vars[i].name; ++i) {
-    var = &fmt->vars[i];
-    if (!var->initvalue)
-      continue; /* defaults to NULL or 0 */
-    config_reset(fmt, options, var, 1);
-  }
-}
-
-/** Allocate and return a new string holding the written-out values of the vars
- * in 'options'.  If 'minimal', do not write out any default-valued vars.
- * Else, if comment_defaults, write default values as comments.
- */
-char *
-config_dump(const config_format_t *fmt, const void *default_options,
-            const void *options, int minimal,
-            int comment_defaults)
-{
-  smartlist_t *elements;
-  const void *defaults = default_options;
-  void *defaults_tmp = NULL;
-  config_line_t *line, *assigned;
-  char *result;
-  int i;
-  char *msg = NULL;
-
-  if (defaults == NULL) {
-    defaults = defaults_tmp = config_new(fmt);
-    config_init(fmt, defaults_tmp);
-  }
-
-  /* XXX use a 1 here so we don't add a new log line while dumping */
-  if (default_options == NULL) {
-    if (fmt->validate_fn(NULL, defaults_tmp, defaults_tmp, 1, &msg) < 0) {
-      log_err(LD_BUG, "Failed to validate default config: %s", msg);
-      tor_free(msg);
-      tor_assert(0);
-    }
-  }
-
-  elements = smartlist_new();
-  for (i=0; fmt->vars[i].name; ++i) {
-    int comment_option = 0;
-    if (fmt->vars[i].type == CONFIG_TYPE_OBSOLETE ||
-        fmt->vars[i].type == CONFIG_TYPE_LINELIST_S)
-      continue;
-    /* Don't save 'hidden' control variables. */
-    if (!strcmpstart(fmt->vars[i].name, "__"))
-      continue;
-    if (minimal && config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      continue;
-    else if (comment_defaults &&
-             config_is_same(fmt, options, defaults, fmt->vars[i].name))
-      comment_option = 1;
-
-    line = assigned =
-      config_get_assigned_option(fmt, options, fmt->vars[i].name, 1);
-
-    for (; line; line = line->next) {
-      if (!strcmpstart(line->key, "__")) {
-        /* This check detects "hidden" variables inside LINELIST_V structures.
-         */
-        continue;
-      }
-      smartlist_add_asprintf(elements, "%s%s %s\n",
-                   comment_option ? "# " : "",
-                   line->key, line->value);
-    }
-    config_free_lines(assigned);
-  }
-
-  if (fmt->extra) {
-    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->var_offset);
-    for (; line; line = line->next) {
-      smartlist_add_asprintf(elements, "%s %s\n", line->key, line->value);
-    }
-  }
-
-  result = smartlist_join_strings(elements, "", 0, NULL);
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-  if (defaults_tmp) {
-    fmt->free_fn(defaults_tmp);
-  }
-  return result;
-}
-
-/** Mapping from a unit name to a multiplier for converting that unit into a
- * base unit.  Used by config_parse_unit. */
-struct unit_table_t {
-  const char *unit; /**< The name of the unit */
-  uint64_t multiplier; /**< How many of the base unit appear in this unit */
-};
-
-/** Table to map the names of memory units to the number of bytes they
- * contain. */
-static struct unit_table_t memory_units[] = {
-  { "",          1 },
-  { "b",         1<< 0 },
-  { "byte",      1<< 0 },
-  { "bytes",     1<< 0 },
-  { "kb",        1<<10 },
-  { "kbyte",     1<<10 },
-  { "kbytes",    1<<10 },
-  { "kilobyte",  1<<10 },
-  { "kilobytes", 1<<10 },
-  { "kilobits",  1<<7  },
-  { "kilobit",   1<<7  },
-  { "kbits",     1<<7  },
-  { "kbit",      1<<7  },
-  { "m",         1<<20 },
-  { "mb",        1<<20 },
-  { "mbyte",     1<<20 },
-  { "mbytes",    1<<20 },
-  { "megabyte",  1<<20 },
-  { "megabytes", 1<<20 },
-  { "megabits",  1<<17 },
-  { "megabit",   1<<17 },
-  { "mbits",     1<<17 },
-  { "mbit",      1<<17 },
-  { "gb",        1<<30 },
-  { "gbyte",     1<<30 },
-  { "gbytes",    1<<30 },
-  { "gigabyte",  1<<30 },
-  { "gigabytes", 1<<30 },
-  { "gigabits",  1<<27 },
-  { "gigabit",   1<<27 },
-  { "gbits",     1<<27 },
-  { "gbit",      1<<27 },
-  { "tb",        UINT64_C(1)<<40 },
-  { "tbyte",     UINT64_C(1)<<40 },
-  { "tbytes",    UINT64_C(1)<<40 },
-  { "terabyte",  UINT64_C(1)<<40 },
-  { "terabytes", UINT64_C(1)<<40 },
-  { "terabits",  UINT64_C(1)<<37 },
-  { "terabit",   UINT64_C(1)<<37 },
-  { "tbits",     UINT64_C(1)<<37 },
-  { "tbit",      UINT64_C(1)<<37 },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of seconds they
- * contain. */
-static struct unit_table_t time_units[] = {
-  { "",         1 },
-  { "second",   1 },
-  { "seconds",  1 },
-  { "minute",   60 },
-  { "minutes",  60 },
-  { "hour",     60*60 },
-  { "hours",    60*60 },
-  { "day",      24*60*60 },
-  { "days",     24*60*60 },
-  { "week",     7*24*60*60 },
-  { "weeks",    7*24*60*60 },
-  { "month",    2629728, }, /* about 30.437 days */
-  { "months",   2629728, },
-  { NULL, 0 },
-};
-
-/** Table to map the names of time units to the number of milliseconds
- * they contain. */
-static struct unit_table_t time_msec_units[] = {
-  { "",         1 },
-  { "msec",     1 },
-  { "millisecond", 1 },
-  { "milliseconds", 1 },
-  { "second",   1000 },
-  { "seconds",  1000 },
-  { "minute",   60*1000 },
-  { "minutes",  60*1000 },
-  { "hour",     60*60*1000 },
-  { "hours",    60*60*1000 },
-  { "day",      24*60*60*1000 },
-  { "days",     24*60*60*1000 },
-  { "week",     7*24*60*60*1000 },
-  { "weeks",    7*24*60*60*1000 },
-  { NULL, 0 },
-};
-
-/** Parse a string <b>val</b> containing a number, zero or more
- * spaces, and an optional unit string.  If the unit appears in the
- * table <b>u</b>, then multiply the number by the unit multiplier.
- * On success, set *<b>ok</b> to 1 and return this product.
- * Otherwise, set *<b>ok</b> to 0.
- */
-static uint64_t
-config_parse_units(const char *val, struct unit_table_t *u, int *ok)
-{
-  uint64_t v = 0;
-  double d = 0;
-  int use_float = 0;
-  char *cp;
-
-  tor_assert(ok);
-
-  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
-  if (!*ok || (cp && *cp == '.')) {
-    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
-    if (!*ok)
-      goto done;
-    use_float = 1;
-  }
-
-  if (!cp) {
-    *ok = 1;
-    v = use_float ? ((uint64_t)d) :  v;
-    goto done;
-  }
-
-  cp = (char*) eat_whitespace(cp);
-
-  for ( ;u->unit;++u) {
-    if (!strcasecmp(u->unit, cp)) {
-      if (use_float)
-        v = (uint64_t)(u->multiplier * d);
-      else
-        v *= u->multiplier;
-      *ok = 1;
-      goto done;
-    }
-  }
-  log_warn(LD_CONFIG, "Unknown unit '%s'.", cp);
-  *ok = 0;
- done:
-
-  if (*ok)
-    return v;
-  else
-    return 0;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
- * and return the number of bytes specified.  Otherwise, set
- * *<b>ok</b> to false and return 0. */
-static uint64_t
-config_parse_memunit(const char *s, int *ok)
-{
-  uint64_t u = config_parse_units(s, memory_units, ok);
-  return u;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of
- * time in milliseconds.  On success, set *<b>ok</b> to true and return
- * the number of milliseconds in the provided interval.  Otherwise, set
- * *<b>ok</b> to 0 and return -1. */
-static int
-config_parse_msec_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_msec_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
-
-/** Parse a string in the format "number unit", where unit is a unit of time.
- * On success, set *<b>ok</b> to true and return the number of seconds in
- * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
- */
-static int
-config_parse_interval(const char *s, int *ok)
-{
-  uint64_t r;
-  r = config_parse_units(s, time_units, ok);
-  if (r > INT_MAX) {
-    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
-    *ok = 0;
-    return -1;
-  }
-  return (int)r;
-}
diff --git a/src/app/config/confparse.h b/src/app/config/confparse.h
deleted file mode 100644
index 57f1ec1762..0000000000
--- a/src/app/config/confparse.h
+++ /dev/null
@@ -1,233 +0,0 @@
-/* Copyright (c) 2001 Matej Pfajfar.
- * Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file confparse.h
- *
- * \brief Header for confparse.c.
- */
-
-#ifndef TOR_CONFPARSE_H
-#define TOR_CONFPARSE_H
-
-/** Enumeration of types which option values can take */
-typedef enum config_type_t {
-  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
-  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
-  CONFIG_TYPE_UINT,         /**< A non-negative integer less than MAX_INT */
-  CONFIG_TYPE_INT,          /**< Any integer. */
-  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
-  CONFIG_TYPE_PORT,         /**< A port from 1...65535, 0 for "not set", or
-                             * "auto".  */
-  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
-  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
-                              * units */
-  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
-  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
-  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
-  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
-                             * 1 for true, and -1 for auto  */
-  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
-  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
-                              * optional whitespace. */
-  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
-                              * optional whitespace, representing intervals in
-                              * seconds, with optional units.  We allow
-                              * multiple values here for legacy reasons, but
-                              * ignore every value after the first. */
-  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
-  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
-                             * mixed with other keywords. */
-  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
-                             * context-sensitive config lines when fetching.
-                             */
-  CONFIG_TYPE_ROUTERSET,    /**< A list of router names, addrs, and fps,
-                             * parsed into a routerset_t. */
-  CONFIG_TYPE_OBSOLETE,     /**< Obsolete (ignored) option. */
-} config_type_t;
-
-#ifdef TOR_UNIT_TESTS
-/**
- * Union used when building in test mode typechecking the members of a type
- * used with confparse.c.  See CONF_CHECK_VAR_TYPE for a description of how
- * it is used. */
-typedef union {
-  char **STRING;
-  char **FILENAME;
-  int *UINT; /* yes, really: Even though the confparse type is called
-              * "UINT", it still uses the C int type -- it just enforces that
-              * the values are in range [0,INT_MAX].
-              */
-  uint64_t *UINT64;
-  int *INT;
-  int *PORT;
-  int *INTERVAL;
-  int *MSEC_INTERVAL;
-  uint64_t *MEMUNIT;
-  double *DOUBLE;
-  int *BOOL;
-  int *AUTOBOOL;
-  time_t *ISOTIME;
-  smartlist_t **CSV;
-  int *CSV_INTERVAL;
-  struct config_line_t **LINELIST;
-  struct config_line_t **LINELIST_S;
-  struct config_line_t **LINELIST_V;
-  routerset_t **ROUTERSET;
-} confparse_dummy_values_t;
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** An abbreviation for a configuration option allowed on the command line. */
-typedef struct config_abbrev_t {
-  const char *abbreviated;
-  const char *full;
-  int commandline_only;
-  int warn;
-} config_abbrev_t;
-
-typedef struct config_deprecation_t {
-  const char *name;
-  const char *why_deprecated;
-} config_deprecation_t;
-
-/* Handy macro for declaring "In the config file or on the command line,
- * you can abbreviate <b>tok</b>s as <b>tok</b>". */
-#define PLURAL(tok) { #tok, #tok "s", 0, 0 }
-
-/** A variable allowed in the configuration file or on the command line. */
-typedef struct config_var_t {
-  const char *name; /**< The full keyword (case insensitive). */
-  config_type_t type; /**< How to interpret the type and turn it into a
-                       * value. */
-  off_t var_offset; /**< Offset of the corresponding member of or_options_t. */
-  const char *initvalue; /**< String (or null) describing initial value. */
-
-#ifdef TOR_UNIT_TESTS
-  /** Used for compiler-magic to typecheck the corresponding field in the
-   * corresponding struct. Only used in unit test mode, at compile-time. */
-  confparse_dummy_values_t var_ptr_dummy;
-#endif
-} config_var_t;
-
-/* Macros to define extra members inside config_var_t fields, and at the
- * end of a list of them.
- */
-#ifdef TOR_UNIT_TESTS
-/* This is a somewhat magic type-checking macro for users of confparse.c.
- * It initializes a union member "confparse_dummy_values_t.conftype" with
- * the address of a static member "tp_dummy.member".   This
- * will give a compiler warning unless the member field is of the correct
- * type.
- *
- * (This warning is mandatory, because a type mismatch here violates the type
- * compatibility constraint for simple assignment, and requires a diagnostic,
- * according to the C spec.)
- *
- * For example, suppose you say:
- *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
- * Then this macro will evaluate to:
- *     { .STRING = &or_options_t_dummy.Address }
- * And since confparse_dummy_values_t.STRING has type "char **", that
- * expression will create a warning unless or_options_t.Address also
- * has type "char *".
- */
-#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
-  { . conftype = &tp ## _dummy . member }
-#define CONF_TEST_MEMBERS(tp, conftype, member) \
-  , CONF_CHECK_VAR_TYPE(tp, conftype, member)
-#define END_OF_CONFIG_VARS                                      \
-  { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL, { .INT=NULL } }
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  static tp tp ## _dummy
-#else /* !(defined(TOR_UNIT_TESTS)) */
-#define CONF_TEST_MEMBERS(tp, conftype, member)
-#define END_OF_CONFIG_VARS { NULL, CONFIG_TYPE_OBSOLETE, 0, NULL }
-/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
-#define DUMMY_TYPECHECK_INSTANCE(tp)            \
-  struct tor_semicolon_eater
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Type of a callback to validate whether a given configuration is
- * well-formed and consistent. See options_trial_assign() for documentation
- * of arguments. */
-typedef int (*validate_fn_t)(void*,void*,void*,int,char**);
-
-/** Callback to free a configuration object. */
-typedef void (*free_cfg_fn_t)(void*);
-
-/** Information on the keys, value types, key-to-struct-member mappings,
- * variable descriptions, validation functions, and abbreviations for a
- * configuration or storage format. */
-typedef struct config_format_t {
-  size_t size; /**< Size of the struct that everything gets parsed into. */
-  uint32_t magic; /**< Required 'magic value' to make sure we have a struct
-                   * of the right type. */
-  off_t magic_offset; /**< Offset of the magic value within the struct. */
-  config_abbrev_t *abbrevs; /**< List of abbreviations that we expand when
-                             * parsing this format. */
-  const config_deprecation_t *deprecations; /** List of deprecated options */
-  config_var_t *vars; /**< List of variables we recognize, their default
-                       * values, and where we stick them in the structure. */
-  validate_fn_t validate_fn; /**< Function to validate config. */
-  free_cfg_fn_t free_fn; /**< Function to free the configuration. */
-  /** If present, extra is a LINELIST variable for unrecognized
-   * lines.  Otherwise, unrecognized lines are an error. */
-  config_var_t *extra;
-} config_format_t;
-
-/** Macro: assert that <b>cfg</b> has the right magic field for format
- * <b>fmt</b>. */
-#define CONFIG_CHECK(fmt, cfg) STMT_BEGIN                               \
-    tor_assert(fmt && cfg);                                             \
-    tor_assert((fmt)->magic ==                                          \
-               *(uint32_t*)STRUCT_VAR_P(cfg,fmt->magic_offset));        \
-  STMT_END
-
-#define CAL_USE_DEFAULTS      (1u<<0)
-#define CAL_CLEAR_FIRST       (1u<<1)
-#define CAL_WARN_DEPRECATIONS (1u<<2)
-
-void *config_new(const config_format_t *fmt);
-void config_free_(const config_format_t *fmt, void *options);
-#define config_free(fmt, options) do {                \
-    config_free_((fmt), (options));                   \
-    (options) = NULL;                                 \
-  } while (0)
-
-struct config_line_t *config_get_assigned_option(const config_format_t *fmt,
-                                          const void *options, const char *key,
-                                          int escape_val);
-int config_is_same(const config_format_t *fmt,
-                   const void *o1, const void *o2,
-                   const char *name);
-void config_init(const config_format_t *fmt, void *options);
-void *config_dup(const config_format_t *fmt, const void *old);
-char *config_dump(const config_format_t *fmt, const void *default_options,
-                  const void *options, int minimal,
-                  int comment_defaults);
-int config_assign(const config_format_t *fmt, void *options,
-                  struct config_line_t *list,
-                  unsigned flags, char **msg);
-config_var_t *config_find_option_mutable(config_format_t *fmt,
-                                         const char *key);
-const char *config_find_deprecation(const config_format_t *fmt,
-                                     const char *key);
-const config_var_t *config_find_option(const config_format_t *fmt,
-                                       const char *key);
-const char *config_expand_abbrev(const config_format_t *fmt,
-                                 const char *option,
-                                 int command_line, int warn_obsolete);
-void warn_deprecated_option(const char *what, const char *why);
-
-/* Helper macros to compare an option across two configuration objects */
-#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
-#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
-#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
-#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
-#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
-
-#endif /* !defined(TOR_CONFPARSE_H) */
diff --git a/src/app/config/include.am b/src/app/config/include.am
new file mode 100644
index 0000000000..14320a6b11
--- /dev/null
+++ b/src/app/config/include.am
@@ -0,0 +1,23 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/app/config/config.c			\
+	src/app/config/quiet_level.c		\
+	src/app/config/resolve_addr.c		\
+	src/app/config/statefile.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/app/config/config.h				\
+	src/app/config/or_options_st.h			\
+	src/app/config/or_state_st.h			\
+	src/app/config/quiet_level.h			\
+	src/app/config/resolve_addr.h			\
+	src/app/config/statefile.h			\
+	src/app/config/tor_cmdline_mode.h
+
+
+noinst_HEADERS +=				\
+	src/app/config/auth_dirs.inc		\
+	src/app/config/fallback_dirs.inc	\
+	src/app/config/testnet.inc
diff --git a/src/app/config/or_options_st.h b/src/app/config/or_options_st.h
index 74d2fefa16..bf58205f89 100644
--- a/src/app/config/or_options_st.h
+++ b/src/app/config/or_options_st.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,11 +13,15 @@
 #ifndef TOR_OR_OPTIONS_ST_H
 #define TOR_OR_OPTIONS_ST_H
 
+#include "core/or/or.h"
 #include "lib/cc/torint.h"
 #include "lib/net/address.h"
+#include "app/config/tor_cmdline_mode.h"
 
 struct smartlist_t;
 struct config_line_t;
+struct config_suite_t;
+struct routerset_t;
 
 /** Enumeration of outbound address configuration types:
  * Exit-only, OR-only, or both */
@@ -25,17 +29,18 @@ typedef enum {OUTBOUND_ADDR_EXIT, OUTBOUND_ADDR_OR,
               OUTBOUND_ADDR_EXIT_AND_OR,
               OUTBOUND_ADDR_MAX} outbound_addr_t;
 
+/** Which protocol to use for TCPProxy. */
+typedef enum {
+  /** Use the HAProxy proxy protocol. */
+  TCP_PROXY_PROTOCOL_HAPROXY
+} tcp_proxy_protocol_t;
+
 /** Configuration options for a Tor process. */
 struct or_options_t {
   uint32_t magic_;
 
   /** What should the tor process actually do? */
-  enum {
-    CMD_RUN_TOR=0, CMD_LIST_FINGERPRINT, CMD_HASH_PASSWORD,
-    CMD_VERIFY_CONFIG, CMD_RUN_UNITTESTS, CMD_DUMP_CONFIG,
-    CMD_KEYGEN,
-    CMD_KEY_EXPIRATION,
-  } command;
+  tor_cmdline_mode_t command;
   char *command_arg; /**< Argument for command-line option. */
 
   struct config_line_t *Logs; /**< New-style list of configuration lines
@@ -69,25 +74,29 @@ struct or_options_t {
   char *Address; /**< OR only: configured address for this onion router. */
   char *PidFile; /**< Where to store PID of Tor process. */
 
-  routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
+  struct routerset_t *ExitNodes; /**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as exits. */
-  routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
+  struct routerset_t *MiddleNodes; /**< Structure containing nicknames,
+                             * digests, country codes and IP address patterns
+                             * of ORs to consider as middles. */
+  struct routerset_t *EntryNodes;/**< Structure containing nicknames, digests,
                            * country codes and IP address patterns of ORs to
                            * consider as entry points. */
   int StrictNodes; /**< Boolean: When none of our EntryNodes or ExitNodes
                     * are up, or we need to access a node in ExcludeNodes,
                     * do we just fail instead? */
-  routerset_t *ExcludeNodes;/**< Structure containing nicknames, digests,
-                             * country codes and IP address patterns of ORs
-                             * not to use in circuits. But see StrictNodes
-                             * above. */
-  routerset_t *ExcludeExitNodes;/**< Structure containing nicknames, digests,
-                                 * country codes and IP address patterns of
-                                 * ORs not to consider as exits. */
+  struct routerset_t *ExcludeNodes;/**< Structure containing nicknames,
+                             * digests, country codes and IP address patterns
+                             * of ORs not to use in circuits. But see
+                             * StrictNodes above. */
+  struct routerset_t *ExcludeExitNodes;/**< Structure containing nicknames,
+                                 * digests, country codes and IP address
+                                 * patterns of ORs not to consider as
+                                 * exits. */
 
   /** Union of ExcludeNodes and ExcludeExitNodes */
-  routerset_t *ExcludeExitNodesUnion_;
+  struct routerset_t *ExcludeExitNodesUnion_;
 
   int DisableAllSwap; /**< Boolean: Attempt to call mlockall() on our
                        * process for all current and future memory. */
@@ -113,12 +122,6 @@ struct or_options_t {
    * [][0] is IPv4, [][1] is IPv6
    */
   tor_addr_t OutboundBindAddresses[OUTBOUND_ADDR_MAX][2];
-  /** Directory server only: which versions of
-   * Tor should we tell users to run? */
-  struct config_line_t *RecommendedVersions;
-  struct config_line_t *RecommendedClientVersions;
-  struct config_line_t *RecommendedServerVersions;
-  struct config_line_t *RecommendedPackages;
   /** Whether dirservers allow router descriptors with private IPs. */
   int DirAllowPrivateAddresses;
   /** Whether routers accept EXTEND cells to routers with private IPs. */
@@ -193,9 +196,6 @@ struct or_options_t {
   int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
   int V3AuthoritativeDir; /**< Boolean: is this an authoritative directory
                            * for version 3 directories? */
-  int VersioningAuthoritativeDir; /**< Boolean: is this an authoritative
-                                   * directory that's willing to recommend
-                                   * versions? */
   int BridgeAuthoritativeDir; /**< Boolean: is this an authoritative directory
                                * that aggregates bridge descriptors? */
 
@@ -245,6 +245,17 @@ struct or_options_t {
    * pad to the server regardless of server support. */
   int ConnectionPadding;
 
+  /** Boolean: if true, then circuit padding will be negotiated by client
+   * and server, subject to consenus limits (default). If 0, it will be fully
+   * disabled. */
+  int CircuitPadding;
+
+  /** Boolean: if true, then this client will only use circuit padding
+   * algorithms that are known to use a low amount of overhead. If false,
+   * we will use all available circuit padding algorithms.
+   */
+  int ReducedCircuitPadding;
+
   /** To what authority types do we publish our descriptor? Choices are
    * "v1", "v2", "v3", "bridge", or "". */
   struct smartlist_t *PublishServerDescriptor;
@@ -255,20 +266,17 @@ struct or_options_t {
   int FetchServerDescriptors; /**< Do we fetch server descriptors as normal? */
   int FetchHidServDescriptors; /**< and hidden service descriptors? */
 
-  int MinUptimeHidServDirectoryV2; /**< As directory authority, accept hidden
-                                    * service directories after what time? */
-
   int FetchUselessDescriptors; /**< Do we fetch non-running descriptors too? */
   int AllDirActionsPrivate; /**< Should every directory action be sent
                              * through a Tor circuit? */
 
   /** A routerset that should be used when picking middle nodes for HS
    *  circuits. */
-  routerset_t *HSLayer2Nodes;
+  struct routerset_t *HSLayer2Nodes;
 
   /** A routerset that should be used when picking third-hop nodes for HS
    *  circuits. */
-  routerset_t *HSLayer3Nodes;
+  struct routerset_t *HSLayer3Nodes;
 
   /** Onion Services in HiddenServiceSingleHopMode make one-hop (direct)
    * circuits between the onion service server, and the introduction and
@@ -409,6 +417,11 @@ struct or_options_t {
   char *Socks5ProxyUsername; /**< Username for SOCKS5 authentication, if any */
   char *Socks5ProxyPassword; /**< Password for SOCKS5 authentication, if any */
 
+  char *TCPProxy; /**< protocol and hostname:port to use as a proxy, if any. */
+  tcp_proxy_protocol_t TCPProxyProtocol; /**< Derived from TCPProxy. */
+  tor_addr_t TCPProxyAddr; /**< Derived from TCPProxy. */
+  uint16_t TCPProxyPort; /**< Derived from TCPProxy. */
+
   /** List of configuration lines for replacement directory authorities.
    * If you just want to replace one class of authority at a time,
    * use the "Alternate*Authority" options below instead. */
@@ -455,21 +468,6 @@ struct or_options_t {
   struct smartlist_t *AuthDirRejectCCs;
   /**@}*/
 
-  int AuthDirListBadExits; /**< True iff we should list bad exits,
-                            * and vote for all other exits as good. */
-  int AuthDirMaxServersPerAddr; /**< Do not permit more than this
-                                 * number of servers per IP address. */
-  int AuthDirHasIPv6Connectivity; /**< Boolean: are we on IPv6?  */
-  int AuthDirPinKeys; /**< Boolean: Do we enforce key-pinning? */
-
-  /** If non-zero, always vote the Fast flag for any relay advertising
-   * this amount of capacity or more. */
-  uint64_t AuthDirFastGuarantee;
-
-  /** If non-zero, this advertised capacity or more is always sufficient
-   * to satisfy the bandwidth requirement for the Guard flag. */
-  uint64_t AuthDirGuardBWGuarantee;
-
   char *AccountingStart; /**< How long is the accounting interval, and when
                           * does it start? */
   uint64_t AccountingMax; /**< How many bytes do we allow per accounting
@@ -526,12 +524,8 @@ struct or_options_t {
                          * protocol, is it a warn or an info in our logs? */
   int TestSocks; /**< Boolean: when we get a socks connection, do we loudly
                   * log whether it was DNS-leaking or not? */
-  int HardwareAccel; /**< Boolean: Should we enable OpenSSL hardware
-                      * acceleration where available? */
   /** Token Bucket Refill resolution in milliseconds. */
   int TokenBucketRefillInterval;
-  char *AccelName; /**< Optional hardware acceleration engine name. */
-  char *AccelDir; /**< Optional hardware acceleration engine search dir. */
 
   /** Boolean: Do we try to enter from a smallish number
    * of fixed nodes? */
@@ -563,7 +557,9 @@ struct or_options_t {
 
   int DirCache; /**< Cache all directory documents and accept requests via
                  * tunnelled dir conns from clients. If 1, enabled (default);
-                 * If 0, disabled. */
+                 * If 0, disabled. Use dir_server_mode() rather than
+                 * referencing this option directly. (Except for routermode
+                 * and relay_config, which do direct checks.) */
 
   char *VirtualAddrNetworkIPv4; /**< Address and mask to hand out for virtual
                                  * MAPADDRESS requests for IPv4 addresses */
@@ -685,14 +681,6 @@ struct or_options_t {
   /** Location of guardfraction file */
   char *GuardfractionFile;
 
-  /** Authority only: key=value pairs that we add to our networkstatus
-   * consensus vote on the 'params' line. */
-  char *ConsensusParams;
-
-  /** Authority only: minimum number of measured bandwidths we must see
-   * before we only believe measured bandwidths to assign flags. */
-  int MinMeasuredBWsForAuthToIgnoreAdvertised;
-
   /** The length of time that we think an initial consensus should be fresh.
    * Only altered on testing networks. */
   int TestingV3AuthInitialVotingInterval;
@@ -709,16 +697,6 @@ struct or_options_t {
       voting. Only altered on testing networks. */
   int TestingV3AuthVotingStartOffset;
 
-  /** If an authority has been around for less than this amount of time, it
-   * does not believe its reachability information is accurate.  Only
-   * altered on testing networks. */
-  int TestingAuthDirTimeToLearnReachability;
-
-  /** Clients don't download any descriptor this recent, since it will
-   * probably not have propagated to enough caches.  Only altered on testing
-   * networks. */
-  int TestingEstimatedDescriptorPropagationTime;
-
   /** Schedule for when servers should download things in general.  Only
    * altered on testing networks. */
   int TestingServerDownloadInitialDelay;
@@ -792,27 +770,6 @@ struct or_options_t {
    * of certain configuration options. */
   int TestingTorNetwork;
 
-  /** Minimum value for the Exit flag threshold on testing networks. */
-  uint64_t TestingMinExitFlagThreshold;
-
-  /** Minimum value for the Fast flag threshold on testing networks. */
-  uint64_t TestingMinFastFlagThreshold;
-
-  /** Relays in a testing network which should be voted Exit
-   * regardless of exit policy. */
-  routerset_t *TestingDirAuthVoteExit;
-  int TestingDirAuthVoteExitIsStrict;
-
-  /** Relays in a testing network which should be voted Guard
-   * regardless of uptime and bandwidth. */
-  routerset_t *TestingDirAuthVoteGuard;
-  int TestingDirAuthVoteGuardIsStrict;
-
-  /** Relays in a testing network which should be voted HSDir
-   * regardless of uptime and DirPort. */
-  routerset_t *TestingDirAuthVoteHSDir;
-  int TestingDirAuthVoteHSDirIsStrict;
-
   /** Enable CONN_BW events.  Only altered on testing networks. */
   int TestingEnableConnBwEvent;
 
@@ -837,7 +794,7 @@ struct or_options_t {
    * to make this false. */
   int ReloadTorrcOnSIGHUP;
 
-  /* The main parameter for picking circuits within a connection.
+  /** The main parameter for picking circuits within a connection.
    *
    * If this value is positive, when picking a cell to relay on a connection,
    * we always relay from the circuit whose weighted cell count is lowest.
@@ -991,12 +948,6 @@ struct or_options_t {
    */
   uint64_t MaxUnparseableDescSizeToLog;
 
-  /** Bool (default: 1): Switch for the shared random protocol. Only
-   * relevant to a directory authority. If off, the authority won't
-   * participate in the protocol. If on (default), a flag is added to the
-   * vote indicating participation. */
-  int AuthDirSharedRandomness;
-
   /** If 1, we skip all OOS checks. */
   int DisableOOSCheck;
 
@@ -1004,11 +955,6 @@ struct or_options_t {
    * If -1, we should do whatever the consensus parameter says. */
   int ExtendByEd25519ID;
 
-  /** Bool (default: 1): When testing routerinfos as a directory authority,
-   * do we enforce Ed25519 identity match? */
-  /* NOTE: remove this option someday. */
-  int AuthDirTestEd25519LinkKeys;
-
   /** Bool (default: 0): Tells if a %include was used on torrc */
   int IncludeUsed;
 
@@ -1033,7 +979,7 @@ struct or_options_t {
   /** The list of scheduler type string ordered by priority that is first one
    * has to be tried first. Default: KIST,KISTLite,Vanilla */
   struct smartlist_t *Schedulers;
-  /* An ordered list of scheduler_types mapped from Schedulers. */
+  /** An ordered list of scheduler_types mapped from Schedulers. */
   struct smartlist_t *SchedulerTypes_;
 
   /** List of files that were opened by %include in torrc and torrc-defaults */
@@ -1072,6 +1018,33 @@ struct or_options_t {
 
   /** Autobool: Do we refuse single hop client rendezvous? */
   int DoSRefuseSingleHopClientRendezvous;
+
+  /** Interval: how long without activity does it take for a client
+   * to become dormant?
+   **/
+  int DormantClientTimeout;
+
+  /** Boolean: true if having an idle stream is sufficient to prevent a client
+   * from becoming dormant.
+   **/
+  int DormantTimeoutDisabledByIdleStreams;
+
+  /** Boolean: true if Tor should be dormant the first time it starts with
+   * a datadirectory; false otherwise. */
+  int DormantOnFirstStartup;
+  /**
+   * Boolean: true if Tor should treat every startup event as cancelling
+   * a possible previous dormant state.
+   **/
+  int DormantCanceledByStartup;
+
+  /**
+   * Configuration objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant configuration object.
+   */
+  struct config_suite_t *subconfigs_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_OPTIONS_ST_H) */
diff --git a/src/app/config/or_state_st.h b/src/app/config/or_state_st.h
index 5f8214d146..8c4e9d5e61 100644
--- a/src/app/config/or_state_st.h
+++ b/src/app/config/or_state_st.h
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file or_state_t
+ * \file or_state_st.h
  *
  * \brief The or_state_t structure, which represents Tor's state file.
  */
@@ -15,6 +15,7 @@
 
 #include "lib/cc/torint.h"
 struct smartlist_t;
+struct config_suite_t;
 
 /** Persistent state for an onion router, as saved to disk. */
 struct or_state_t {
@@ -87,6 +88,14 @@ struct or_state_t {
 
   /** When did we last rotate our onion key?  "0" for 'no idea'. */
   time_t LastRotatedOnionKey;
+
+  /**
+   * State objects for individual modules.
+   *
+   * Never access this field or its members directly: instead, use the module
+   * in question to get its relevant state object if you must.
+   */
+  struct config_suite_t *substates_;
 };
 
-#endif
+#endif /* !defined(TOR_OR_STATE_ST_H) */
diff --git a/src/app/config/quiet_level.c b/src/app/config/quiet_level.c
new file mode 100644
index 0000000000..e04faaef3a
--- /dev/null
+++ b/src/app/config/quiet_level.c
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file quiet_level.c
+ * @brief Code to handle default logging level (quiet/hush/normal).
+ **/
+
+#include "orconfig.h"
+#include "lib/log/log.h"
+#include "app/config/quiet_level.h"
+
+/** Decides our behavior when no logs are configured/before any logs have been
+ * configured.  For QUIET_NONE, we log notice to stdout as normal.  For
+ * QUIET_HUSH, we log warnings only.  For QUIET_SILENT, we log nothing.
+ */
+quiet_level_t quiet_level = 0;
+
+/** Add a default log (or not), depending on the value of <b>quiet</b>. */
+void
+add_default_log_for_quiet_level(quiet_level_t quiet)
+{
+  switch (quiet) {
+    case QUIET_SILENT:
+      /* --quiet: no initial logging */
+      return;
+    case QUIET_HUSH:
+    /* --hush: log at warning or higher. */
+      add_default_log(LOG_WARN);
+      break;
+    case QUIET_NONE: FALLTHROUGH;
+    default:
+      add_default_log(LOG_NOTICE);
+  }
+}
diff --git a/src/app/config/quiet_level.h b/src/app/config/quiet_level.h
new file mode 100644
index 0000000000..3a630b90e7
--- /dev/null
+++ b/src/app/config/quiet_level.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file quiet_level.h
+ * \brief Declare the quiet_level enumeration and global.
+ **/
+
+#ifndef QUIET_LEVEL_H
+#define QUIET_LEVEL_H
+
+/** Enumeration to define how quietly Tor should log at startup. */
+typedef enum {
+   /** Default quiet level: we log everything of level NOTICE or higher. */
+   QUIET_NONE = 0,
+   /** "--hush" quiet level: we log everything of level WARNING or higher. */
+   QUIET_HUSH = 1 ,
+   /** "--quiet" quiet level: we log nothing at all. */
+   QUIET_SILENT = 2
+} quiet_level_t;
+
+/** How quietly should Tor log at startup? */
+extern quiet_level_t quiet_level;
+
+void add_default_log_for_quiet_level(quiet_level_t quiet);
+
+#endif /* !defined(QUIET_LEVEL_H) */
diff --git a/src/app/config/resolve_addr.c b/src/app/config/resolve_addr.c
new file mode 100644
index 0000000000..9d1a8e0260
--- /dev/null
+++ b/src/app/config/resolve_addr.c
@@ -0,0 +1,314 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file resolve_addr.c
+ * \brief Implement resolving address functions
+ **/
+
+#define RESOLVE_ADDR_PRIVATE
+
+#include "app/config/config.h"
+#include "app/config/resolve_addr.h"
+
+#include "core/mainloop/mainloop.h"
+
+#include "feature/control/control_events.h"
+
+#include "lib/net/gethostname.h"
+#include "lib/net/resolve.h"
+
+/** Last value actually set by resolve_my_address. */
+static uint32_t last_resolved_addr = 0;
+
+/** Accessor for last_resolved_addr from outside this file. */
+uint32_t
+get_last_resolved_addr(void)
+{
+  return last_resolved_addr;
+}
+
+/** Reset last_resolved_addr from outside this file. */
+void
+reset_last_resolved_addr(void)
+{
+  last_resolved_addr = 0;
+}
+
+/**
+ * Attempt getting our non-local (as judged by tor_addr_is_internal()
+ * function) IP address using following techniques, listed in
+ * order from best (most desirable, try first) to worst (least
+ * desirable, try if everything else fails).
+ *
+ * First, attempt using <b>options-\>Address</b> to get our
+ * non-local IP address.
+ *
+ * If <b>options-\>Address</b> represents a non-local IP address,
+ * consider it ours.
+ *
+ * If <b>options-\>Address</b> is a DNS name that resolves to
+ * a non-local IP address, consider this IP address ours.
+ *
+ * If <b>options-\>Address</b> is NULL, fall back to getting local
+ * hostname and using it in above-described ways to try and
+ * get our IP address.
+ *
+ * In case local hostname cannot be resolved to a non-local IP
+ * address, try getting an IP address of network interface
+ * in hopes it will be non-local one.
+ *
+ * Fail if one or more of the following is true:
+ *   - DNS name in <b>options-\>Address</b> cannot be resolved.
+ *   - <b>options-\>Address</b> is a local host address.
+ *   - Attempt at getting local hostname fails.
+ *   - Attempt at getting network interface address fails.
+ *
+ * Return 0 if all is well, or -1 if we can't find a suitable
+ * public IP address.
+ *
+ * If we are returning 0:
+ *   - Put our public IP address (in host order) into *<b>addr_out</b>.
+ *   - If <b>method_out</b> is non-NULL, set *<b>method_out</b> to a static
+ *     string describing how we arrived at our answer.
+ *      - "CONFIGURED" - parsed from IP address string in
+ *        <b>options-\>Address</b>
+ *      - "RESOLVED" - resolved from DNS name in <b>options-\>Address</b>
+ *      - "GETHOSTNAME" - resolved from a local hostname.
+ *      - "INTERFACE" - retrieved from a network interface.
+ *   - If <b>hostname_out</b> is non-NULL, and we resolved a hostname to
+ *     get our address, set *<b>hostname_out</b> to a newly allocated string
+ *     holding that hostname. (If we didn't get our address by resolving a
+ *     hostname, set *<b>hostname_out</b> to NULL.)
+ *
+ * XXXX ipv6
+ */
+int
+resolve_my_address(int warn_severity, const or_options_t *options,
+                   uint32_t *addr_out,
+                   const char **method_out, char **hostname_out)
+{
+  struct in_addr in;
+  uint32_t addr; /* host order */
+  char hostname[256];
+  const char *method_used;
+  const char *hostname_used;
+  int explicit_ip=1;
+  int explicit_hostname=1;
+  int from_interface=0;
+  char *addr_string = NULL;
+  const char *address = options->Address;
+  int notice_severity = warn_severity <= LOG_NOTICE ?
+    LOG_NOTICE : warn_severity;
+
+  tor_addr_t myaddr;
+  tor_assert(addr_out);
+
+  /*
+   * Step one: Fill in 'hostname' to be our best guess.
+   */
+
+  if (address && *address) {
+    strlcpy(hostname, address, sizeof(hostname));
+    log_debug(LD_CONFIG, "Trying configured Address '%s' as local hostname",
+              hostname);
+  } else { /* then we need to guess our address */
+    explicit_ip = 0; /* it's implicit */
+    explicit_hostname = 0; /* it's implicit */
+
+    if (tor_gethostname(hostname, sizeof(hostname)) < 0) {
+      log_fn(warn_severity, LD_NET,"Error obtaining local hostname");
+      return -1;
+    }
+    log_debug(LD_CONFIG, "Guessed local host name as '%s'", hostname);
+  }
+
+  /*
+   * Step two: Now that we know 'hostname', parse it or resolve it. If
+   * it doesn't parse or resolve, look at the interface address. Set 'addr'
+   * to be our (host-order) 32-bit answer.
+   */
+
+  if (tor_inet_aton(hostname, &in) == 0) {
+    /* then we have to resolve it */
+    log_debug(LD_CONFIG, "Local hostname '%s' is DNS address. "
+              "Trying to resolve to IP address.", hostname);
+    explicit_ip = 0;
+    if (tor_lookup_hostname(hostname, &addr)) { /* failed to resolve */
+      uint32_t interface_ip; /* host order */
+
+      if (explicit_hostname) {
+        log_fn(warn_severity, LD_CONFIG,
+               "Could not resolve local Address '%s'. Failing.", hostname);
+        return -1;
+      }
+      log_fn(notice_severity, LD_CONFIG,
+             "Could not resolve guessed local hostname '%s'. "
+             "Trying something else.", hostname);
+      if (get_interface_address(warn_severity, &interface_ip)) {
+        log_fn(warn_severity, LD_CONFIG,
+               "Could not get local interface IP address. Failing.");
+        return -1;
+      }
+      from_interface = 1;
+      addr = interface_ip;
+      log_fn(notice_severity, LD_CONFIG, "Learned IP address '%s' for "
+             "local interface. Using that.", fmt_addr32(addr));
+      strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
+    } else { /* resolved hostname into addr */
+      tor_addr_from_ipv4h(&myaddr, addr);
+
+      if (!explicit_hostname &&
+          tor_addr_is_internal(&myaddr, 0)) {
+        tor_addr_t interface_ip;
+
+        log_fn(notice_severity, LD_CONFIG, "Guessed local hostname '%s' "
+               "resolves to a private IP address (%s). Trying something "
+               "else.", hostname, fmt_addr32(addr));
+
+        if (get_interface_address6(warn_severity, AF_INET, &interface_ip)<0) {
+          log_fn(warn_severity, LD_CONFIG,
+                 "Could not get local interface IP address. Too bad.");
+        } else if (tor_addr_is_internal(&interface_ip, 0)) {
+          log_fn(notice_severity, LD_CONFIG,
+                 "Interface IP address '%s' is a private address too. "
+                 "Ignoring.", fmt_addr(&interface_ip));
+        } else {
+          from_interface = 1;
+          addr = tor_addr_to_ipv4h(&interface_ip);
+          log_fn(notice_severity, LD_CONFIG,
+                 "Learned IP address '%s' for local interface."
+                 " Using that.", fmt_addr32(addr));
+          strlcpy(hostname, "<guessed from interfaces>", sizeof(hostname));
+        }
+      }
+    }
+  } else {
+    log_debug(LD_CONFIG, "Local hostname '%s' is already IP address, "
+              "skipping DNS resolution", hostname);
+    addr = ntohl(in.s_addr); /* set addr so that addr_string is not
+                              * illformed */
+  }
+
+  /*
+   * Step three: Check whether 'addr' is an internal IP address, and error
+   * out if it is and we don't want that.
+   */
+
+  tor_addr_from_ipv4h(&myaddr,addr);
+
+  addr_string = tor_dup_ip(addr);
+  if (addr_string && tor_addr_is_internal(&myaddr, 0)) {
+    /* make sure we're ok with publishing an internal IP */
+    if (using_default_dir_authorities(options)) {
+      /* if they are using the default authorities, disallow internal IPs
+       * always. For IPv6 ORPorts, this check is done in
+       * router_get_advertised_ipv6_or_ap(). See #33681. */
+      log_fn(warn_severity, LD_CONFIG,
+             "Address '%s' resolves to private IP address '%s'. "
+             "Tor servers that use the default DirAuthorities must have "
+             "public IP addresses.", hostname, addr_string);
+      tor_free(addr_string);
+      return -1;
+    }
+    if (!explicit_ip) {
+      /* even if they've set their own authorities, require an explicit IP if
+       * they're using an internal address. */
+      log_fn(warn_severity, LD_CONFIG, "Address '%s' resolves to private "
+             "IP address '%s'. Please set the Address config option to be "
+             "the IP address you want to use.", hostname, addr_string);
+      tor_free(addr_string);
+      return -1;
+    }
+  }
+
+  /*
+   * Step four: We have a winner! 'addr' is our answer for sure, and
+   * 'addr_string' is its string form. Fill out the various fields to
+   * say how we decided it.
+   */
+
+  log_debug(LD_CONFIG, "Resolved Address to '%s'.", addr_string);
+
+  if (explicit_ip) {
+    method_used = "CONFIGURED";
+    hostname_used = NULL;
+  } else if (explicit_hostname) {
+    method_used = "RESOLVED";
+    hostname_used = hostname;
+  } else if (from_interface) {
+    method_used = "INTERFACE";
+    hostname_used = NULL;
+  } else {
+    method_used = "GETHOSTNAME";
+    hostname_used = hostname;
+  }
+
+  *addr_out = addr;
+  if (method_out)
+    *method_out = method_used;
+  if (hostname_out)
+    *hostname_out = hostname_used ? tor_strdup(hostname_used) : NULL;
+
+  /*
+   * Step five: Check if the answer has changed since last time (or if
+   * there was no last time), and if so call various functions to keep
+   * us up-to-date.
+   */
+
+  if (last_resolved_addr && last_resolved_addr != *addr_out) {
+    /* Leave this as a notice, regardless of the requested severity,
+     * at least until dynamic IP address support becomes bulletproof. */
+    log_notice(LD_NET,
+               "Your IP address seems to have changed to %s "
+               "(METHOD=%s%s%s). Updating.",
+               addr_string, method_used,
+               hostname_used ? " HOSTNAME=" : "",
+               hostname_used ? hostname_used : "");
+    ip_address_changed(0);
+  }
+
+  if (last_resolved_addr != *addr_out) {
+    control_event_server_status(LOG_NOTICE,
+                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=%s%s%s",
+                                addr_string, method_used,
+                                hostname_used ? " HOSTNAME=" : "",
+                                hostname_used ? hostname_used : "");
+  }
+  last_resolved_addr = *addr_out;
+
+  /*
+   * And finally, clean up and return success.
+   */
+
+  tor_free(addr_string);
+  return 0;
+}
+
+/** Return true iff <b>addr</b> is judged to be on the same network as us, or
+ * on a private network.
+ */
+MOCK_IMPL(int,
+is_local_addr, (const tor_addr_t *addr))
+{
+  if (tor_addr_is_internal(addr, 0))
+    return 1;
+  /* Check whether ip is on the same /24 as we are. */
+  if (get_options()->EnforceDistinctSubnets == 0)
+    return 0;
+  if (tor_addr_family(addr) == AF_INET) {
+    uint32_t ip = tor_addr_to_ipv4h(addr);
+
+    /* It's possible that this next check will hit before the first time
+     * resolve_my_address actually succeeds.  (For clients, it is likely that
+     * resolve_my_address will never be called at all).  In those cases,
+     * last_resolved_addr will be 0, and so checking to see whether ip is on
+     * the same /24 as last_resolved_addr will be the same as checking whether
+     * it was on net 0, which is already done by tor_addr_is_internal.
+     */
+    if ((last_resolved_addr & (uint32_t)0xffffff00ul)
+        == (ip & (uint32_t)0xffffff00ul))
+      return 1;
+  }
+  return 0;
+}
diff --git a/src/app/config/resolve_addr.h b/src/app/config/resolve_addr.h
new file mode 100644
index 0000000000..3747546402
--- /dev/null
+++ b/src/app/config/resolve_addr.h
@@ -0,0 +1,28 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file resolve_addr.h
+ * \brief Header file for resolve_addr.c.
+ **/
+
+#ifndef TOR_CONFIG_RESOLVE_ADDR_H
+#define TOR_CONFIG_RESOLVE_ADDR_H
+
+#include "app/config/or_options_st.h"
+
+int resolve_my_address(int warn_severity, const or_options_t *options,
+                       uint32_t *addr_out,
+                       const char **method_out, char **hostname_out);
+
+uint32_t get_last_resolved_addr(void);
+void reset_last_resolved_addr(void);
+
+MOCK_DECL(int, is_local_addr, (const tor_addr_t *addr));
+
+#ifdef RESOLVE_ADDR_PRIVATE
+
+#endif /* RESOLVE_ADDR_PRIVATE */
+
+#endif /* TOR_CONFIG_RESOLVE_ADDR_H */
+
diff --git a/src/app/config/statefile.c b/src/app/config/statefile.c
index 89039a05b5..dcc55f1898 100644
--- a/src/app/config/statefile.c
+++ b/src/app/config/statefile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
  *
  * This 'state' file is a typed key-value store that allows multiple
  * entries for the same key.  It follows the same metaformat as described
- * in confparse.c, and uses the same code to read and write itself.
+ * in confmgt.c, and uses the same code to read and write itself.
  *
  * The state file is most suitable for small values that don't change too
  * frequently.  For values that become very large, we typically use a separate
@@ -32,10 +32,12 @@
 #include "core/or/or.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "feature/relay/transport_config.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/entrynodes.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/stats/rephist.h"
@@ -43,8 +45,10 @@
 #include "feature/relay/routermode.h"
 #include "lib/sandbox/sandbox.h"
 #include "app/config/statefile.h"
+#include "app/main/subsysmgr.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
@@ -68,16 +72,14 @@ static config_abbrev_t state_abbrevs_[] = {
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(or_state_t);
 
-/*XXXX these next two are duplicates or near-duplicates from config.c */
-#define VAR(name,conftype,member,initvalue)                             \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(or_state_t, member),       \
-      initvalue CONF_TEST_MEMBERS(or_state_t, conftype, member) }
-/** As VAR, but the option name and member name are the same. */
-#define V(member,conftype,initvalue)                                    \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(or_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
 
 /** Array of "state" variables saved to the ~/.tor/state file. */
-static config_var_t state_vars_[] = {
+// clang-format off
+static const config_var_t state_vars_[] = {
   /* Remember to document these in state-contents.txt ! */
 
   V(AccountingBytesReadInInterval,    MEMUNIT,  NULL),
@@ -103,19 +105,19 @@ static config_var_t state_vars_[] = {
   V(HidServRevCounter,            LINELIST, NULL),
 
   V(BWHistoryReadEnds,                ISOTIME,  NULL),
-  V(BWHistoryReadInterval,            UINT,     "900"),
+  V(BWHistoryReadInterval,            POSINT,     "900"),
   V(BWHistoryReadValues,              CSV,      ""),
   V(BWHistoryReadMaxima,              CSV,      ""),
   V(BWHistoryWriteEnds,               ISOTIME,  NULL),
-  V(BWHistoryWriteInterval,           UINT,     "900"),
+  V(BWHistoryWriteInterval,           POSINT,     "900"),
   V(BWHistoryWriteValues,             CSV,      ""),
   V(BWHistoryWriteMaxima,             CSV,      ""),
   V(BWHistoryDirReadEnds,             ISOTIME,  NULL),
-  V(BWHistoryDirReadInterval,         UINT,     "900"),
+  V(BWHistoryDirReadInterval,         POSINT,     "900"),
   V(BWHistoryDirReadValues,           CSV,      ""),
   V(BWHistoryDirReadMaxima,           CSV,      ""),
   V(BWHistoryDirWriteEnds,            ISOTIME,  NULL),
-  V(BWHistoryDirWriteInterval,        UINT,     "900"),
+  V(BWHistoryDirWriteInterval,        POSINT,     "900"),
   V(BWHistoryDirWriteValues,          CSV,      ""),
   V(BWHistoryDirWriteMaxima,          CSV,      ""),
 
@@ -126,48 +128,70 @@ static config_var_t state_vars_[] = {
   V(LastRotatedOnionKey,              ISOTIME,  NULL),
   V(LastWritten,                      ISOTIME,  NULL),
 
-  V(TotalBuildTimes,                  UINT,     NULL),
-  V(CircuitBuildAbandonedCount,       UINT,     "0"),
+  V(TotalBuildTimes,                  POSINT,     NULL),
+  V(CircuitBuildAbandonedCount,       POSINT,     "0"),
   VAR("CircuitBuildTimeBin",          LINELIST_S, BuildtimeHistogram, NULL),
   VAR("BuildtimeHistogram",           LINELIST_V, BuildtimeHistogram, NULL),
 
   END_OF_CONFIG_VARS
 };
+// clang-format on
 
 #undef VAR
 #undef V
 
 static int or_state_validate(or_state_t *state, char **msg);
 
-static int or_state_validate_cb(void *old_options, void *options,
-                                void *default_options,
-                                int from_setconf, char **msg);
-
-static void or_state_free_cb(void *state);
+static int or_state_validate_cb(const void *old_options,
+                                void *options, char **msg);
 
 /** Magic value for or_state_t. */
 #define OR_STATE_MAGIC 0x57A73f57
 
 /** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST, offsetof(or_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(or_state_t, LINELIST, ExtraLines)
+static struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(or_state_t, ExtraLines),
 };
 
 /** Configuration format for or_state_t. */
 static const config_format_t state_format = {
-  sizeof(or_state_t),
-  OR_STATE_MAGIC,
-  offsetof(or_state_t, magic_),
-  state_abbrevs_,
-  NULL,
-  state_vars_,
-  or_state_validate_cb,
-  or_state_free_cb,
-  &state_extra_var,
+  .size = sizeof(or_state_t),
+  .magic = {
+   "or_state_t",
+   OR_STATE_MAGIC,
+   offsetof(or_state_t, magic_),
+  },
+  .abbrevs = state_abbrevs_,
+  .vars = state_vars_,
+  .legacy_validate_fn = or_state_validate_cb,
+  .extra = &state_extra_var,
+  .has_config_suite = true,
+  .config_suite_offset = offsetof(or_state_t, substates_),
 };
 
+/* A global configuration manager for state-file objects */
+static config_mgr_t *state_mgr = NULL;
+
+/** Return the configuration manager for state-file objects. */
+STATIC const config_mgr_t *
+get_state_mgr(void)
+{
+  if (PREDICT_UNLIKELY(state_mgr == NULL)) {
+    state_mgr = config_mgr_new(&state_format);
+    int rv = subsystems_register_state_formats(state_mgr);
+    tor_assert(rv == 0);
+    config_mgr_freeze(state_mgr);
+  }
+  return state_mgr;
+}
+
+#define CHECK_STATE_MAGIC(s) STMT_BEGIN                        \
+    config_check_toplevel_magic(get_state_mgr(), (s));         \
+  STMT_END
+
 /** Persistent serialized state. */
 static or_state_t *global_state = NULL;
 
@@ -249,25 +273,6 @@ validate_transports_in_state(or_state_t *state)
   return 0;
 }
 
-static int
-or_state_validate_cb(void *old_state, void *state, void *default_state,
-                     int from_setconf, char **msg)
-{
-  /* We don't use these; only options do. Still, we need to match that
-   * signature. */
-  (void) from_setconf;
-  (void) default_state;
-  (void) old_state;
-
-  return or_state_validate(state, msg);
-}
-
-static void
-or_state_free_cb(void *state)
-{
-  or_state_free_(state);
-}
-
 /** Return 0 if every setting in <b>state</b> is reasonable, and a
  * permissible transition from <b>old_state</b>.  Else warn and return -1.
  * Should have no side effects, except for normalizing the contents of
@@ -276,6 +281,23 @@ or_state_free_cb(void *state)
 static int
 or_state_validate(or_state_t *state, char **msg)
 {
+  return config_validate(get_state_mgr(), NULL, state, msg);
+}
+
+/**
+ * Legacy validation/normalization callback for or_state_t.  See
+ * legacy_validate_fn_t for more information.
+ */
+static int
+or_state_validate_cb(const void *old_state, void *state_, char **msg)
+{
+  /* There is not a meaningful concept of a state-to-state transition,
+   * since we do not reload the state after we start. */
+  (void) old_state;
+  CHECK_STATE_MAGIC(state_);
+
+  or_state_t *state = state_;
+
   if (entry_guards_parse_state(state, 0, msg)<0)
     return -1;
 
@@ -292,8 +314,11 @@ or_state_set(or_state_t *new_state)
   char *err = NULL;
   int ret = 0;
   tor_assert(new_state);
-  config_free(&state_format, global_state);
+  config_free(get_state_mgr(), global_state);
   global_state = new_state;
+  if (subsystems_set_state(get_state_mgr(), global_state) < 0) {
+    ret = -1;
+  }
   if (entry_guards_parse_state(global_state, 1, &err)<0) {
     log_warn(LD_GENERAL,"%s",err);
     tor_free(err);
@@ -308,6 +333,7 @@ or_state_set(or_state_t *new_state)
       get_circuit_build_times_mutable(),global_state) < 0) {
     ret = -1;
   }
+
   return ret;
 }
 
@@ -353,9 +379,8 @@ or_state_save_broken(char *fname)
 STATIC or_state_t *
 or_state_new(void)
 {
-  or_state_t *new_state = tor_malloc_zero(sizeof(or_state_t));
-  new_state->magic_ = OR_STATE_MAGIC;
-  config_init(&state_format, new_state);
+  or_state_t *new_state = config_new(get_state_mgr());
+  config_init(get_state_mgr(), new_state);
 
   return new_state;
 }
@@ -396,7 +421,7 @@ or_state_load(void)
     int assign_retval;
     if (config_get_lines(contents, &lines, 0)<0)
       goto done;
-    assign_retval = config_assign(&state_format, new_state,
+    assign_retval = config_assign(get_state_mgr(), new_state,
                                   lines, 0, &errmsg);
     config_free_lines(lines);
     if (assign_retval<0)
@@ -423,7 +448,7 @@ or_state_load(void)
     or_state_save_broken(fname);
 
     tor_free(contents);
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
     new_state = or_state_new();
   } else if (contents) {
@@ -456,7 +481,7 @@ or_state_load(void)
   tor_free(fname);
   tor_free(contents);
   if (new_state)
-    config_free(&state_format, new_state);
+    config_free(get_state_mgr(), new_state);
 
   return r;
 }
@@ -496,9 +521,11 @@ or_state_save(time_t now)
 
   /* Call everything else that might dirty the state even more, in order
    * to avoid redundant writes. */
+  (void) subsystems_flush_state(get_state_mgr(), global_state);
   entry_guards_update_state(global_state);
   rep_hist_update_state(global_state);
   circuit_build_times_update_state(get_circuit_build_times(), global_state);
+
   if (accounting_is_enabled(get_options()))
     accounting_run_housekeeping(now);
 
@@ -507,7 +534,7 @@ or_state_save(time_t now)
   tor_free(global_state->TorVersion);
   tor_asprintf(&global_state->TorVersion, "Tor %s", get_version());
 
-  state = config_dump(&state_format, NULL, global_state, 1, 0);
+  state = config_dump(get_state_mgr(), NULL, global_state, 1, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&contents,
                "# Tor state file last generated on %s local time\n"
@@ -617,7 +644,7 @@ get_stored_bindaddr_for_server_transport(const char *transport)
   {
     /* See if the user explicitly asked for a specific listening
        address for this transport. */
-    char *conf_bindaddr = get_transport_bindaddr_from_config(transport);
+    char *conf_bindaddr = pt_get_bindaddr_from_config(transport);
     if (conf_bindaddr)
       return conf_bindaddr;
   }
@@ -717,7 +744,7 @@ or_state_free_(or_state_t *state)
   if (!state)
     return;
 
-  config_free(&state_format, state);
+  config_free(get_state_mgr(), state);
 }
 
 void
@@ -725,4 +752,5 @@ or_state_free_all(void)
 {
   or_state_free(global_state);
   global_state = NULL;
+  config_mgr_free(state_mgr);
 }
diff --git a/src/app/config/statefile.h b/src/app/config/statefile.h
index 1950078450..98d9d2dda1 100644
--- a/src/app/config/statefile.h
+++ b/src/app/config/statefile.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,6 +31,8 @@ STATIC struct config_line_t *get_transport_in_state_by_name(
 STATIC void or_state_free_(or_state_t *state);
 #define or_state_free(st) FREE_AND_NULL(or_state_t, or_state_free_, (st))
 STATIC or_state_t *or_state_new(void);
-#endif
+struct config_mgr_t;
+STATIC const struct config_mgr_t *get_state_mgr(void);
+#endif /* defined(STATEFILE_PRIVATE) */
 
 #endif /* !defined(TOR_STATEFILE_H) */
diff --git a/src/app/config/testnet.inc b/src/app/config/testnet.inc
new file mode 100644
index 0000000000..907c35f97c
--- /dev/null
+++ b/src/app/config/testnet.inc
@@ -0,0 +1,34 @@
+// When modifying, don't forget to update the defaults
+// for 'TestingTorNetwork' in 'doc/tor.1.txt'
+{ "DirAllowPrivateAddresses", "1" },
+{ "EnforceDistinctSubnets", "0" },
+{ "AssumeReachable", "1" },
+{ "AuthDirMaxServersPerAddr", "0" },
+{ "ClientBootstrapConsensusAuthorityDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusFallbackDownloadInitialDelay", "0" },
+{ "ClientBootstrapConsensusAuthorityOnlyDownloadInitialDelay", "0" },
+{ "ClientDNSRejectInternalAddresses", "0" },
+{ "ClientRejectInternalAddresses", "0" },
+{ "CountPrivateBandwidth", "1" },
+{ "ExitPolicyRejectPrivate", "0" },
+{ "ExtendAllowPrivateAddresses", "1" },
+{ "V3AuthVotingInterval", "5 minutes" },
+{ "V3AuthVoteDelay", "20 seconds" },
+{ "V3AuthDistDelay", "20 seconds" },
+{ "TestingV3AuthInitialVotingInterval", "150 seconds" },
+{ "TestingV3AuthInitialVoteDelay", "20 seconds" },
+{ "TestingV3AuthInitialDistDelay", "20 seconds" },
+{ "TestingAuthDirTimeToLearnReachability", "0 minutes" },
+{ "MinUptimeHidServDirectoryV2", "0 minutes" },
+{ "TestingServerDownloadInitialDelay", "0" },
+{ "TestingClientDownloadInitialDelay", "0" },
+{ "TestingServerConsensusDownloadInitialDelay", "0" },
+{ "TestingClientConsensusDownloadInitialDelay", "0" },
+{ "TestingBridgeDownloadInitialDelay", "10" },
+{ "TestingBridgeBootstrapDownloadInitialDelay", "0" },
+{ "TestingClientMaxIntervalWithoutRequest", "5 seconds" },
+{ "TestingDirConnectionMaxStall", "30 seconds" },
+{ "TestingEnableConnBwEvent", "1" },
+{ "TestingEnableCellStatsEvent", "1" },
+{ "RendPostPeriod", "2 minutes" },
+{ "___UsingTestNetworkDefaults", "1" },
diff --git a/src/app/config/tor_cmdline_mode.h b/src/app/config/tor_cmdline_mode.h
new file mode 100644
index 0000000000..30a339a438
--- /dev/null
+++ b/src/app/config/tor_cmdline_mode.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tor_cmdline_mode.h
+ * \brief Declare the tor_cmdline_mode_t enumeration
+ **/
+
+#ifndef TOR_CMDLINE_MODE_H
+#define TOR_CMDLINE_MODE_H
+
+/**
+ * Enumeration to describe which command Tor is running.  These commands
+ * are controlled by command-line options.
+ **/
+typedef enum {
+  CMD_RUN_TOR=0, /**< The default: run Tor as a daemon. */
+  CMD_LIST_FINGERPRINT, /**< Running --list-fingerprint. */
+  CMD_HASH_PASSWORD, /**< Running --hash-password. */
+  CMD_VERIFY_CONFIG, /**< Running --verify-config. */
+  CMD_DUMP_CONFIG, /**< Running --dump-config. */
+  CMD_KEYGEN, /**< Running --keygen */
+  CMD_KEY_EXPIRATION, /**< Running --key-expiration */
+  CMD_IMMEDIATE, /**< Special value: indicates a command that is handled
+                  * immediately during configuration processing. */
+  CMD_RUN_UNITTESTS, /**< Special value: indicates that we have entered
+                      * the Tor code from the unit tests, not from the
+                      * regular Tor binary at all. */
+} tor_cmdline_mode_t;
+
+#endif /* !defined(TOR_CMDLINE_MODE_H) */
diff --git a/src/app/main/.may_include b/src/app/main/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/app/main/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/app/main/app_main.md b/src/app/main/app_main.md
new file mode 100644
index 0000000000..b8c789716c
--- /dev/null
+++ b/src/app/main/app_main.md
@@ -0,0 +1,2 @@
+@dir /app/main
+@brief app/main: Entry point for tor.
diff --git a/src/app/main/include.am b/src/app/main/include.am
new file mode 100644
index 0000000000..ea392a8581
--- /dev/null
+++ b/src/app/main/include.am
@@ -0,0 +1,18 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/app/main/main.c			\
+	src/app/main/shutdown.c			\
+	src/app/main/subsystem_list.c		\
+	src/app/main/subsysmgr.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/app/main/main.h				\
+	src/app/main/ntmain.h				\
+	src/app/main/shutdown.h 			\
+	src/app/main/subsysmgr.h
+
+if BUILD_NT_SERVICES
+LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
+endif
diff --git a/src/app/main/main.c b/src/app/main/main.c
index 8a5d4cfd15..fd166638db 100644
--- a/src/app/main/main.c
+++ b/src/app/main/main.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,80 +13,69 @@
 
 #include "app/config/config.h"
 #include "app/config/statefile.h"
+#include "app/config/quiet_level.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/cpuworker.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
-#include "core/or/channeltls.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitlist.h"
-#include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/dos.h"
-#include "core/or/policies.h"
-#include "core/or/protover.h"
 #include "core/or/relay.h"
-#include "core/or/scheduler.h"
 #include "core/or/status.h"
 #include "feature/api/tor_api.h"
 #include "feature/api/tor_api_internal.h"
 #include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/entrynodes.h"
-#include "feature/client/transports.h"
 #include "feature/control/control.h"
-#include "feature/dirauth/bwauth.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
 #include "feature/dirparse/routerparse.h"
 #include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
-#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/relay/onion_queue.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
-#include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/err/backtrace.h"
-#include "lib/geoip/geoip.h"
+#include "lib/net/resolve.h"
 
 #include "lib/process/waitpid.h"
+#include "lib/pubsub/pubsub_build.h"
 
 #include "lib/meminfo/meminfo.h"
 #include "lib/osinfo/uname.h"
 #include "lib/sandbox/sandbox.h"
 #include "lib/fs/lockfile.h"
-#include "lib/net/resolve.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/encoding/confline.h"
 #include "lib/evloop/timers.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
 
@@ -107,8 +96,6 @@
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
-void evdns_shutdown(int);
-
 #ifdef HAVE_RUST
 // helper function defined in Rust to output a log message indicating if tor is
 // running with Rust enabled. See src/rust/tor_util
@@ -121,16 +108,6 @@ static void dumpmemusage(int severity);
 static void dumpstats(int severity); /* log stats */
 static void process_signal(int sig);
 
-/********* START VARIABLES **********/
-
-/** Decides our behavior when no logs are configured/before any
- * logs have been configured.  For 0, we log notice to stdout as normal.
- * For 1, we log warnings only.  For 2, we log nothing.
- */
-int quiet_level = 0;
-
-/********* END VARIABLES ************/
-
 /** Called when we get a SIGHUP: reload configuration files and keys,
  * retry all connections, and so on. */
 static int
@@ -301,9 +278,35 @@ process_signal(int sig)
       log_heartbeat(time(NULL));
       control_event_signal(sig);
       break;
+    case SIGACTIVE:
+      /* "SIGACTIVE" counts as ersatz user activity. */
+      note_user_activity(approx_time());
+      control_event_signal(sig);
+      break;
+    case SIGDORMANT:
+      /* "SIGDORMANT" means to ignore past user activity */
+      log_notice(LD_GENERAL, "Going dormant because of controller request.");
+      reset_user_activity(0);
+      set_network_participation(false);
+      schedule_rescan_periodic_events();
+      control_event_signal(sig);
+      break;
   }
 }
 
+#ifdef _WIN32
+/** Activate SIGINT on reciving a control signal in console */
+static BOOL WINAPI
+process_win32_console_ctrl(DWORD ctrl_type)
+{
+  /* Ignore type of the ctrl signal */
+  (void) ctrl_type;
+
+  activate_signal(SIGINT);
+  return TRUE;
+}
+#endif
+
 /**
  * Write current memory usage information to the log.
  */
@@ -423,18 +426,7 @@ dumpstats(int severity)
 
   rep_hist_dump_stats(now,severity);
   rend_service_dump_stats(severity);
-}
-
-/** Called by exit() as we shut down the process.
- */
-static void
-exit_function(void)
-{
-  /* NOTE: If we ever daemonize, this gets called immediately.  That's
-   * okay for now, because we only use this on Windows.  */
-#ifdef _WIN32
-  WSACleanup();
-#endif
+  hs_service_dump_stats(severity);
 }
 
 #ifdef _WIN32
@@ -481,6 +473,8 @@ static struct {
   { SIGNEWNYM, 0, NULL },
   { SIGCLEARDNSCACHE, 0, NULL },
   { SIGHEARTBEAT, 0, NULL },
+  { SIGACTIVE, 0, NULL },
+  { SIGDORMANT, 0, NULL },
   { -1, -1, NULL }
 };
 
@@ -515,6 +509,13 @@ handle_signals(void)
                       &signal_handlers[i].signal_value);
     }
   }
+
+#ifdef _WIN32
+    /* Windows lacks traditional POSIX signals but WinAPI provides a function
+     * to handle control signals like Ctrl+C in the console, we can use this to
+     * simulate the SIGINT signal */
+    if (enabled) SetConsoleCtrlHandler(process_win32_console_ctrl, TRUE);
+#endif
 }
 
 /* Cause the signal handler for signal_num to be called in the event loop. */
@@ -537,7 +538,7 @@ int
 tor_init(int argc, char *argv[])
 {
   char progname[256];
-  int quiet = 0;
+  quiet_level_t quiet = QUIET_NONE;
 
   time_of_process_start = time(NULL);
   tor_init_connection_lists();
@@ -545,59 +546,28 @@ tor_init(int argc, char *argv[])
   tor_snprintf(progname, sizeof(progname), "Tor %s", get_version());
   log_set_application_name(progname);
 
-  /* Set up the crypto nice and early */
-  if (crypto_early_init() < 0) {
-    log_err(LD_GENERAL, "Unable to initialize the crypto subsystem!");
-    return -1;
-  }
-
   /* Initialize the history structures. */
   rep_hist_init();
   /* Initialize the service cache. */
   rend_cache_init();
   addressmap_init(); /* Init the client dns cache. Do it always, since it's
                       * cheap. */
+
   /* Initialize the HS subsystem. */
   hs_init();
 
   {
-  /* We search for the "quiet" option first, since it decides whether we
-   * will log anything at all to the command line. */
-    config_line_t *opts = NULL, *cmdline_opts = NULL;
-    const config_line_t *cl;
-    (void) config_parse_commandline(argc, argv, 1, &opts, &cmdline_opts);
-    for (cl = cmdline_opts; cl; cl = cl->next) {
-      if (!strcmp(cl->key, "--hush"))
-        quiet = 1;
-      if (!strcmp(cl->key, "--quiet") ||
-          !strcmp(cl->key, "--dump-config"))
-        quiet = 2;
-      /* The following options imply --hush */
-      if (!strcmp(cl->key, "--version") || !strcmp(cl->key, "--digests") ||
-          !strcmp(cl->key, "--list-torrc-options") ||
-          !strcmp(cl->key, "--library-versions") ||
-          !strcmp(cl->key, "--list-modules") ||
-          !strcmp(cl->key, "--hash-password") ||
-          !strcmp(cl->key, "-h") || !strcmp(cl->key, "--help")) {
-        if (quiet < 1)
-          quiet = 1;
-      }
-    }
-    config_free_lines(opts);
-    config_free_lines(cmdline_opts);
+    /* We check for the "quiet"/"hush" settings first, since they decide
+       whether we log anything at all to stdout. */
+    parsed_cmdline_t *cmdline;
+    cmdline = config_parse_commandline(argc, argv, 1);
+    if (cmdline)
+      quiet = cmdline->quiet_level;
+    parsed_cmdline_free(cmdline);
   }
 
  /* give it somewhere to log to initially */
-  switch (quiet) {
-    case 2:
-      /* no initial logging */
-      break;
-    case 1:
-      add_temp_log(LOG_WARN);
-      break;
-    default:
-      add_temp_log(LOG_NOTICE);
-  }
+  add_default_log_for_quiet_level(quiet);
   quiet_level = quiet;
 
   {
@@ -631,12 +601,6 @@ tor_init(int argc, char *argv[])
   rust_log_welcome_string();
 #endif /* defined(HAVE_RUST) */
 
-  if (network_init()<0) {
-    log_err(LD_BUG,"Error initializing network; exiting.");
-    return -1;
-  }
-  atexit(exit_function);
-
   int init_rv = options_init_from_torrc(argc,argv);
   if (init_rv < 0) {
     log_err(LD_CONFIG,"Reading config failed--see warnings above.");
@@ -647,12 +611,17 @@ tor_init(int argc, char *argv[])
     return 1;
   }
 
-  /* The options are now initialised */
-  const or_options_t *options = get_options();
-
-  /* Initialize channelpadding parameters to defaults until we get
-   * a consensus */
+  /* Initialize channelpadding and circpad parameters to defaults
+   * until we get a consensus */
   channelpadding_new_consensus_params(NULL);
+  circpad_new_consensus_params(NULL);
+
+  /* Initialize circuit padding to defaults+torrc until we get a consensus */
+  circpad_machines_init();
+
+  /* Initialize hidden service DoS subsystem. We need to do this once the
+   * configuration object has been set because it can be accessed. */
+  hs_dos_init();
 
   /* Initialize predicted ports list after loading options */
   predicted_ports_init();
@@ -663,17 +632,6 @@ tor_init(int argc, char *argv[])
              "and you probably shouldn't.");
 #endif
 
-  if (crypto_global_init(options->HardwareAccel,
-                         options->AccelName,
-                         options->AccelDir)) {
-    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
-    return -1;
-  }
-  stream_choice_seed_weak_rng();
-  if (tor_init_libevent_rng() < 0) {
-    log_warn(LD_NET, "Problem initializing libevent RNG.");
-  }
-
   /* Scan/clean unparseable descriptors; after reading config */
   routerparse_init();
 
@@ -742,86 +700,6 @@ release_lockfile(void)
   }
 }
 
-/** Free all memory that we might have allocated somewhere.
- * If <b>postfork</b>, we are a worker process and we want to free
- * only the parts of memory that we won't touch. If !<b>postfork</b>,
- * Tor is shutting down and we should free everything.
- *
- * Helps us find the real leaks with sanitizers and the like. Also valgrind
- * should then report 0 reachable in its leak report (in an ideal world --
- * in practice libevent, SSL, libc etc never quite free everything). */
-void
-tor_free_all(int postfork)
-{
-  if (!postfork) {
-    evdns_shutdown(1);
-  }
-  geoip_free_all();
-  geoip_stats_free_all();
-  dirvote_free_all();
-  routerlist_free_all();
-  networkstatus_free_all();
-  addressmap_free_all();
-  dirserv_free_fingerprint_list();
-  dirserv_free_all();
-  dirserv_clear_measured_bw_cache();
-  rend_cache_free_all();
-  rend_service_authorization_free_all();
-  rep_hist_free_all();
-  dns_free_all();
-  clear_pending_onions();
-  circuit_free_all();
-  entry_guards_free_all();
-  pt_free_all();
-  channel_tls_free_all();
-  channel_free_all();
-  connection_free_all();
-  connection_edge_free_all();
-  scheduler_free_all();
-  nodelist_free_all();
-  microdesc_free_all();
-  routerparse_free_all();
-  ext_orport_free_all();
-  control_free_all();
-  tor_free_getaddrinfo_cache();
-  protover_free_all();
-  bridges_free_all();
-  consdiffmgr_free_all();
-  hs_free_all();
-  dos_free_all();
-  circuitmux_ewma_free_all();
-  accounting_free_all();
-
-  if (!postfork) {
-    config_free_all();
-    or_state_free_all();
-    router_free_all();
-    routerkeys_free_all();
-    policies_free_all();
-  }
-  if (!postfork) {
-    tor_tls_free_all();
-#ifndef _WIN32
-    tor_getpwnam(NULL);
-#endif
-  }
-  /* stuff in main.c */
-
-  tor_mainloop_free_all();
-
-  if (!postfork) {
-    release_lockfile();
-  }
-  tor_libevent_free_all();
-  /* Stuff in util.c and address.c*/
-  if (!postfork) {
-    escaped(NULL);
-    esc_router_info(NULL);
-    clean_up_backtrace_handler();
-    logs_free_all(); /* free log strings. do this last so logs keep working. */
-  }
-}
-
 /**
  * Remove the specified file, and log a warning if the operation fails for
  * any reason other than the file not existing. Ignores NULL filenames.
@@ -835,51 +713,6 @@ tor_remove_file(const char *filename)
   }
 }
 
-/** Do whatever cleanup is necessary before shutting Tor down. */
-void
-tor_cleanup(void)
-{
-  const or_options_t *options = get_options();
-  if (options->command == CMD_RUN_TOR) {
-    time_t now = time(NULL);
-    /* Remove our pid file. We don't care if there was an error when we
-     * unlink, nothing we could do about it anyways. */
-    tor_remove_file(options->PidFile);
-    /* Remove control port file */
-    tor_remove_file(options->ControlPortWriteToFile);
-    /* Remove cookie authentication file */
-    {
-      char *cookie_fname = get_controller_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    /* Remove Extended ORPort cookie authentication file */
-    {
-      char *cookie_fname = get_ext_or_auth_cookie_file_name();
-      tor_remove_file(cookie_fname);
-      tor_free(cookie_fname);
-    }
-    if (accounting_is_enabled(options))
-      accounting_record_bandwidth_usage(now, get_or_state());
-    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
-    or_state_save(now);
-    if (authdir_mode(options)) {
-      sr_save_and_cleanup();
-    }
-    if (authdir_mode_tests_reachability(options))
-      rep_hist_record_mtbf_data(now, 0);
-    keypin_close_journal();
-  }
-
-  timers_shutdown();
-
-  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
-                      later, if it makes shutdown unacceptably slow.  But for
-                      now, leave it here: it's helped us catch bugs in the
-                      past. */
-  crypto_global_cleanup();
-}
-
 /** Read/create keys as needed, and echo our fingerprint to stdout. */
 static int
 do_list_fingerprint(void)
@@ -988,6 +821,9 @@ sandbox_init_filter(void)
 #define OPEN(name)                              \
   sandbox_cfg_allow_open_filename(&cfg, tor_strdup(name))
 
+#define OPENDIR(dir)                            \
+  sandbox_cfg_allow_opendir_dirname(&cfg, tor_strdup(dir))
+
 #define OPEN_DATADIR(name)                      \
   sandbox_cfg_allow_open_filename(&cfg, get_datadir_fname(name))
 
@@ -1004,8 +840,10 @@ sandbox_init_filter(void)
     OPEN_DATADIR2(name, name2 suffix);                  \
   } while (0)
 
+// KeyDirectory is a directory, but it is only opened in check_private_dir
+// which calls open instead of opendir
 #define OPEN_KEY_DIRECTORY() \
-  sandbox_cfg_allow_open_filename(&cfg, tor_strdup(options->KeyDirectory))
+  OPEN(options->KeyDirectory)
 #define OPEN_CACHEDIR(name)                      \
   sandbox_cfg_allow_open_filename(&cfg, get_cachedir_fname(name))
 #define OPEN_CACHEDIR_SUFFIX(name, suffix) do {  \
@@ -1019,6 +857,8 @@ sandbox_init_filter(void)
     OPEN_KEYDIR(name suffix);                    \
   } while (0)
 
+  // DataDirectory is a directory, but it is only opened in check_private_dir
+  // which calls open instead of opendir
   OPEN(options->DataDirectory);
   OPEN_KEY_DIRECTORY();
 
@@ -1066,7 +906,11 @@ sandbox_init_filter(void)
   }
 
   SMARTLIST_FOREACH(options->FilesOpenedByIncludes, char *, f, {
-    OPEN(f);
+    if (file_status(f) == FN_DIR) {
+      OPENDIR(f);
+    } else {
+      OPEN(f);
+    }
   });
 
 #define RENAME_SUFFIX(name, suffix)        \
@@ -1179,7 +1023,7 @@ sandbox_init_filter(void)
      * directory that holds it. */
     char *dirname = tor_strdup(port->unix_addr);
     if (get_parent_directory(dirname) == 0) {
-      OPEN(dirname);
+      OPENDIR(dirname);
     }
     tor_free(dirname);
     sandbox_cfg_allow_chmod_filename(&cfg, tor_strdup(port->unix_addr));
@@ -1273,7 +1117,6 @@ int
 run_tor_main_loop(void)
 {
   handle_signals();
-  monotime_init();
   timers_initialize();
   initialize_mainloop_events();
 
@@ -1378,6 +1221,32 @@ run_tor_main_loop(void)
   return do_main_loop();
 }
 
+/** Install the publish/subscribe relationships for all the subsystems. */
+void
+pubsub_install(void)
+{
+    pubsub_builder_t *builder = pubsub_builder_new();
+    int r = subsystems_add_pubsub(builder);
+    tor_assert(r == 0);
+    r = tor_mainloop_connect_pubsub(builder); // consumes builder
+    tor_assert(r == 0);
+}
+
+/** Connect the mainloop to its publish/subscribe message delivery events if
+ * appropriate, and configure the global channels appropriately. */
+void
+pubsub_connect(void)
+{
+  if (get_options()->command == CMD_RUN_TOR) {
+    tor_mainloop_connect_pubsub_events();
+    /* XXXX For each pubsub channel, its delivery strategy should be set at
+     * this XXXX point, using tor_mainloop_set_delivery_strategy().
+     */
+    tor_mainloop_set_delivery_strategy("orconn", DELIV_IMMEDIATE);
+    tor_mainloop_set_delivery_strategy("ocirc", DELIV_IMMEDIATE);
+  }
+}
+
 /* Main entry point for the Tor process.  Called from tor_main(), and by
  * anybody embedding Tor. */
 int
@@ -1385,54 +1254,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
 {
   int result = 0;
 
-#ifdef _WIN32
-#ifndef HeapEnableTerminationOnCorruption
-#define HeapEnableTerminationOnCorruption 1
-#endif
-  /* On heap corruption, just give up; don't try to play along. */
-  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
-
-  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
-   * (On 64-bit Windows it always fails, and some compilers don't like the
-   * PSETDEP cast.)
-   * 32-bit Windows defines _WIN32.
-   * 64-bit Windows defines _WIN32 and _WIN64. */
-#ifndef _WIN64
-  /* Call SetProcessDEPPolicy to permanently enable DEP.
-     The function will not resolve on earlier versions of Windows,
-     and failure is not dangerous. */
-  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
-  if (hMod) {
-    typedef BOOL (WINAPI *PSETDEP)(DWORD);
-    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
-                           "SetProcessDEPPolicy");
-    if (setdeppolicy) {
-      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
-      setdeppolicy(3);
-    }
-  }
-#endif /* !defined(_WIN64) */
-#endif /* defined(_WIN32) */
-
-  {
-    int bt_err = configure_backtrace_handler(get_version());
-    if (bt_err < 0) {
-      log_warn(LD_BUG, "Unable to install backtrace handler: %s",
-               strerror(-bt_err));
-    }
-  }
-
 #ifdef EVENT_SET_MEM_FUNCTIONS_IMPLEMENTED
   event_set_mem_functions(tor_malloc_, tor_realloc_, tor_free_);
 #endif
 
-  init_protocol_warning_severity_level();
+  subsystems_init();
 
-  update_approx_time(time(NULL));
-  tor_threads_init();
-  tor_compress_init();
-  init_logging(0);
-  monotime_init();
+  init_protocol_warning_severity_level();
 
   int argc = tor_cfg->argc + tor_cfg->argc_owned;
   char **argv = tor_calloc(argc, sizeof(char*));
@@ -1441,15 +1269,13 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     memcpy(argv + tor_cfg->argc, tor_cfg->argv_owned,
            tor_cfg->argc_owned*sizeof(char*));
 
-#ifdef NT_SERVICE
-  {
-     int done = 0;
-     result = nt_service_parse_options(argc, argv, &done);
-     if (done) {
-       goto done;
-     }
-  }
-#endif /* defined(NT_SERVICE) */
+  int done = 0;
+  result = nt_service_parse_options(argc, argv, &done);
+  if (POSSIBLE(done))
+    goto done;
+
+  pubsub_install();
+
   {
     int init_rv = tor_init(argc, argv);
     if (init_rv) {
@@ -1459,6 +1285,8 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     }
   }
 
+  pubsub_connect();
+
   if (get_options()->Sandbox && get_options()->command == CMD_RUN_TOR) {
     sandbox_cfg_t* cfg = sandbox_init_filter();
 
@@ -1468,6 +1296,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
       tor_free_all(0);
       return -1;
     }
+    tor_make_getaddrinfo_cache_active();
 
     // registering libevent rng
 #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE
@@ -1478,9 +1307,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
 
   switch (get_options()->command) {
   case CMD_RUN_TOR:
-#ifdef NT_SERVICE
     nt_service_set_state(SERVICE_RUNNING);
-#endif
     result = run_tor_main_loop();
     break;
   case CMD_KEYGEN:
@@ -1498,7 +1325,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     result = 0;
     break;
   case CMD_VERIFY_CONFIG:
-    if (quiet_level == 0)
+    if (quiet_level == QUIET_NONE)
       printf("Configuration was valid\n");
     result = 0;
     break;
@@ -1506,6 +1333,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
     result = do_dump_config();
     break;
   case CMD_RUN_UNITTESTS: /* only set by test.c */
+  case CMD_IMMEDIATE: /* Handled in config.c */
   default:
     log_warn(LD_BUG,"Illegal command number %d: internal error.",
              get_options()->command);
diff --git a/src/app/main/main.h b/src/app/main/main.h
index bbbbf984fb..e6ed978c61 100644
--- a/src/app/main/main.h
+++ b/src/app/main/main.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,11 +21,11 @@ void release_lockfile(void);
 
 void tor_remove_file(const char *filename);
 
-void tor_cleanup(void);
-void tor_free_all(int postfork);
-
 int tor_init(int argc, char **argv);
 
 int run_tor_main_loop(void);
 
+void pubsub_install(void);
+void pubsub_connect(void);
+
 #endif /* !defined(TOR_MAIN_H) */
diff --git a/src/app/main/ntmain.c b/src/app/main/ntmain.c
index 05d203b0be..5dc0edd591 100644
--- a/src/app/main/ntmain.c
+++ b/src/app/main/ntmain.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,6 +24,7 @@
 #include "app/config/config.h"
 #include "app/main/main.h"
 #include "app/main/ntmain.h"
+#include "app/main/shutdown.h"
 #include "core/mainloop/mainloop.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/fs/winlib.h"
@@ -65,7 +66,7 @@ static int nt_service_cmd_stop(void);
 
 /** Struct to hold dynamically loaded NT-service related function pointers.
  */
-struct service_fns {
+struct {
   int loaded;
 
   /** @{ */
@@ -282,7 +283,9 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_install();
   r = tor_init(backup_argc, backup_argv);
+
   if (r) {
     /* Failed to start the Tor service */
     r = NT_SERVICE_ERROR_TORINIT_FAILED;
@@ -293,6 +296,8 @@ nt_service_body(int argc, char **argv)
     return;
   }
 
+  pubsub_connect();
+
   /* Set the service's status to SERVICE_RUNNING and start the main
    * event loop */
   service_status.dwCurrentState = SERVICE_RUNNING;
@@ -321,9 +326,12 @@ nt_service_main(void)
     errmsg = format_win32_error(result);
     printf("Service error %d : %s\n", (int) result, errmsg);
     tor_free(errmsg);
+
+    pubsub_install();
     if (result == ERROR_FAILED_SERVICE_CONTROLLER_CONNECT) {
       if (tor_init(backup_argc, backup_argv))
         return;
+      pubsub_connect();
       switch (get_options()->command) {
       case CMD_RUN_TOR:
         run_tor_main_loop();
@@ -339,6 +347,7 @@ nt_service_main(void)
                "or --key-expiration) in NT service.");
         break;
       case CMD_RUN_UNITTESTS:
+      case CMD_IMMEDIATE:
       default:
         log_err(LD_CONFIG, "Illegal command number %d: internal error.",
                 get_options()->command);
@@ -594,7 +603,7 @@ nt_service_install(int argc, char **argv)
       /* Genericity is apparently _so_ last year in Redmond, where some
        * accounts are accounts that you can look up, and some accounts
        * are magic and undetectable via the security subsystem. See
-       * http://msdn2.microsoft.com/en-us/library/ms684188.aspx
+       * https://msdn2.microsoft.com/en-us/library/ms684188.aspx
        */
       printf("Running on a Post-Win2K OS, so we'll assume that the "
              "LocalService account exists.\n");
@@ -607,6 +616,7 @@ nt_service_install(int argc, char **argv)
                             &sidUse) == 0) {
     /* XXXX For some reason, the above test segfaults. Fix that. */
     printf("User \"%s\" doesn't seem to exist.\n", user_acct);
+    tor_free(command);
     return -1;
   } else {
     printf("Will try to install service as user \"%s\".\n", user_acct);
diff --git a/src/app/main/ntmain.h b/src/app/main/ntmain.h
index c39386c054..c2d6e23da7 100644
--- a/src/app/main/ntmain.h
+++ b/src/app/main/ntmain.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,7 +22,8 @@ int nt_service_is_stopping(void);
 void nt_service_set_state(DWORD state);
 #else
 #define nt_service_is_stopping() 0
+#define nt_service_parse_options(a, b, c) (0)
+#define nt_service_set_state(s) STMT_NIL
 #endif /* defined(NT_SERVICE) */
 
 #endif /* !defined(TOR_NTMAIN_H) */
-
diff --git a/src/app/main/shutdown.c b/src/app/main/shutdown.c
new file mode 100644
index 0000000000..aac15246b9
--- /dev/null
+++ b/src/app/main/shutdown.c
@@ -0,0 +1,169 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file shutdown.c
+ * @brief Code to free global resources used by Tor.
+ *
+ * In the future, this should all be handled by the subsystem manager. */
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/main/main.h"
+#include "app/main/shutdown.h"
+#include "app/main/subsysmgr.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop_pubsub.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitmux_ewma.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/connection_edge.h"
+#include "core/or/dos.h"
+#include "core/or/scheduler.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/client/transports.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/shared_random.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirparse/routerparse.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_common.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/relay_config.h"
+#include "feature/rend/rendcache.h"
+#include "feature/rend/rendclient.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/rephist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/geoip/geoip.h"
+
+void evdns_shutdown(int);
+
+/** Do whatever cleanup is necessary before shutting Tor down. */
+void
+tor_cleanup(void)
+{
+  const or_options_t *options = get_options();
+  if (options->command == CMD_RUN_TOR) {
+    time_t now = time(NULL);
+    /* Remove our pid file. We don't care if there was an error when we
+     * unlink, nothing we could do about it anyways. */
+    tor_remove_file(options->PidFile);
+    /* Remove control port file */
+    tor_remove_file(options->ControlPortWriteToFile);
+    /* Remove cookie authentication file */
+    {
+      char *cookie_fname = get_controller_cookie_file_name();
+      tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    /* Remove Extended ORPort cookie authentication file */
+    {
+      char *cookie_fname = get_ext_or_auth_cookie_file_name();
+      if (cookie_fname)
+        tor_remove_file(cookie_fname);
+      tor_free(cookie_fname);
+    }
+    if (accounting_is_enabled(options))
+      accounting_record_bandwidth_usage(now, get_or_state());
+    or_state_mark_dirty(get_or_state(), 0); /* force an immediate save. */
+    or_state_save(now);
+    if (authdir_mode(options)) {
+      sr_save_and_cleanup();
+    }
+    if (authdir_mode_tests_reachability(options))
+      rep_hist_record_mtbf_data(now, 0);
+  }
+
+  timers_shutdown();
+
+  tor_free_all(0); /* We could move tor_free_all back into the ifdef below
+                      later, if it makes shutdown unacceptably slow.  But for
+                      now, leave it here: it's helped us catch bugs in the
+                      past. */
+}
+
+/** Free all memory that we might have allocated somewhere.
+ * If <b>postfork</b>, we are a worker process and we want to free
+ * only the parts of memory that we won't touch. If !<b>postfork</b>,
+ * Tor is shutting down and we should free everything.
+ *
+ * Helps us find the real leaks with sanitizers and the like. Also valgrind
+ * should then report 0 reachable in its leak report (in an ideal world --
+ * in practice libevent, SSL, libc etc never quite free everything). */
+void
+tor_free_all(int postfork)
+{
+  if (!postfork) {
+    evdns_shutdown(1);
+  }
+  geoip_free_all();
+  geoip_stats_free_all();
+  routerlist_free_all();
+  networkstatus_free_all();
+  addressmap_free_all();
+  dirserv_free_all();
+  rend_cache_free_all();
+  rend_service_authorization_free_all();
+  rep_hist_free_all();
+  circuit_free_all();
+  circpad_machines_free();
+  entry_guards_free_all();
+  pt_free_all();
+  channel_tls_free_all();
+  channel_free_all();
+  connection_free_all();
+  connection_edge_free_all();
+  scheduler_free_all();
+  nodelist_free_all();
+  microdesc_free_all();
+  routerparse_free_all();
+  control_free_all();
+  bridges_free_all();
+  consdiffmgr_free_all();
+  hs_free_all();
+  dos_free_all();
+  circuitmux_ewma_free_all();
+  accounting_free_all();
+  circpad_free_all();
+
+  if (!postfork) {
+    config_free_all();
+    relay_config_free_all();
+    or_state_free_all();
+  }
+  if (!postfork) {
+#ifndef _WIN32
+    tor_getpwnam(NULL);
+#endif
+  }
+  /* stuff in main.c */
+
+  tor_mainloop_disconnect_pubsub();
+
+  if (!postfork) {
+    release_lockfile();
+  }
+
+  subsystems_shutdown();
+
+  /* Stuff in util.c and address.c*/
+  if (!postfork) {
+    esc_router_info(NULL);
+  }
+}
diff --git a/src/app/main/shutdown.h b/src/app/main/shutdown.h
new file mode 100644
index 0000000000..623ae9525b
--- /dev/null
+++ b/src/app/main/shutdown.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file shutdown.h
+ * \brief Header file for shutdown.c.
+ **/
+
+#ifndef TOR_SHUTDOWN_H
+#define TOR_SHUTDOWN_H
+
+void tor_cleanup(void);
+void tor_free_all(int postfork);
+
+#endif /* !defined(TOR_SHUTDOWN_H) */
diff --git a/src/app/main/subsysmgr.c b/src/app/main/subsysmgr.c
new file mode 100644
index 0000000000..de601d28cd
--- /dev/null
+++ b/src/app/main/subsysmgr.c
@@ -0,0 +1,478 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsysmgr.c
+ * @brief Manager for Tor's subsystems.
+ *
+ * This code is responsible for initializing, configuring, and shutting
+ * down all of Tor's individual subsystems.
+ **/
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+
+#include "lib/confmgt/confmgt.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/err/torerr.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * True iff we have checked tor_subsystems for consistency.
+ **/
+static bool subsystem_array_validated = false;
+
+/** Index value indicating that a subsystem has no options/state object, and
+ * so that object does not have an index. */
+#define IDX_NONE (-1)
+
+/**
+ * Runtime status of a single subsystem.
+ **/
+typedef struct subsys_status_t {
+  /** True if the given subsystem is initialized. */
+  bool initialized;
+  /** Index for this subsystem's options object, or IDX_NONE for none. */
+  int options_idx;
+  /** Index for this subsystem's state object, or IDX_NONE for none. */
+  int state_idx;
+} subsys_status_t;
+
+/** An overestimate of the number of subsystems. */
+#define N_SYS_STATUS 128
+/**
+ * True if a given subsystem is initialized.  Expand this array if there
+ * are more than this number of subsystems.  (We'd rather not
+ * dynamically allocate in this module.)
+ **/
+static subsys_status_t sys_status[N_SYS_STATUS];
+
+/** Set <b>status</b> to a default (not set-up) state. */
+static void
+subsys_status_clear(subsys_status_t *status)
+{
+  if (!status)
+    return;
+  memset(status, 0, sizeof(*status));
+  status->initialized = false;
+  status->state_idx = IDX_NONE;
+  status->options_idx = IDX_NONE;
+}
+
+/**
+ * Exit with a raw assertion if the subsystems list is inconsistent;
+ * initialize the subsystem_initialized array.
+ **/
+static void
+check_and_setup(void)
+{
+  if (subsystem_array_validated)
+    return;
+
+  raw_assert(ARRAY_LENGTH(sys_status) >= n_tor_subsystems);
+  memset(sys_status, 0, sizeof(sys_status));
+
+  int last_level = MIN_SUBSYS_LEVEL;
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->level < MIN_SUBSYS_LEVEL || sys->level > MAX_SUBSYS_LEVEL) {
+      fprintf(stderr, "BUG: Subsystem %s (at %u) has an invalid level %d. "
+              "It is supposed to be between %d and %d (inclusive).\n",
+              sys->name, i, sys->level, MIN_SUBSYS_LEVEL, MAX_SUBSYS_LEVEL);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    if (sys->level < last_level) {
+      fprintf(stderr, "BUG: Subsystem %s (at #%u) is in the wrong position. "
+              "Its level is %d; but the previous subsystem's level was %d.\n",
+              sys->name, i, sys->level, last_level);
+      raw_assert_unreached_msg("There is a bug in subsystem_list.c");
+    }
+    subsys_status_clear(&sys_status[i]);
+
+    last_level = sys->level;
+  }
+
+  subsystem_array_validated = true;
+}
+
+/**
+ * Initialize all the subsystems; exit on failure.
+ **/
+int
+subsystems_init(void)
+{
+  return subsystems_init_upto(MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Initialize all the subsystems whose level is less than or equal to
+ * <b>target_level</b>; exit on failure.
+ **/
+int
+subsystems_init_upto(int target_level)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (sys_status[i].initialized)
+      continue;
+    int r = 0;
+    if (sys->initialize) {
+      // Note that the logging subsystem is designed so that it does no harm
+      // to log a message in an uninitialized state.  These messages will be
+      // discarded for now, however.
+      log_debug(LD_GENERAL, "Initializing %s", sys->name);
+      r = sys->initialize();
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) initialization failed.\n",
+              sys->name, i);
+      raw_assert_unreached_msg("A subsystem couldn't be initialized.");
+    }
+    sys_status[i].initialized = true;
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems of level no more than <b>target_level</b>.
+ **/
+int
+subsystems_add_pubsub_upto(pubsub_builder_t *builder,
+                           int target_level)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level > target_level)
+      break;
+    if (! sys_status[i].initialized)
+      continue;
+    int r = 0;
+    if (sys->add_pubsub) {
+      subsys_id_t sysid = get_subsys_id(sys->name);
+      raw_assert(sysid != ERROR_ID);
+      pubsub_connector_t *connector;
+      connector = pubsub_connector_for_subsystem(builder, sysid);
+      r = sys->add_pubsub(connector);
+      pubsub_connector_free(connector);
+    }
+    if (r < 0) {
+      fprintf(stderr, "BUG: subsystem %s (at %u) could not connect to "
+              "publish/subscribe system.", sys->name, sys->level);
+      raw_assert_unreached_msg("A subsystem couldn't be connected.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Add publish/subscribe relationships to <b>builder</b> for all
+ * initialized subsystems.
+ **/
+int
+subsystems_add_pubsub(pubsub_builder_t *builder)
+{
+  return subsystems_add_pubsub_upto(builder, MAX_SUBSYS_LEVEL);
+}
+
+/**
+ * Shut down all the subsystems.
+ **/
+void
+subsystems_shutdown(void)
+{
+  subsystems_shutdown_downto(MIN_SUBSYS_LEVEL - 1);
+}
+
+/**
+ * Shut down all the subsystems whose level is above <b>target_level</b>.
+ **/
+void
+subsystems_shutdown_downto(int target_level)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (sys->level <= target_level)
+      break;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->shutdown) {
+      log_debug(LD_GENERAL, "Shutting down %s", sys->name);
+      sys->shutdown();
+    }
+    subsys_status_clear(&sys_status[i]);
+  }
+}
+
+/**
+ * Run pre-fork code on all subsystems that declare any
+ **/
+void
+subsystems_prefork(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->prefork) {
+      log_debug(LD_GENERAL, "Pre-fork: %s", sys->name);
+      sys->prefork();
+    }
+  }
+}
+
+/**
+ * Run post-fork code on all subsystems that declare any
+ **/
+void
+subsystems_postfork(void)
+{
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->postfork) {
+      log_debug(LD_GENERAL, "Post-fork: %s", sys->name);
+      sys->postfork();
+    }
+  }
+}
+
+/**
+ * Run thread-cleanup code on all subsystems that declare any
+ **/
+void
+subsystems_thread_cleanup(void)
+{
+  check_and_setup();
+
+  for (int i = (int)n_tor_subsystems - 1; i >= 0; --i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (!sys->supported)
+      continue;
+    if (! sys_status[i].initialized)
+      continue;
+    if (sys->thread_cleanup) {
+      log_debug(LD_GENERAL, "Thread cleanup: %s", sys->name);
+      sys->thread_cleanup();
+    }
+  }
+}
+
+/**
+ * Dump a human- and machine-readable list of all the subsystems to stdout,
+ * in their initialization order, prefixed with their level.
+ **/
+void
+subsystems_dump_list(void)
+{
+  for (unsigned i = 0; i < n_tor_subsystems - 1; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    printf("% 4d\t%16s\t%s\n", sys->level, sys->name,
+           sys->location?sys->location:"");
+  }
+}
+
+/**
+ * Register all subsystem-declared options formats in <b>mgr</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+subsystems_register_options_formats(config_mgr_t *mgr)
+{
+  tor_assert(mgr);
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->options_format) {
+      int options_idx = config_mgr_add_format(mgr, sys->options_format);
+      sys_status[i].options_idx = options_idx;
+      log_debug(LD_CONFIG, "Added options format for %s with index %d",
+                sys->name, options_idx);
+    }
+  }
+  return 0;
+}
+
+/**
+ * Register all subsystem-declared state formats in <b>mgr</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+subsystems_register_state_formats(config_mgr_t *mgr)
+{
+  tor_assert(mgr);
+  check_and_setup();
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys->state_format) {
+      int state_idx = config_mgr_add_format(mgr, sys->state_format);
+      sys_status[i].state_idx = state_idx;
+      log_debug(LD_CONFIG, "Added state format for %s with index %d",
+                sys->name, state_idx);
+    }
+  }
+  return 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Helper: look up the index for <b>sys</b>.  Return -1 if the subsystem
+ * is not recognized.
+ **/
+static int
+subsys_get_idx(const subsys_fns_t *sys)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    if (sys == tor_subsystems[i])
+      return (int)i;
+  }
+  return -1;
+}
+
+/**
+ * Return the current state-manager's index for any state held by the
+ * subsystem <b>sys</b>.  If <b>sys</b> has no options, return -1.
+ *
+ * Using raw indices can be error-prone: only do this from the unit
+ * tests. If you need a way to access another subsystem's configuration,
+ * that subsystem should provide access functions.
+ **/
+int
+subsystems_get_options_idx(const subsys_fns_t *sys)
+{
+  int i = subsys_get_idx(sys);
+  tor_assert(i >= 0);
+  return sys_status[i].options_idx;
+}
+
+/**
+ * Return the current state-manager's index for any state held by the
+ * subsystem <b>sys</b>.  If <b>sys</b> has no state, return -1.
+ *
+ * Using raw indices can be error-prone: only do this from the unit
+ * tests.  If you need a way to access another subsystem's state
+ * that subsystem should provide access functions.
+ **/
+int
+subsystems_get_state_idx(const subsys_fns_t *sys)
+{
+  int i = subsys_get_idx(sys);
+  tor_assert(i >= 0);
+  return sys_status[i].state_idx;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Call all appropriate set_options() methods to tell the various subsystems
+ * about a new set of torrc options.  Return 0 on success, -1 on
+ * nonrecoverable failure.
+ **/
+int
+subsystems_set_options(const config_mgr_t *mgr, struct or_options_t *options)
+{
+  /* XXXX This does not yet handle reversible option assignment; I'll
+   * do that later in this branch. */
+
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys_status[i].options_idx >= 0 && sys->set_options) {
+      void *obj = config_mgr_get_obj_mutable(mgr, options,
+                                             sys_status[i].options_idx);
+      int rv = sys->set_options(obj);
+      if (rv < 0) {
+        log_err(LD_CONFIG, "Error when handling option for %s; "
+                "cannot proceed.", sys->name);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/**
+ * Call all appropriate set_state() methods to tell the various subsystems
+ * about an initial DataDir/state file.  Return 0 on success, -1 on
+ * nonrecoverable failure.
+ **/
+int
+subsystems_set_state(const config_mgr_t *mgr, struct or_state_t *state)
+{
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys_status[i].state_idx >= 0 && sys->set_state) {
+      void *obj = config_mgr_get_obj_mutable(mgr, state,
+                                             sys_status[i].state_idx);
+      int rv = sys->set_state(obj);
+      if (rv < 0) {
+        log_err(LD_CONFIG, "Error when handling state for %s; "
+                "cannot proceed.", sys->name);
+        return -1;
+      }
+    }
+  }
+  return 0;
+}
+
+/**
+ * Call all appropriate flush_state() methods to tell the various subsystems
+ * to update the state objects in <b>state</b>.  Return 0 on success,
+ * -1 on failure.
+ **/
+int
+subsystems_flush_state(const config_mgr_t *mgr, struct or_state_t *state)
+{
+  int result = 0;
+  for (unsigned i = 0; i < n_tor_subsystems; ++i) {
+    const subsys_fns_t *sys = tor_subsystems[i];
+    if (sys_status[i].state_idx >= 0 && sys->flush_state) {
+      void *obj = config_mgr_get_obj_mutable(mgr, state,
+                                             sys_status[i].state_idx);
+      int rv = sys->flush_state(obj);
+      if (rv < 0) {
+        log_warn(LD_CONFIG, "Error when flushing state to state object for %s",
+                sys->name);
+        result = -1;
+      }
+    }
+  }
+  return result;
+}
diff --git a/src/app/main/subsysmgr.h b/src/app/main/subsysmgr.h
new file mode 100644
index 0000000000..ae0b3df469
--- /dev/null
+++ b/src/app/main/subsysmgr.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsysmgr.h
+ * @brief Header for subsysmgr.c
+ **/
+
+#ifndef TOR_SUBSYSMGR_T
+#define TOR_SUBSYSMGR_T
+
+#include "lib/subsys/subsys.h"
+
+extern const struct subsys_fns_t *tor_subsystems[];
+extern const unsigned n_tor_subsystems;
+
+int subsystems_init(void);
+int subsystems_init_upto(int level);
+
+struct pubsub_builder_t;
+int subsystems_add_pubsub_upto(struct pubsub_builder_t *builder,
+                               int target_level);
+int subsystems_add_pubsub(struct pubsub_builder_t *builder);
+
+void subsystems_shutdown(void);
+void subsystems_shutdown_downto(int level);
+
+void subsystems_prefork(void);
+void subsystems_postfork(void);
+void subsystems_thread_cleanup(void);
+
+void subsystems_dump_list(void);
+
+struct config_mgr_t;
+int subsystems_register_options_formats(struct config_mgr_t *mgr);
+int subsystems_register_state_formats(struct config_mgr_t *mgr);
+struct or_options_t;
+struct or_state_t;
+int subsystems_set_options(const struct config_mgr_t *mgr,
+                           struct or_options_t *options);
+int subsystems_set_state(const struct config_mgr_t *mgr,
+                         struct or_state_t *state);
+int subsystems_flush_state(const struct config_mgr_t *mgr,
+                           struct or_state_t *state);
+
+#ifdef TOR_UNIT_TESTS
+int subsystems_get_options_idx(const subsys_fns_t *sys);
+int subsystems_get_state_idx(const subsys_fns_t *sys);
+#endif
+
+#endif /* !defined(TOR_SUBSYSMGR_T) */
diff --git a/src/app/main/subsystem_list.c b/src/app/main/subsystem_list.c
new file mode 100644
index 0000000000..e32083537f
--- /dev/null
+++ b/src/app/main/subsystem_list.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsystem_list.c
+ * @brief List of Tor's subsystems.
+ **/
+
+#include "orconfig.h"
+#include "app/main/subsysmgr.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+
+#include "core/mainloop/mainloop_sys.h"
+#include "core/or/or_sys.h"
+#include "feature/control/btrack_sys.h"
+#include "lib/compress/compress_sys.h"
+#include "lib/crypt_ops/crypto_sys.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/log/log_sys.h"
+#include "lib/net/network_sys.h"
+#include "lib/process/process_sys.h"
+#include "lib/llharden/winprocess_sys.h"
+#include "lib/thread/thread_sys.h"
+#include "lib/time/time_sys.h"
+#include "lib/tls/tortls_sys.h"
+#include "lib/wallclock/wallclock_sys.h"
+#include "lib/evloop/evloop_sys.h"
+
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/relay/relay_sys.h"
+
+#include <stddef.h>
+
+/**
+ * Global list of the subsystems in Tor, in the order of their initialization.
+ * Want to know the exact level numbers?
+ * We'll implement a level dump command in #31614.
+ **/
+const subsys_fns_t *tor_subsystems[] = {
+  &sys_winprocess,
+  &sys_torerr,
+
+  &sys_wallclock,
+  &sys_logging,
+  &sys_threads,
+
+  &sys_time,
+
+  &sys_crypto,
+  &sys_compress,
+  &sys_network,
+  &sys_tortls,
+
+  &sys_evloop,
+  &sys_process,
+
+  &sys_mainloop,
+  &sys_or,
+
+  &sys_relay,
+
+  &sys_btrack,
+
+  &sys_dirauth,
+};
+
+const unsigned n_tor_subsystems = ARRAY_LENGTH(tor_subsystems);
diff --git a/src/app/main/tor_main.c b/src/app/main/tor_main.c
index 8a887ed269..0ee03fd5e9 100644
--- a/src/app/main/tor_main.c
+++ b/src/app/main/tor_main.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/arch_goals.md b/src/arch_goals.md
new file mode 100644
index 0000000000..92c86d9df8
--- /dev/null
+++ b/src/arch_goals.md
@@ -0,0 +1,31 @@
+@page arch_goals High level code design practices
+
+This page describes the high level design practices for Tor's code.
+This design is a long-term goal of what we want our code to look like,
+rather than a description of how it currently is.
+
+Overall, we want various parts of tor's code to interact with each
+other through a small number of interfaces.
+
+We want to avoid having "god objects" or "god modules".  These are
+objects or modules that know far too much about other parts of the
+code.  God objects/modules are generally recognized to be an
+antipattern of software design.
+
+Historically, there have been modules in tor that have tended toward
+becoming god modules.  These include modules that help more
+specialized code communicate with the outside world: the configuration
+and control modules, for example.  Others are modules that deal with
+global state, initialization, or shutdown.
+
+If a centralized module needs to invoke code in almost every other
+module in the system, it is better if it exports a small, general
+interface that other modules call.  The centralized module should not
+explicitly call out to all the modules that interact with it.
+
+Instead, modules that interact with the centralized module should call
+registration interfaces.  These interfaces allow modules to register
+handlers for things like configuration parsing and control command
+execution.  (The config and control modules are examples of this.)
+Alternatively, registration can happen through statically initialized
+data structures.  (The subsystem mechanism is an example of this.)
diff --git a/src/config/mmdb-convert.py b/src/config/mmdb-convert.py
index 3a454a3fc1..4acfea6c0e 100644
--- a/src/config/mmdb-convert.py
+++ b/src/config/mmdb-convert.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/python
 
 #   This software has been dedicated to the public domain under the CC0
 #   public domain dedication.
@@ -9,7 +9,7 @@
 #
 #   You should have received a copy of the CC0 legalcode along with this
 #   work in doc/cc0.txt.  If not, see
-#      <http://creativecommons.org/publicdomain/zero/1.0/>.
+#      <https://creativecommons.org/publicdomain/zero/1.0/>.
 
 #  Nick Mathewson is responsible for this kludge, but takes no
 #  responsibility for it.
@@ -28,6 +28,11 @@
    pieces.
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import struct
 import bisect
 import socket
@@ -77,7 +82,7 @@ def to_int32(s):
 
 def to_int28(s):
     "Parse a pair of big-endian 28-bit integers from bytestring s."
-    a, b = unpack("!LL", s + b'\x00')
+    a, b = struct.unpack("!LL", s + b'\x00')
     return (((a & 0xf0) << 20) + (a >> 8)), ((a & 0x0f) << 24) + (b >> 8)
 
 class Tree(object):
diff --git a/src/config/torrc.minimal.in-staging b/src/config/torrc.minimal.in-staging
index cb3adca35c..90bad7f7cc 100644
--- a/src/config/torrc.minimal.in-staging
+++ b/src/config/torrc.minimal.in-staging
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
diff --git a/src/config/torrc.sample.in b/src/config/torrc.sample.in
index c2ae707e93..51e1c3af4b 100644
--- a/src/config/torrc.sample.in
+++ b/src/config/torrc.sample.in
@@ -88,6 +88,9 @@
 ## yourself to make this work.
 #ORPort 443 NoListen
 #ORPort 127.0.0.1:9090 NoAdvertise
+## If you want to listen on IPv6 your numeric address must be explictly
+## between square brackets as follows. You must also listen on IPv4.
+#ORPort [2001:DB8::1]:9050
 
 ## The IP address or full DNS name for incoming connections to your
 ## relay. Leave commented out and Tor will guess.
@@ -174,13 +177,11 @@
 
 ## Uncomment this if you want your relay to be an exit, with the default
 ## exit policy (or whatever exit policy you set below).
-## (If ReducedExitPolicy or ExitPolicy are set, relays are exits.
-## If neither exit policy option is set, relays are non-exits.)
+## (If ReducedExitPolicy, ExitPolicy, or IPv6Exit are set, relays are exits.
+## If none of these options are set, relays are non-exits.)
 #ExitRelay 1
 
 ## Uncomment this if you want your relay to allow IPv6 exit traffic.
-## You must also set ExitRelay, ReducedExitPolicy, or ExitPolicy to make your
-## relay into an exit.
 ## (Relays do not allow any exit traffic by default.)
 #IPv6Exit 1
 
diff --git a/src/core/core.md b/src/core/core.md
new file mode 100644
index 0000000000..8ecc43eaae
--- /dev/null
+++ b/src/core/core.md
@@ -0,0 +1,18 @@
+@dir /core
+@brief core: main loop and onion routing functionality
+
+The "core" directory has the central protocols for Tor, which every
+client and relay must implement in order to perform onion routing.
+
+It is divided into three lower-level pieces:
+
+   - \refdir{core/crypto} -- Tor-specific cryptography.
+
+   - \refdir{core/proto} -- Protocol encoding/decoding.
+
+   - \refdir{core/mainloop} -- A connection-oriented asynchronous mainloop.
+
+and one high-level piece:
+
+   - \refdir{core/or} -- Implements onion routing itself.
+
diff --git a/src/core/crypto/.may_include b/src/core/crypto/.may_include
new file mode 100644
index 0000000000..5782a36797
--- /dev/null
+++ b/src/core/crypto/.may_include
@@ -0,0 +1,10 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/cc/*.h
+lib/log/*.h
+
+core/crypto/*.h
diff --git a/src/core/crypto/core_crypto.md b/src/core/crypto/core_crypto.md
new file mode 100644
index 0000000000..26ade1f8f8
--- /dev/null
+++ b/src/core/crypto/core_crypto.md
@@ -0,0 +1,6 @@
+@dir /core/crypto
+@brief core/crypto: Tor-specific cryptography
+
+This module implements Tor's circuit-construction crypto and Tor's
+relay crypto.
+
diff --git a/src/core/crypto/hs_ntor.c b/src/core/crypto/hs_ntor.c
index c34073690e..07bcdc566c 100644
--- a/src/core/crypto/hs_ntor.c
+++ b/src/core/crypto/hs_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /** \file hs_ntor.c
@@ -170,19 +170,18 @@ get_rendezvous1_key_material(const uint8_t *rend_secret_hs_input,
  * necessary key material, and return 0. */
 static void
 get_introduce1_key_material(const uint8_t *secret_input,
-                        const uint8_t *subcredential,
+                        const hs_subcredential_t *subcredential,
                         hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
 {
   uint8_t keystream[CIPHER256_KEY_LEN + DIGEST256_LEN];
   uint8_t info_blob[INFO_BLOB_LEN];
   uint8_t kdf_input[KDF_INPUT_LEN];
-  crypto_xof_t *xof;
   uint8_t *ptr;
 
   /* Let's build info */
   ptr = info_blob;
   APPEND(ptr, M_HSEXPAND, strlen(M_HSEXPAND));
-  APPEND(ptr, subcredential, DIGEST256_LEN);
+  APPEND(ptr, subcredential->subcred, SUBCRED_LEN);
   tor_assert(ptr == info_blob + sizeof(info_blob));
 
   /* Let's build the input to the KDF */
@@ -193,10 +192,8 @@ get_introduce1_key_material(const uint8_t *secret_input,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Now we need to run kdf_input over SHAKE-256 */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream)) ;
-  crypto_xof_free(xof);
+  crypto_xof(keystream, sizeof(keystream),
+             kdf_input, sizeof(kdf_input));
 
   { /* Get the keys */
     memcpy(&hs_ntor_intro_cell_keys_out->enc_key, keystream,CIPHER256_KEY_LEN);
@@ -320,7 +317,7 @@ hs_ntor_client_get_introduce1_keys(
                       const ed25519_public_key_t *intro_auth_pubkey,
                       const curve25519_public_key_t *intro_enc_pubkey,
                       const curve25519_keypair_t *client_ephemeral_enc_keypair,
-                      const uint8_t *subcredential,
+                      const hs_subcredential_t *subcredential,
                       hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
 {
   int bad = 0;
@@ -453,9 +450,31 @@ hs_ntor_service_get_introduce1_keys(
                     const ed25519_public_key_t *intro_auth_pubkey,
                     const curve25519_keypair_t *intro_enc_keypair,
                     const curve25519_public_key_t *client_ephemeral_enc_pubkey,
-                    const uint8_t *subcredential,
+                    const hs_subcredential_t *subcredential,
                     hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
 {
+  return hs_ntor_service_get_introduce1_keys_multi(
+                             intro_auth_pubkey,
+                             intro_enc_keypair,
+                             client_ephemeral_enc_pubkey,
+                             1,
+                             subcredential,
+                             hs_ntor_intro_cell_keys_out);
+}
+
+/**
+ * As hs_ntor_service_get_introduce1_keys(), but take multiple subcredentials
+ * as input, and yield multiple sets of keys as output.
+ **/
+int
+hs_ntor_service_get_introduce1_keys_multi(
+            const struct ed25519_public_key_t *intro_auth_pubkey,
+            const struct curve25519_keypair_t *intro_enc_keypair,
+            const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
+            size_t n_subcredentials,
+            const hs_subcredential_t *subcredentials,
+            hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out)
+{
   int bad = 0;
   uint8_t secret_input[INTRO_SECRET_HS_INPUT_LEN];
   uint8_t dh_result[CURVE25519_OUTPUT_LEN];
@@ -463,7 +482,8 @@ hs_ntor_service_get_introduce1_keys(
   tor_assert(intro_auth_pubkey);
   tor_assert(intro_enc_keypair);
   tor_assert(client_ephemeral_enc_pubkey);
-  tor_assert(subcredential);
+  tor_assert(n_subcredentials >= 1);
+  tor_assert(subcredentials);
   tor_assert(hs_ntor_intro_cell_keys_out);
 
   /* Compute EXP(X, b) */
@@ -479,13 +499,16 @@ hs_ntor_service_get_introduce1_keys(
                             secret_input);
   bad |= safe_mem_is_zero(secret_input, CURVE25519_OUTPUT_LEN);
 
-  /* Get ENC_KEY and MAC_KEY! */
-  get_introduce1_key_material(secret_input, subcredential,
-                              hs_ntor_intro_cell_keys_out);
+  for (unsigned i = 0; i < n_subcredentials; ++i) {
+    /* Get ENC_KEY and MAC_KEY! */
+    get_introduce1_key_material(secret_input, &subcredentials[i],
+                                &hs_ntor_intro_cell_keys_out[i]);
+  }
 
   memwipe(secret_input,  0, sizeof(secret_input));
   if (bad) {
-    memwipe(hs_ntor_intro_cell_keys_out, 0, sizeof(hs_ntor_intro_cell_keys_t));
+    memwipe(hs_ntor_intro_cell_keys_out, 0,
+            sizeof(hs_ntor_intro_cell_keys_t) * n_subcredentials);
   }
 
   return bad ? -1 : 0;
@@ -594,7 +617,6 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
 {
   uint8_t *ptr;
   uint8_t kdf_input[NTOR_KEY_EXPANSION_KDF_INPUT_LEN];
-  crypto_xof_t *xof;
 
   /* Sanity checks on lengths to make sure we are good */
   if (BUG(seed_len != DIGEST256_LEN)) {
@@ -611,10 +633,8 @@ hs_ntor_circuit_key_expansion(const uint8_t *ntor_key_seed, size_t seed_len,
   tor_assert(ptr == kdf_input + sizeof(kdf_input));
 
   /* Generate the keys */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, kdf_input, sizeof(kdf_input));
-  crypto_xof_squeeze_bytes(xof, keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN);
-  crypto_xof_free(xof);
+  crypto_xof(keys_out, HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN,
+             kdf_input, sizeof(kdf_input));
 
   return 0;
 }
diff --git a/src/core/crypto/hs_ntor.h b/src/core/crypto/hs_ntor.h
index e5a5171915..9a975dd83f 100644
--- a/src/core/crypto/hs_ntor.h
+++ b/src/core/crypto/hs_ntor.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file hs_ntor.h
+ * @brief Header for hs_ntor.c
+ **/
+
 #ifndef TOR_HS_NTOR_H
 #define TOR_HS_NTOR_H
 
@@ -14,7 +19,7 @@ struct curve25519_keypair_t;
   (DIGEST256_LEN*2 + CIPHER256_KEY_LEN*2)
 
 /* Key material needed to encode/decode INTRODUCE1 cells */
-typedef struct {
+typedef struct hs_ntor_intro_cell_keys_t {
   /* Key used for encryption of encrypted INTRODUCE1 blob */
   uint8_t enc_key[CIPHER256_KEY_LEN];
   /* MAC key used to protect encrypted INTRODUCE1 blob */
@@ -22,7 +27,7 @@ typedef struct {
 } hs_ntor_intro_cell_keys_t;
 
 /* Key material needed to encode/decode RENDEZVOUS1 cells */
-typedef struct {
+typedef struct hs_ntor_rend_cell_keys_t {
   /* This is the MAC of the HANDSHAKE_INFO field */
   uint8_t rend_cell_auth_mac[DIGEST256_LEN];
   /* This is the key seed used to derive further rendezvous crypto keys as
@@ -30,11 +35,20 @@ typedef struct {
   uint8_t ntor_key_seed[DIGEST256_LEN];
 } hs_ntor_rend_cell_keys_t;
 
+#define SUBCRED_LEN DIGEST256_LEN
+
+/**
+ * A 'subcredential' used to prove knowledge of a hidden service.
+ **/
+typedef struct hs_subcredential_t {
+  uint8_t subcred[SUBCRED_LEN];
+} hs_subcredential_t;
+
 int hs_ntor_client_get_introduce1_keys(
               const struct ed25519_public_key_t *intro_auth_pubkey,
               const struct curve25519_public_key_t *intro_enc_pubkey,
               const struct curve25519_keypair_t *client_ephemeral_enc_keypair,
-              const uint8_t *subcredential,
+              const hs_subcredential_t *subcredential,
               hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
 
 int hs_ntor_client_get_rendezvous1_keys(
@@ -44,11 +58,19 @@ int hs_ntor_client_get_rendezvous1_keys(
           const struct curve25519_public_key_t *service_ephemeral_rend_pubkey,
           hs_ntor_rend_cell_keys_t *hs_ntor_rend_cell_keys_out);
 
+int hs_ntor_service_get_introduce1_keys_multi(
+            const struct ed25519_public_key_t *intro_auth_pubkey,
+            const struct curve25519_keypair_t *intro_enc_keypair,
+            const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
+            size_t n_subcredentials,
+            const hs_subcredential_t *subcredentials,
+            hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
+
 int hs_ntor_service_get_introduce1_keys(
             const struct ed25519_public_key_t *intro_auth_pubkey,
             const struct curve25519_keypair_t *intro_enc_keypair,
             const struct curve25519_public_key_t *client_ephemeral_enc_pubkey,
-            const uint8_t *subcredential,
+            const hs_subcredential_t *subcredential,
             hs_ntor_intro_cell_keys_t *hs_ntor_intro_cell_keys_out);
 
 int hs_ntor_service_get_rendezvous1_keys(
diff --git a/src/core/crypto/include.am b/src/core/crypto/include.am
new file mode 100644
index 0000000000..28b7e22905
--- /dev/null
+++ b/src/core/crypto/include.am
@@ -0,0 +1,18 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/crypto/hs_ntor.c		\
+	src/core/crypto/onion_crypto.c		\
+	src/core/crypto/onion_fast.c		\
+	src/core/crypto/onion_ntor.c		\
+	src/core/crypto/onion_tap.c		\
+	src/core/crypto/relay_crypto.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/crypto/hs_ntor.h			\
+	src/core/crypto/onion_crypto.h	        	\
+	src/core/crypto/onion_fast.h			\
+	src/core/crypto/onion_ntor.h			\
+	src/core/crypto/onion_tap.h			\
+	src/core/crypto/relay_crypto.h
diff --git a/src/core/crypto/onion_crypto.c b/src/core/crypto/onion_crypto.c
index 56b02e2996..69b4dc40aa 100644
--- a/src/core/crypto/onion_crypto.c
+++ b/src/core/crypto/onion_crypto.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_crypto.h b/src/core/crypto/onion_crypto.h
index 1cddde3610..2665d326a3 100644
--- a/src/core/crypto/onion_crypto.h
+++ b/src/core/crypto/onion_crypto.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,4 +44,4 @@ void server_onion_keys_free_(server_onion_keys_t *keys);
 #define server_onion_keys_free(keys) \
   FREE_AND_NULL(server_onion_keys_t, server_onion_keys_free_, (keys))
 
-#endif
+#endif /* !defined(TOR_ONION_CRYPTO_H) */
diff --git a/src/core/crypto/onion_fast.c b/src/core/crypto/onion_fast.c
index 31bd20235f..d760549325 100644
--- a/src/core/crypto/onion_fast.c
+++ b/src/core/crypto/onion_fast.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_fast.h b/src/core/crypto/onion_fast.h
index 0ba8cbbc35..da983a56d9 100644
--- a/src/core/crypto/onion_fast.h
+++ b/src/core/crypto/onion_fast.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_ntor.c b/src/core/crypto/onion_ntor.c
index 7087fe1bd7..5a77230d02 100644
--- a/src/core/crypto/onion_ntor.c
+++ b/src/core/crypto/onion_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_ntor.h b/src/core/crypto/onion_ntor.h
index 51e72b4083..9473409e40 100644
--- a/src/core/crypto/onion_ntor.h
+++ b/src/core/crypto/onion_ntor.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file onion_ntor.h
+ * @brief Header for onion_ntor.c
+ **/
+
 #ifndef TOR_ONION_NTOR_H
 #define TOR_ONION_NTOR_H
 
diff --git a/src/core/crypto/onion_tap.c b/src/core/crypto/onion_tap.c
index 854889d88d..119f55f206 100644
--- a/src/core/crypto/onion_tap.c
+++ b/src/core/crypto/onion_tap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/onion_tap.h b/src/core/crypto/onion_tap.h
index 0e43b9c8ba..78174b1fab 100644
--- a/src/core/crypto/onion_tap.h
+++ b/src/core/crypto/onion_tap.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/crypto/relay_crypto.c b/src/core/crypto/relay_crypto.c
index 0b83b2d0a5..3e6167e0e1 100644
--- a/src/core/crypto/relay_crypto.c
+++ b/src/core/crypto/relay_crypto.c
@@ -1,17 +1,24 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file relay_crypto.h
+ * @brief Header for relay_crypto.c
+ **/
+
 #include "core/or/or.h"
 #include "core/or/circuitlist.h"
+#include "core/or/crypt_path.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "core/crypto/hs_ntor.h" // for HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
+#include "core/or/sendme.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
@@ -20,7 +27,7 @@
 /** Update digest from the payload of cell. Assign integrity part to
  * cell.
  */
-static void
+void
 relay_set_digest(crypto_digest_t *digest, cell_t *cell)
 {
   char integrity[4];
@@ -84,12 +91,39 @@ relay_digest_matches(crypto_digest_t *digest, cell_t *cell)
  *
  * Note that we use the same operation for encrypting and for decrypting.
  */
-static void
+void
 relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in)
 {
   crypto_cipher_crypt_inplace(cipher, (char*) in, CELL_PAYLOAD_SIZE);
 }
 
+/** Return the sendme_digest within the <b>crypto</b> object. */
+uint8_t *
+relay_crypto_get_sendme_digest(relay_crypto_t *crypto)
+{
+  tor_assert(crypto);
+  return crypto->sendme_digest;
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                  bool is_foward_digest)
+{
+  struct crypto_digest_t *digest;
+
+  tor_assert(crypto);
+
+  digest = crypto->b_digest;
+  if (is_foward_digest) {
+    digest = crypto->f_digest;
+  }
+
+  crypto_digest_get_digest(digest, (char *) crypto->sendme_digest,
+                           sizeof(crypto->sendme_digest));
+}
+
 /** Do the appropriate en/decryptions for <b>cell</b> arriving on
  * <b>circ</b> in direction <b>cell_direction</b>.
  *
@@ -134,12 +168,12 @@ relay_decrypt_cell(circuit_t *circ, cell_t *cell,
         tor_assert(thishop);
 
         /* decrypt one layer */
-        relay_crypt_one_payload(thishop->crypto.b_crypto, cell->payload);
+        cpath_crypt_cell(thishop, cell->payload, true);
 
         relay_header_unpack(&rh, cell->payload);
         if (rh.recognized == 0) {
           /* it's possibly recognized. have to check digest to be sure. */
-          if (relay_digest_matches(thishop->crypto.b_digest, cell)) {
+          if (relay_digest_matches(cpath_get_incoming_digest(thishop), cell)) {
             *recognized = 1;
             *layer_hint = thishop;
             return 0;
@@ -187,14 +221,17 @@ relay_encrypt_cell_outbound(cell_t *cell,
                             crypt_path_t *layer_hint)
 {
   crypt_path_t *thishop; /* counter for repeated crypts */
-  relay_set_digest(layer_hint->crypto.f_digest, cell);
+  cpath_set_cell_forward_digest(layer_hint, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(circ), layer_hint);
 
   thishop = layer_hint;
   /* moving from farthest to nearest hop */
   do {
     tor_assert(thishop);
     log_debug(LD_OR,"encrypting a layer of the relay cell.");
-    relay_crypt_one_payload(thishop->crypto.f_crypto, cell->payload);
+    cpath_crypt_cell(thishop, cell->payload, false);
 
     thishop = thishop->prev;
   } while (thishop != circ->cpath->prev);
@@ -212,6 +249,10 @@ relay_encrypt_cell_inbound(cell_t *cell,
                            or_circuit_t *or_circ)
 {
   relay_set_digest(or_circ->crypto.b_digest, cell);
+
+  /* Record cell digest as the SENDME digest if need be. */
+  sendme_record_sending_cell_digest(TO_CIRCUIT(or_circ), NULL);
+
   /* encrypt one layer */
   relay_crypt_one_payload(or_circ->crypto.b_crypto, cell->payload);
 }
diff --git a/src/core/crypto/relay_crypto.h b/src/core/crypto/relay_crypto.h
index 45a21d14ab..5e36c7678c 100644
--- a/src/core/crypto/relay_crypto.h
+++ b/src/core/crypto/relay_crypto.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,5 +27,16 @@ void relay_crypto_clear(relay_crypto_t *crypto);
 
 void relay_crypto_assert_ok(const relay_crypto_t *crypto);
 
+uint8_t *relay_crypto_get_sendme_digest(relay_crypto_t *crypto);
+
+void relay_crypto_record_sendme_digest(relay_crypto_t *crypto,
+                                       bool is_foward_digest);
+
+void
+relay_crypt_one_payload(crypto_cipher_t *cipher, uint8_t *in);
+
+void
+relay_set_digest(crypto_digest_t *digest, cell_t *cell);
+
 #endif /* !defined(TOR_RELAY_CRYPTO_H) */
 
diff --git a/src/core/include.am b/src/core/include.am
index 1b8ef2ac58..7752a7974b 100644
--- a/src/core/include.am
+++ b/src/core/include.am
@@ -1,174 +1,21 @@
 
-noinst_LIBRARIES += \
-	src/core/libtor-app.a
-if UNITTESTS_ENABLED
-noinst_LIBRARIES += \
-	src/core/libtor-app-testing.a
-endif
-
-LIBTOR_APP_A_SOURCES = 				\
-	src/app/config/config.c			\
-	src/app/config/confparse.c		\
-	src/app/config/statefile.c		\
-	src/app/main/main.c			\
-	src/core/crypto/hs_ntor.c		\
-	src/core/crypto/onion_crypto.c		\
-	src/core/crypto/onion_fast.c		\
-	src/core/crypto/onion_ntor.c		\
-	src/core/crypto/onion_tap.c		\
-	src/core/crypto/relay_crypto.c		\
-	src/core/mainloop/connection.c		\
-	src/core/mainloop/cpuworker.c		\
-	src/core/mainloop/mainloop.c		\
-	src/core/mainloop/netstatus.c		\
-	src/core/mainloop/periodic.c		\
-	src/core/or/address_set.c		\
-	src/core/or/channel.c			\
-	src/core/or/channelpadding.c		\
-	src/core/or/channeltls.c		\
-	src/core/or/circuitbuild.c		\
-	src/core/or/circuitlist.c		\
-	src/core/or/circuitmux.c		\
-	src/core/or/circuitmux_ewma.c		\
-	src/core/or/circuitstats.c		\
-	src/core/or/circuituse.c		\
-	src/core/or/command.c			\
-	src/core/or/connection_edge.c		\
-	src/core/or/connection_or.c		\
-	src/core/or/dos.c			\
-	src/core/or/onion.c			\
-	src/core/or/policies.c			\
-	src/core/or/protover.c			\
-	src/core/or/protover_rust.c		\
-	src/core/or/reasons.c			\
-	src/core/or/relay.c			\
-	src/core/or/scheduler.c			\
-	src/core/or/scheduler_kist.c		\
-	src/core/or/scheduler_vanilla.c		\
-	src/core/or/status.c			\
-	src/core/or/versions.c			\
-	src/core/proto/proto_cell.c		\
-	src/core/proto/proto_control0.c		\
-	src/core/proto/proto_ext_or.c		\
-	src/core/proto/proto_http.c		\
-	src/core/proto/proto_socks.c		\
-	src/feature/api/tor_api.c		\
-	src/feature/client/addressmap.c		\
-	src/feature/client/bridges.c		\
-	src/feature/client/circpathbias.c	\
-	src/feature/client/dnsserv.c		\
-	src/feature/client/entrynodes.c		\
-	src/feature/client/transports.c		\
-	src/feature/control/control.c		\
-	src/feature/control/fmt_serverstatus.c  \
-	src/feature/control/getinfo_geoip.c	\
-	src/feature/dirauth/keypin.c		\
-	src/feature/dircache/conscache.c	\
-	src/feature/dircache/consdiffmgr.c	\
-	src/feature/dircache/dircache.c		\
-	src/feature/dircache/dirserv.c		\
-	src/feature/dirclient/dirclient.c	\
-	src/feature/dirclient/dlstatus.c	\
-	src/feature/dircommon/consdiff.c	\
-	src/feature/dircommon/directory.c	\
-	src/feature/dircommon/fp_pair.c		\
-	src/feature/dircommon/voting_schedule.c	\
-	src/feature/dirparse/authcert_parse.c	\
-	src/feature/dirparse/microdesc_parse.c	\
-	src/feature/dirparse/ns_parse.c		\
-	src/feature/dirparse/parsecommon.c	\
-	src/feature/dirparse/policy_parse.c	\
-	src/feature/dirparse/routerparse.c	\
-	src/feature/dirparse/sigcommon.c	\
-	src/feature/dirparse/signing.c		\
-	src/feature/dirparse/unparseable.c	\
-	src/feature/hibernate/hibernate.c	\
-	src/feature/hs/hs_cache.c		\
-	src/feature/hs/hs_cell.c		\
-	src/feature/hs/hs_circuit.c		\
-	src/feature/hs/hs_circuitmap.c		\
-	src/feature/hs/hs_client.c		\
-	src/feature/hs/hs_common.c		\
-	src/feature/hs/hs_config.c		\
-	src/feature/hs/hs_control.c		\
-	src/feature/hs/hs_descriptor.c		\
-	src/feature/hs/hs_ident.c		\
-	src/feature/hs/hs_intropoint.c		\
-	src/feature/hs/hs_service.c		\
-	src/feature/hs/hs_stats.c		\
-	src/feature/hs_common/replaycache.c	\
-	src/feature/hs_common/shared_random_client.c	\
-	src/feature/keymgt/loadkey.c		\
-	src/feature/dirauth/keypin.c		\
-	src/feature/nodelist/authcert.c		\
-	src/feature/nodelist/describe.c		\
-	src/feature/nodelist/dirlist.c		\
-	src/feature/nodelist/microdesc.c	\
-	src/feature/nodelist/networkstatus.c	\
-	src/feature/nodelist/nickname.c		\
-	src/feature/nodelist/nodelist.c		\
-	src/feature/nodelist/node_select.c	\
-	src/feature/nodelist/routerinfo.c	\
-	src/feature/nodelist/routerlist.c	\
-	src/feature/nodelist/routerset.c	\
-	src/feature/nodelist/fmt_routerstatus.c	\
-	src/feature/nodelist/torcert.c		\
-	src/feature/relay/dns.c			\
-	src/feature/relay/ext_orport.c		\
-	src/feature/relay/onion_queue.c		\
-	src/feature/relay/router.c		\
-	src/feature/relay/routerkeys.c		\
-	src/feature/relay/routermode.c		\
-	src/feature/relay/selftest.c		\
-	src/feature/rend/rendcache.c		\
-	src/feature/rend/rendclient.c		\
-	src/feature/rend/rendcommon.c		\
-	src/feature/rend/rendmid.c		\
-	src/feature/rend/rendparse.c		\
-	src/feature/rend/rendservice.c		\
-	src/feature/stats/geoip_stats.c		\
-	src/feature/stats/rephist.c		\
-	src/feature/stats/predict_ports.c
-
-# These should eventually move into module_dirauth_sources, but for now
-# the separation is only in the code location.
-LIBTOR_APP_A_SOURCES += 			\
-	src/feature/dirauth/bwauth.c		\
-	src/feature/dirauth/dsigs_parse.c	\
-	src/feature/dirauth/guardfraction.c	\
-	src/feature/dirauth/reachability.c	\
-	src/feature/dirauth/recommend_pkg.c	\
-	src/feature/dirauth/process_descs.c	\
-	src/feature/dirauth/voteflags.c
-
-if BUILD_NT_SERVICES
-LIBTOR_APP_A_SOURCES += src/app/main/ntmain.c
-endif
-
 #
-# Modules are conditionnally compiled in tor starting here. We add the C files
+# Modules are conditionally compiled in tor starting here. We add the C files
 # only if the modules has been enabled at configure time. We always add the
 # source files of every module to libtor-testing.a so we can build the unit
 # tests for everything. See the UNITTESTS_ENABLED branch below.
 #
 LIBTOR_APP_TESTING_A_SOURCES = $(LIBTOR_APP_A_SOURCES)
 
-# The Directory Authority module.
-MODULE_DIRAUTH_SOURCES = 					\
-	src/feature/dirauth/authmode.c				\
-	src/feature/dirauth/dircollate.c			\
-	src/feature/dirauth/dirvote.c				\
-	src/feature/dirauth/shared_random.c			\
-	src/feature/dirauth/shared_random_state.c
-
-if BUILD_MODULE_DIRAUTH
-LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
-endif
+src_core_libtor_app_a_SOURCES = \
+	$(LIBTOR_APP_A_SOURCES) \
+	$(LIBTOR_APP_A_STUB_SOURCES)
 
-src_core_libtor_app_a_SOURCES = $(LIBTOR_APP_A_SOURCES)
 if UNITTESTS_ENABLED
 
 # Add the sources of the modules that are needed for tests to work here.
+LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_RELAY_SOURCES)
+LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRCACHE_SOURCES)
 LIBTOR_APP_TESTING_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
 
 src_core_libtor_app_testing_a_SOURCES = $(LIBTOR_APP_TESTING_A_SOURCES)
@@ -183,204 +30,6 @@ AM_CPPFLAGS += -DSHARE_DATADIR="\"$(datadir)\""		\
 src_core_libtor_app_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_core_libtor_app_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
-noinst_HEADERS +=					\
-	src/app/config/config.h				\
-	src/app/config/confparse.h			\
-	src/app/config/or_options_st.h			\
-	src/app/config/or_state_st.h			\
-	src/app/config/statefile.h			\
-	src/app/main/main.h				\
-	src/app/main/ntmain.h				\
-	src/core/crypto/hs_ntor.h			\
-	src/core/crypto/onion_crypto.h	        	\
-	src/core/crypto/onion_fast.h			\
-	src/core/crypto/onion_ntor.h			\
-	src/core/crypto/onion_tap.h			\
-	src/core/crypto/relay_crypto.h			\
-	src/core/mainloop/connection.h			\
-	src/core/mainloop/cpuworker.h			\
-	src/core/mainloop/mainloop.h			\
-	src/core/mainloop/netstatus.h			\
-	src/core/mainloop/periodic.h			\
-	src/core/or/addr_policy_st.h			\
-	src/core/or/address_set.h			\
-	src/core/or/cell_queue_st.h			\
-	src/core/or/cell_st.h				\
-	src/core/or/channel.h				\
-	src/core/or/channelpadding.h			\
-	src/core/or/channeltls.h			\
-	src/core/or/circuit_st.h			\
-	src/core/or/circuitbuild.h			\
-	src/core/or/circuitlist.h			\
-	src/core/or/circuitmux.h			\
-	src/core/or/circuitmux_ewma.h			\
-	src/core/or/circuitstats.h			\
-	src/core/or/circuituse.h			\
-	src/core/or/command.h				\
-	src/core/or/connection_edge.h			\
-	src/core/or/connection_or.h			\
-	src/core/or/connection_st.h			\
-	src/core/or/cpath_build_state_st.h		\
-	src/core/or/crypt_path_reference_st.h		\
-	src/core/or/crypt_path_st.h			\
-	src/core/or/destroy_cell_queue_st.h		\
-	src/core/or/dos.h				\
-	src/core/or/edge_connection_st.h		\
-	src/core/or/half_edge_st.h		\
-	src/core/or/entry_connection_st.h		\
-	src/core/or/entry_port_cfg_st.h			\
-	src/core/or/extend_info_st.h			\
-	src/core/or/listener_connection_st.h		\
-	src/core/or/onion.h				\
-	src/core/or/or.h				\
-	src/core/or/or_circuit_st.h			\
-	src/core/or/or_connection_st.h			\
-	src/core/or/or_handshake_certs_st.h		\
-	src/core/or/or_handshake_state_st.h		\
-	src/core/or/origin_circuit_st.h			\
-	src/core/or/policies.h				\
-	src/core/or/port_cfg_st.h			\
-	src/core/or/protover.h				\
-	src/core/or/reasons.h				\
-	src/core/or/relay.h				\
-	src/core/or/relay_crypto_st.h			\
-	src/core/or/scheduler.h				\
-	src/core/or/server_port_cfg_st.h		\
-	src/core/or/socks_request_st.h			\
-	src/core/or/status.h				\
-	src/core/or/tor_version_st.h			\
-	src/core/or/var_cell_st.h			\
-	src/core/or/versions.h				\
-	src/core/proto/proto_cell.h			\
-	src/core/proto/proto_control0.h			\
-	src/core/proto/proto_ext_or.h			\
-	src/core/proto/proto_http.h			\
-	src/core/proto/proto_socks.h			\
-	src/feature/api/tor_api_internal.h		\
-	src/feature/client/addressmap.h			\
-	src/feature/client/bridges.h			\
-	src/feature/client/circpathbias.h		\
-	src/feature/client/dnsserv.h			\
-	src/feature/client/entrynodes.h			\
-	src/feature/client/transports.h			\
-	src/feature/control/control.h			\
-	src/feature/control/control_connection_st.h	\
-	src/feature/control/fmt_serverstatus.h		\
-	src/feature/control/getinfo_geoip.h		\
-	src/feature/dirauth/authmode.h			\
-	src/feature/dirauth/bwauth.h			\
-	src/feature/dirauth/dircollate.h		\
-	src/feature/dirauth/dirvote.h			\
-	src/feature/dirauth/dsigs_parse.h		\
-	src/feature/dirauth/guardfraction.h		\
-	src/feature/dirauth/keypin.h			\
-	src/feature/dirauth/ns_detached_signatures_st.h	\
-	src/feature/dirauth/reachability.h		\
-	src/feature/dirauth/recommend_pkg.h		\
-	src/feature/dirauth/process_descs.h		\
-	src/feature/dirauth/shared_random.h		\
-	src/feature/dirauth/shared_random_state.h	\
-	src/feature/dirauth/vote_microdesc_hash_st.h	\
-	src/feature/dirauth/voteflags.h			\
-	src/feature/dircache/cached_dir_st.h		\
-	src/feature/dircache/conscache.h		\
-	src/feature/dircache/consdiffmgr.h		\
-	src/feature/dircache/dircache.h			\
-	src/feature/dircache/dirserv.h			\
-	src/feature/dirclient/dir_server_st.h		\
-	src/feature/dirclient/dirclient.h		\
-	src/feature/dirclient/dlstatus.h		\
-	src/feature/dirclient/download_status_st.h	\
-	src/feature/dircommon/consdiff.h		\
-	src/feature/dircommon/dir_connection_st.h	\
-	src/feature/dircommon/directory.h		\
-	src/feature/dircommon/fp_pair.h			\
-	src/feature/dircommon/vote_timing_st.h		\
-	src/feature/dircommon/voting_schedule.h		\
-	src/feature/dirparse/authcert_members.i		\
-	src/feature/dirparse/authcert_parse.h		\
-	src/feature/dirparse/microdesc_parse.h		\
-	src/feature/dirparse/ns_parse.h			\
-	src/feature/dirparse/parsecommon.h		\
-	src/feature/dirparse/policy_parse.h		\
-	src/feature/dirparse/routerparse.h		\
-	src/feature/dirparse/sigcommon.h		\
-	src/feature/dirparse/signing.h			\
-	src/feature/dirparse/unparseable.h		\
-	src/feature/hibernate/hibernate.h		\
-	src/feature/hs/hs_cache.h			\
-	src/feature/hs/hs_cell.h			\
-	src/feature/hs/hs_circuit.h			\
-	src/feature/hs/hs_circuitmap.h			\
-	src/feature/hs/hs_client.h			\
-	src/feature/hs/hs_common.h			\
-	src/feature/hs/hs_config.h			\
-	src/feature/hs/hs_control.h			\
-	src/feature/hs/hs_descriptor.h			\
-	src/feature/hs/hs_ident.h			\
-	src/feature/hs/hs_intropoint.h			\
-	src/feature/hs/hs_service.h			\
-	src/feature/hs/hs_stats.h			\
-	src/feature/hs/hsdir_index_st.h			\
-	src/feature/hs_common/replaycache.h		\
-	src/feature/hs_common/shared_random_client.h	\
-	src/feature/keymgt/loadkey.h			\
-	src/feature/nodelist/authcert.h			\
-	src/feature/nodelist/authority_cert_st.h	\
-	src/feature/nodelist/describe.h			\
-	src/feature/nodelist/desc_store_st.h		\
-	src/feature/nodelist/dirlist.h			\
-	src/feature/nodelist/document_signature_st.h	\
-	src/feature/nodelist/extrainfo_st.h		\
-	src/feature/nodelist/microdesc.h		\
-	src/feature/nodelist/microdesc_st.h		\
-	src/feature/nodelist/networkstatus.h		\
-	src/feature/nodelist/networkstatus_sr_info_st.h	\
-	src/feature/nodelist/networkstatus_st.h		\
-	src/feature/nodelist/networkstatus_voter_info_st.h	\
-	src/feature/nodelist/nickname.h			\
-	src/feature/nodelist/node_st.h			\
-	src/feature/nodelist/nodelist.h			\
-	src/feature/nodelist/node_select.h		\
-	src/feature/nodelist/routerinfo.h		\
-	src/feature/nodelist/routerinfo_st.h		\
-	src/feature/nodelist/routerlist.h		\
-	src/feature/nodelist/routerlist_st.h		\
-	src/feature/nodelist/routerset.h		\
-	src/feature/nodelist/fmt_routerstatus.h		\
-	src/feature/nodelist/routerstatus_st.h		\
-	src/feature/nodelist/signed_descriptor_st.h	\
-	src/feature/nodelist/torcert.h			\
-	src/feature/nodelist/vote_routerstatus_st.h	\
-	src/feature/relay/dns.h				\
-	src/feature/relay/dns_structs.h			\
-	src/feature/relay/ext_orport.h			\
-	src/feature/relay/onion_queue.h			\
-	src/feature/relay/router.h			\
-	src/feature/relay/routerkeys.h			\
-	src/feature/relay/routermode.h			\
-	src/feature/relay/selftest.h			\
-	src/feature/rend/rend_authorized_client_st.h	\
-	src/feature/rend/rend_encoded_v2_service_descriptor_st.h	\
-	src/feature/rend/rend_intro_point_st.h		\
-	src/feature/rend/rend_service_descriptor_st.h	\
-	src/feature/rend/rendcache.h			\
-	src/feature/rend/rendclient.h			\
-	src/feature/rend/rendcommon.h			\
-	src/feature/rend/rendmid.h			\
-	src/feature/rend/rendparse.h			\
-	src/feature/rend/rendservice.h			\
-	src/feature/stats/geoip_stats.h			\
-	src/feature/stats/rephist.h			\
-	src/feature/stats/predict_ports.h
-
-noinst_HEADERS +=			\
-	src/app/config/auth_dirs.inc	\
-	src/app/config/fallback_dirs.inc
-
-# This may someday want to be an installed file?
-noinst_HEADERS += src/feature/api/tor_api.h
-
 micro-revision.i: FORCE
 	$(AM_V_at)rm -f micro-revision.tmp; \
 	if test -r "$(top_srcdir)/.git" && \
diff --git a/src/core/mainloop/.may_include b/src/core/mainloop/.may_include
new file mode 100644
index 0000000000..8e01cf910e
--- /dev/null
+++ b/src/core/mainloop/.may_include
@@ -0,0 +1,24 @@
+!advisory
+
+orconfig.h
+
+lib/conf/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/evloop/*.h
+lib/pubsub/*.h
+lib/subsys/*.h
+lib/buf/*.h
+lib/crypt_ops/*.h
+lib/err/*.h
+lib/tls/*.h
+lib/net/*.h
+lib/evloop/*.h
+lib/geoip/*.h
+lib/sandbox/*.h
+lib/smartlist_core/*.h
+lib/compress/*.h
+lib/log/*.h
+
+core/mainloop/*.h
+core/mainloop/*.inc
\ No newline at end of file
diff --git a/src/core/mainloop/connection.c b/src/core/mainloop/connection.c
index 21d4332758..36b2c6ef63 100644
--- a/src/core/mainloop/connection.c
+++ b/src/core/mainloop/connection.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -57,7 +57,7 @@
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/err/backtrace.h"
 
@@ -65,9 +65,9 @@
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
-#define TOR_CHANNEL_INTERNAL_
-#define CONNECTION_PRIVATE
+#define CHANNEL_OBJECT_PRIVATE
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/netstatus.h"
@@ -82,13 +82,17 @@
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
+#include "core/proto/proto_haproxy.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_config.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
@@ -105,6 +109,7 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/geoip/geoip.h"
 
+#include "lib/cc/ctassert.h"
 #include "lib/sandbox/sandbox.h"
 #include "lib/net/buffers_net.h"
 #include "lib/tls/tortls.h"
@@ -641,7 +646,7 @@ connection_free_minimal(connection_t *conn)
     }
   }
 
-  tor_free(conn->address);
+  tor_str_wipe_and_free(conn->address);
 
   if (connection_speaks_cells(conn)) {
     or_connection_t *or_conn = TO_OR_CONN(conn);
@@ -661,7 +666,7 @@ connection_free_minimal(connection_t *conn)
     }
     or_handshake_state_free(or_conn->handshake_state);
     or_conn->handshake_state = NULL;
-    tor_free(or_conn->nickname);
+    tor_str_wipe_and_free(or_conn->nickname);
     if (or_conn->chan) {
       /* Owww, this shouldn't happen, but... */
       channel_t *base_chan = TLS_CHAN_TO_BASE(or_conn->chan);
@@ -681,8 +686,8 @@ connection_free_minimal(connection_t *conn)
   }
   if (conn->type == CONN_TYPE_AP) {
     entry_connection_t *entry_conn = TO_ENTRY_CONN(conn);
-    tor_free(entry_conn->chosen_exit_name);
-    tor_free(entry_conn->original_dest_address);
+    tor_str_wipe_and_free(entry_conn->chosen_exit_name);
+    tor_str_wipe_and_free(entry_conn->original_dest_address);
     if (entry_conn->socks_request)
       socks_request_free(entry_conn->socks_request);
     if (entry_conn->pending_optimistic_data) {
@@ -700,6 +705,7 @@ connection_free_minimal(connection_t *conn)
     control_connection_t *control_conn = TO_CONTROL_CONN(conn);
     tor_free(control_conn->safecookie_client_hash);
     tor_free(control_conn->incoming_cmd);
+    tor_free(control_conn->current_cmd);
     if (control_conn->ephemeral_onion_services) {
       SMARTLIST_FOREACH(control_conn->ephemeral_onion_services, char *, cp, {
         memwipe(cp, 0, strlen(cp));
@@ -719,11 +725,7 @@ connection_free_minimal(connection_t *conn)
     tor_free(dir_conn->requested_resource);
 
     tor_compress_free(dir_conn->compress_state);
-    if (dir_conn->spool) {
-      SMARTLIST_FOREACH(dir_conn->spool, spooled_resource_t *, spooled,
-                        spooled_resource_free(spooled));
-      smartlist_free(dir_conn->spool);
-    }
+    dir_conn_clear_spool(dir_conn);
 
     rend_data_free(dir_conn->rend_data);
     hs_ident_dir_conn_free(dir_conn->hs_ident);
@@ -1199,7 +1201,7 @@ make_win32_socket_exclusive(tor_socket_t sock)
     return -1;
   }
   return 0;
-#else /* !(defined(SO_EXCLUSIVEADDRUSE)) */
+#else /* !defined(SO_EXCLUSIVEADDRUSE) */
   (void) sock;
   return 0;
 #endif /* defined(SO_EXCLUSIVEADDRUSE) */
@@ -1470,6 +1472,20 @@ connection_listener_new(const struct sockaddr *listensockaddr,
                tor_socket_strerror(tor_socket_errno(s)));
       goto err;
     }
+
+#ifndef __APPLE__
+    /* This code was introduced to help debug #28229. */
+    int value;
+    socklen_t len = sizeof(value);
+
+    if (!getsockopt(s, SOL_SOCKET, SO_ACCEPTCONN, &value, &len)) {
+      if (value == 0) {
+        log_err(LD_NET, "Could not listen on %s - "
+                        "getsockopt(.,SO_ACCEPTCONN,.) yields 0.", address);
+        goto err;
+      }
+    }
+#endif /* !defined(__APPLE__) */
 #endif /* defined(HAVE_SYS_UN_H) */
   } else {
     log_err(LD_BUG, "Got unexpected address family %d.",
@@ -1503,10 +1519,11 @@ connection_listener_new(const struct sockaddr *listensockaddr,
     }
   }
 
+  /* Force IPv4 and IPv6 traffic on for non-SOCKSPorts.
+   * Forcing options on isn't a good idea, see #32994 and #33607. */
   if (type != CONN_TYPE_AP_LISTENER) {
     lis_conn->entry_cfg.ipv4_traffic = 1;
     lis_conn->entry_cfg.ipv6_traffic = 1;
-    lis_conn->entry_cfg.prefer_ipv6 = 0;
   }
 
   if (connection_add(conn) < 0) { /* no space, forget it */
@@ -1648,7 +1665,7 @@ check_sockaddr(const struct sockaddr *sa, int len, int level)
              len,(int)sizeof(struct sockaddr_in6));
       ok = 0;
     }
-    if (tor_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
+    if (fast_mem_is_zero((void*)sin6->sin6_addr.s6_addr, 16) ||
         sin6->sin6_port == 0) {
       log_fn(level, LD_NET,
              "Address for new connection has address/port equal to zero.");
@@ -1871,7 +1888,7 @@ connection_init_accepted_conn(connection_t *conn,
       /* Initiate Extended ORPort authentication. */
       return connection_ext_or_start_auth(TO_OR_CONN(conn));
     case CONN_TYPE_OR:
-      control_event_or_conn_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
+      connection_or_event_status(TO_OR_CONN(conn), OR_CONN_EVENT_NEW, 0);
       rv = connection_tls_start_handshake(TO_OR_CONN(conn), 1);
       if (rv < 0) {
         connection_or_close_for_error(TO_OR_CONN(conn), 0);
@@ -1884,11 +1901,16 @@ connection_init_accepted_conn(connection_t *conn,
       TO_ENTRY_CONN(conn)->nym_epoch = get_signewnym_epoch();
       TO_ENTRY_CONN(conn)->socks_request->listener_type = listener->base_.type;
 
+      /* Any incoming connection on an entry port counts as user activity. */
+      note_user_activity(approx_time());
+
       switch (TO_CONN(listener)->type) {
         case CONN_TYPE_AP_LISTENER:
           conn->state = AP_CONN_STATE_SOCKS_WAIT;
           TO_ENTRY_CONN(conn)->socks_request->socks_prefer_no_auth =
             listener->entry_cfg.socks_prefer_no_auth;
+          TO_ENTRY_CONN(conn)->socks_request->socks_use_extended_errors =
+            listener->entry_cfg.extended_socks5_codes;
           break;
         case CONN_TYPE_AP_TRANS_LISTENER:
           TO_ENTRY_CONN(conn)->is_transparent_ap = 1;
@@ -2083,6 +2105,11 @@ connection_connect_log_client_use_ip_version(const connection_t *conn)
     return;
   }
 
+  if (fascist_firewall_use_ipv6(options)) {
+    log_info(LD_NET, "Our outgoing connection is using IPv%d.",
+             tor_addr_family(&real_addr) == AF_INET6 ? 6 : 4);
+  }
+
   /* Check if we couldn't satisfy an address family preference */
   if ((!pref_ipv6 && tor_addr_family(&real_addr) == AF_INET6)
       || (pref_ipv6 && tor_addr_family(&real_addr) == AF_INET)) {
@@ -2261,9 +2288,12 @@ connection_proxy_state_to_string(int state)
     "PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929",
     "PROXY_SOCKS5_WANT_AUTH_RFC1929_OK",
     "PROXY_SOCKS5_WANT_CONNECT_OK",
+    "PROXY_HAPROXY_WAIT_FOR_FLUSH",
     "PROXY_CONNECTED",
   };
 
+  CTASSERT(ARRAY_LENGTH(states) == PROXY_CONNECTED+1);
+
   if (state < PROXY_NONE || state > PROXY_CONNECTED)
     return unknown;
 
@@ -2296,7 +2326,11 @@ conn_get_proxy_type(const connection_t *conn)
     return PROXY_SOCKS4;
   else if (options->Socks5Proxy)
     return PROXY_SOCKS5;
-  else
+  else if (options->TCPProxy) {
+    /* The only supported protocol in TCPProxy is haproxy. */
+    tor_assert(options->TCPProxyProtocol == TCP_PROXY_PROTOCOL_HAPROXY);
+    return PROXY_HAPROXY;
+  } else
     return PROXY_NONE;
 }
 
@@ -2305,165 +2339,245 @@ conn_get_proxy_type(const connection_t *conn)
    username NUL: */
 #define SOCKS4_STANDARD_BUFFER_SIZE (1 + 1 + 2 + 4 + 1)
 
-/** Write a proxy request of <b>type</b> (socks4, socks5, https) to conn
- * for conn->addr:conn->port, authenticating with the auth details given
- * in the configuration (if available). SOCKS 5 and HTTP CONNECT proxies
- * support authentication.
+/** Write a proxy request of https to conn for conn->addr:conn->port,
+ * authenticating with the auth details given in the configuration
+ * (if available).
  *
  * Returns -1 if conn->addr is incompatible with the proxy protocol, and
  * 0 otherwise.
- *
- * Use connection_read_proxy_handshake() to complete the handshake.
  */
-int
-connection_proxy_connect(connection_t *conn, int type)
+static int
+connection_https_proxy_connect(connection_t *conn)
 {
-  const or_options_t *options;
+  tor_assert(conn);
+
+  const or_options_t *options = get_options();
+  char buf[1024];
+  char *base64_authenticator = NULL;
+  const char *authenticator = options->HTTPSProxyAuthenticator;
+
+  /* Send HTTP CONNECT and authentication (if available) in
+   * one request */
+
+  if (authenticator) {
+    base64_authenticator = alloc_http_authenticator(authenticator);
+    if (!base64_authenticator)
+      log_warn(LD_OR, "Encoding https authenticator failed");
+  }
+
+  if (base64_authenticator) {
+    const char *addrport = fmt_addrport(&conn->addr, conn->port);
+    tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.1\r\n"
+        "Host: %s\r\n"
+        "Proxy-Authorization: Basic %s\r\n\r\n",
+        addrport,
+        addrport,
+        base64_authenticator);
+    tor_free(base64_authenticator);
+  } else {
+    tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.0\r\n\r\n",
+        fmt_addrport(&conn->addr, conn->port));
+  }
+
+  connection_buf_add(buf, strlen(buf), conn);
+  conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
+
+  return 0;
+}
 
+/** Write a proxy request of socks4 to conn for conn->addr:conn->port.
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ */
+static int
+connection_socks4_proxy_connect(connection_t *conn)
+{
   tor_assert(conn);
 
-  options = get_options();
+  unsigned char *buf;
+  uint16_t portn;
+  uint32_t ip4addr;
+  size_t buf_size = 0;
+  char *socks_args_string = NULL;
 
-  switch (type) {
-    case PROXY_CONNECT: {
-      char buf[1024];
-      char *base64_authenticator=NULL;
-      const char *authenticator = options->HTTPSProxyAuthenticator;
-
-      /* Send HTTP CONNECT and authentication (if available) in
-       * one request */
-
-      if (authenticator) {
-        base64_authenticator = alloc_http_authenticator(authenticator);
-        if (!base64_authenticator)
-          log_warn(LD_OR, "Encoding https authenticator failed");
-      }
+  /* Send a SOCKS4 connect request */
 
-      if (base64_authenticator) {
-        const char *addrport = fmt_addrport(&conn->addr, conn->port);
-        tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.1\r\n"
-                     "Host: %s\r\n"
-                     "Proxy-Authorization: Basic %s\r\n\r\n",
-                     addrport,
-                     addrport,
-                     base64_authenticator);
-        tor_free(base64_authenticator);
-      } else {
-        tor_snprintf(buf, sizeof(buf), "CONNECT %s HTTP/1.0\r\n\r\n",
-                     fmt_addrport(&conn->addr, conn->port));
-      }
+  if (tor_addr_family(&conn->addr) != AF_INET) {
+    log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6");
+    return -1;
+  }
 
-      connection_buf_add(buf, strlen(buf), conn);
-      conn->proxy_state = PROXY_HTTPS_WANT_CONNECT_OK;
-      break;
+  { /* If we are here because we are trying to connect to a
+       pluggable transport proxy, check if we have any SOCKS
+       arguments to transmit. If we do, compress all arguments to
+       a single string in 'socks_args_string': */
+
+    if (conn_get_proxy_type(conn) == PROXY_PLUGGABLE) {
+      socks_args_string =
+        pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port);
+      if (socks_args_string)
+        log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.",
+            socks_args_string);
     }
+  }
 
-    case PROXY_SOCKS4: {
-      unsigned char *buf;
-      uint16_t portn;
-      uint32_t ip4addr;
-      size_t buf_size = 0;
-      char *socks_args_string = NULL;
+  { /* Figure out the buffer size we need for the SOCKS message: */
 
-      /* Send a SOCKS4 connect request */
+    buf_size = SOCKS4_STANDARD_BUFFER_SIZE;
 
-      if (tor_addr_family(&conn->addr) != AF_INET) {
-        log_warn(LD_NET, "SOCKS4 client is incompatible with IPv6");
-        return -1;
-      }
+    /* If we have a SOCKS argument string, consider its size when
+       calculating the buffer size: */
+    if (socks_args_string)
+      buf_size += strlen(socks_args_string);
+  }
 
-      { /* If we are here because we are trying to connect to a
-           pluggable transport proxy, check if we have any SOCKS
-           arguments to transmit. If we do, compress all arguments to
-           a single string in 'socks_args_string': */
+  buf = tor_malloc_zero(buf_size);
 
-        if (conn_get_proxy_type(conn) == PROXY_PLUGGABLE) {
-          socks_args_string =
-            pt_get_socks_args_for_proxy_addrport(&conn->addr, conn->port);
-          if (socks_args_string)
-            log_debug(LD_NET, "Sending out '%s' as our SOCKS argument string.",
-                      socks_args_string);
-        }
-      }
+  ip4addr = tor_addr_to_ipv4n(&conn->addr);
+  portn = htons(conn->port);
 
-      { /* Figure out the buffer size we need for the SOCKS message: */
+  buf[0] = 4; /* version */
+  buf[1] = SOCKS_COMMAND_CONNECT; /* command */
+  memcpy(buf + 2, &portn, 2); /* port */
+  memcpy(buf + 4, &ip4addr, 4); /* addr */
+
+  /* Next packet field is the userid. If we have pluggable
+     transport SOCKS arguments, we have to embed them
+     there. Otherwise, we use an empty userid.  */
+  if (socks_args_string) { /* place the SOCKS args string: */
+    tor_assert(strlen(socks_args_string) > 0);
+    tor_assert(buf_size >=
+        SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string));
+    strlcpy((char *)buf + 8, socks_args_string, buf_size - 8);
+    tor_free(socks_args_string);
+  } else {
+    buf[8] = 0; /* no userid */
+  }
 
-        buf_size = SOCKS4_STANDARD_BUFFER_SIZE;
+  connection_buf_add((char *)buf, buf_size, conn);
+  tor_free(buf);
 
-        /* If we have a SOCKS argument string, consider its size when
-           calculating the buffer size: */
-        if (socks_args_string)
-          buf_size += strlen(socks_args_string);
-      }
+  conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
+  return 0;
+}
 
-      buf = tor_malloc_zero(buf_size);
-
-      ip4addr = tor_addr_to_ipv4n(&conn->addr);
-      portn = htons(conn->port);
-
-      buf[0] = 4; /* version */
-      buf[1] = SOCKS_COMMAND_CONNECT; /* command */
-      memcpy(buf + 2, &portn, 2); /* port */
-      memcpy(buf + 4, &ip4addr, 4); /* addr */
-
-      /* Next packet field is the userid. If we have pluggable
-         transport SOCKS arguments, we have to embed them
-         there. Otherwise, we use an empty userid.  */
-      if (socks_args_string) { /* place the SOCKS args string: */
-        tor_assert(strlen(socks_args_string) > 0);
-        tor_assert(buf_size >=
-                   SOCKS4_STANDARD_BUFFER_SIZE + strlen(socks_args_string));
-        strlcpy((char *)buf + 8, socks_args_string, buf_size - 8);
-        tor_free(socks_args_string);
-      } else {
-        buf[8] = 0; /* no userid */
-      }
+/** Write a proxy request of socks5 to conn for conn->addr:conn->port,
+ * authenticating with the auth details given in the configuration
+ * (if available).
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ */
+static int
+connection_socks5_proxy_connect(connection_t *conn)
+{
+  tor_assert(conn);
 
-      connection_buf_add((char *)buf, buf_size, conn);
-      tor_free(buf);
+  const or_options_t *options = get_options();
+  unsigned char buf[4]; /* fields: vers, num methods, method list */
 
-      conn->proxy_state = PROXY_SOCKS4_WANT_CONNECT_OK;
-      break;
-    }
+  /* Send a SOCKS5 greeting (connect request must wait) */
 
-    case PROXY_SOCKS5: {
-      unsigned char buf[4]; /* fields: vers, num methods, method list */
+  buf[0] = 5; /* version */
 
-      /* Send a SOCKS5 greeting (connect request must wait) */
+  /* We have to use SOCKS5 authentication, if we have a
+     Socks5ProxyUsername or if we want to pass arguments to our
+     pluggable transport proxy: */
+  if ((options->Socks5ProxyUsername) ||
+      (conn_get_proxy_type(conn) == PROXY_PLUGGABLE &&
+       (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) {
+  /* number of auth methods */
+    buf[1] = 2;
+    buf[2] = 0x00; /* no authentication */
+    buf[3] = 0x02; /* rfc1929 Username/Passwd auth */
+    conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929;
+  } else {
+    buf[1] = 1;
+    buf[2] = 0x00; /* no authentication */
+    conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
+  }
 
-      buf[0] = 5; /* version */
+  connection_buf_add((char *)buf, 2 + buf[1], conn);
+  return 0;
+}
 
-      /* We have to use SOCKS5 authentication, if we have a
-         Socks5ProxyUsername or if we want to pass arguments to our
-         pluggable transport proxy: */
-      if ((options->Socks5ProxyUsername) ||
-          (conn_get_proxy_type(conn) == PROXY_PLUGGABLE &&
-           (get_socks_args_by_bridge_addrport(&conn->addr, conn->port)))) {
-      /* number of auth methods */
-        buf[1] = 2;
-        buf[2] = 0x00; /* no authentication */
-        buf[3] = 0x02; /* rfc1929 Username/Passwd auth */
-        conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_RFC1929;
-      } else {
-        buf[1] = 1;
-        buf[2] = 0x00; /* no authentication */
-        conn->proxy_state = PROXY_SOCKS5_WANT_AUTH_METHOD_NONE;
-      }
+/** Write a proxy request of haproxy to conn for conn->addr:conn->port.
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ */
+static int
+connection_haproxy_proxy_connect(connection_t *conn)
+{
+  int ret = 0;
+  tor_addr_port_t *addr_port = tor_addr_port_new(&conn->addr, conn->port);
+  char *buf = haproxy_format_proxy_header_line(addr_port);
+
+  if (buf == NULL) {
+    ret = -1;
+    goto done;
+  }
+
+  connection_buf_add(buf, strlen(buf), conn);
+  /* In haproxy, we don't have to wait for the response, but we wait for ack.
+   * So we can set the state to be PROXY_HAPROXY_WAIT_FOR_FLUSH. */
+  conn->proxy_state = PROXY_HAPROXY_WAIT_FOR_FLUSH;
+
+  ret = 0;
+ done:
+  tor_free(buf);
+  tor_free(addr_port);
+  return ret;
+}
+
+/** Write a proxy request of <b>type</b> (socks4, socks5, https, haproxy)
+ * to conn for conn->addr:conn->port, authenticating with the auth details
+ * given in the configuration (if available). SOCKS 5 and HTTP CONNECT
+ * proxies support authentication.
+ *
+ * Returns -1 if conn->addr is incompatible with the proxy protocol, and
+ * 0 otherwise.
+ *
+ * Use connection_read_proxy_handshake() to complete the handshake.
+ */
+int
+connection_proxy_connect(connection_t *conn, int type)
+{
+  int ret = 0;
+
+  tor_assert(conn);
+
+  switch (type) {
+    case PROXY_CONNECT:
+      ret = connection_https_proxy_connect(conn);
+      break;
 
-      connection_buf_add((char *)buf, 2 + buf[1], conn);
+    case PROXY_SOCKS4:
+      ret = connection_socks4_proxy_connect(conn);
+      break;
+
+    case PROXY_SOCKS5:
+      ret = connection_socks5_proxy_connect(conn);
+      break;
+
+    case PROXY_HAPROXY:
+      ret = connection_haproxy_proxy_connect(conn);
       break;
-    }
 
     default:
       log_err(LD_BUG, "Invalid proxy protocol, %d", type);
       tor_fragile_assert();
-      return -1;
+      ret = -1;
+      break;
   }
 
-  log_debug(LD_NET, "set state %s",
-            connection_proxy_state_to_string(conn->proxy_state));
+  if (ret == 0) {
+    log_debug(LD_NET, "set state %s",
+              connection_proxy_state_to_string(conn->proxy_state));
+  }
 
-  return 0;
+  return ret;
 }
 
 /** Read conn's inbuf. If the http response from the proxy is all
@@ -2841,7 +2955,7 @@ retry_listener_ports(smartlist_t *old_conns,
           SMARTLIST_DEL_CURRENT(old_conns, conn);
           break;
         }
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
       }
     } SMARTLIST_FOREACH_END(wanted);
 
@@ -2905,6 +3019,10 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
     retval = -1;
 
 #ifdef ENABLE_LISTENER_REBIND
+  if (smartlist_len(replacements))
+    log_debug(LD_NET, "%d replacements - starting rebinding loop.",
+              smartlist_len(replacements));
+
   SMARTLIST_FOREACH_BEGIN(replacements, listener_replacement_t *, r) {
     int addr_in_use = 0;
     int skip = 0;
@@ -2916,8 +3034,11 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
       connection_listener_new_for_port(r->new_port, &skip, &addr_in_use);
     connection_t *old_conn = r->old_conn;
 
-    if (skip)
+    if (skip) {
+      log_debug(LD_NET, "Skipping creating new listener for %s:%d",
+                old_conn->address, old_conn->port);
       continue;
+    }
 
     connection_close_immediate(old_conn);
     connection_mark_for_close(old_conn);
@@ -2943,7 +3064,7 @@ retry_all_listeners(smartlist_t *new_conns, int close_all_noncontrol)
                conn_type_to_string(old_conn->type), old_conn->address,
                old_conn->port, new_conn->address, new_conn->port);
   } SMARTLIST_FOREACH_END(r);
-#endif
+#endif /* defined(ENABLE_LISTENER_REBIND) */
 
   /* Any members that were still in 'listeners' don't correspond to
    * any configured port.  Kill 'em. */
@@ -3030,7 +3151,7 @@ connection_mark_all_noncontrol_connections(void)
  * uses pluggable transports, since we should then limit it even if it
  * comes from an internal IP address. */
 static int
-connection_is_rate_limited(connection_t *conn)
+connection_is_rate_limited(const connection_t *conn)
 {
   const or_options_t *options = get_options();
   if (conn->linked)
@@ -3166,14 +3287,14 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
                                      global_bucket_val, conn_bucket);
 }
 
-/** Return 1 if the global write buckets are low enough that we
+/** Return true iff the global write buckets are low enough that we
  * shouldn't send <b>attempt</b> bytes of low-priority directory stuff
- * out to <b>conn</b>. Else return 0.
-
- * Priority was 1 for v1 requests (directories and running-routers),
- * and 2 for v2 requests and later (statuses and descriptors).
+ * out to <b>conn</b>.
+ *
+ * If we are a directory authority, always answer dir requests thus true is
+ * always returned.
  *
- * There are a lot of parameters we could use here:
+ * Note: There are a lot of parameters we could use here:
  * - global_relayed_write_bucket. Low is bad.
  * - global_write_bucket. Low is bad.
  * - bandwidthrate. Low is bad.
@@ -3185,39 +3306,40 @@ connection_bucket_write_limit(connection_t *conn, time_t now)
  *   mean is "total directory bytes added to outbufs recently", but
  *   that's harder to quantify and harder to keep track of.
  */
-int
-global_write_bucket_low(connection_t *conn, size_t attempt, int priority)
+bool
+connection_dir_is_global_write_low(const connection_t *conn, size_t attempt)
 {
   size_t smaller_bucket =
     MIN(token_bucket_rw_get_write(&global_bucket),
         token_bucket_rw_get_write(&global_relayed_bucket));
-  if (authdir_mode(get_options()) && priority>1)
-    return 0; /* there's always room to answer v2 if we're an auth dir */
+
+  /* Special case for authorities (directory only). */
+  if (authdir_mode_v3(get_options())) {
+    /* Are we configured to possibly reject requests under load? */
+    if (!dirauth_should_reject_requests_under_load()) {
+      /* Answer request no matter what. */
+      return false;
+    }
+    /* Always answer requests from a known relay which includes the other
+     * authorities. The following looks up the addresses for relays that we
+     * have their descriptor _and_ any configured trusted directories. */
+    if (nodelist_probably_contains_address(&conn->addr)) {
+      return false;
+    }
+  }
 
   if (!connection_is_rate_limited(conn))
-    return 0; /* local conns don't get limited */
+    return false; /* local conns don't get limited */
 
   if (smaller_bucket < attempt)
-    return 1; /* not enough space no matter the priority */
+    return true; /* not enough space. */
 
   {
     const time_t diff = approx_time() - write_buckets_last_empty_at;
     if (diff <= 1)
-      return 1; /* we're already hitting our limits, no more please */
+      return true; /* we're already hitting our limits, no more please */
   }
-
-  if (priority == 1) { /* old-style v1 query */
-    /* Could we handle *two* of these requests within the next two seconds? */
-    const or_options_t *options = get_options();
-    size_t can_write = (size_t) (smaller_bucket
-      + 2*(options->RelayBandwidthRate ? options->RelayBandwidthRate :
-           options->BandwidthRate));
-    if (can_write < 2*attempt)
-      return 1;
-  } else { /* v2 query */
-    /* no further constraints yet */
-  }
-  return 0;
+  return false;
 }
 
 /** When did we last tell the accounting subsystem about transmitted
@@ -3239,8 +3361,17 @@ record_num_bytes_transferred_impl(connection_t *conn,
       rep_hist_note_dir_bytes_written(num_written, now);
   }
 
+  /* Linked connections and internal IPs aren't counted for statistics or
+   * accounting:
+   *  - counting linked connections would double-count BEGINDIR bytes, because
+   *    they are sent as Dir bytes on the linked connection, and OR bytes on
+   *    the OR connection;
+   *  - relays and clients don't connect to internal IPs, unless specifically
+   *    configured to do so. If they are configured that way, we don't count
+   *    internal bytes.
+   */
   if (!connection_is_rate_limited(conn))
-    return; /* local IPs are free */
+    return;
 
   if (conn->type == CONN_TYPE_OR)
     rep_hist_note_or_conn_bytes(conn->global_identifier, num_read,
@@ -3696,6 +3827,12 @@ connection_buf_read_from_socket(connection_t *conn, ssize_t *max_to_read,
     at_most = connection_bucket_read_limit(conn, approx_time());
   }
 
+  /* Do not allow inbuf to grow past BUF_MAX_LEN. */
+  const ssize_t maximum = BUF_MAX_LEN - buf_datalen(conn->inbuf);
+  if (at_most > maximum) {
+    at_most = maximum;
+  }
+
   slack_in_buf = buf_slack(conn->inbuf);
  again:
   if ((size_t)at_most > slack_in_buf && slack_in_buf >= 1024) {
@@ -3943,9 +4080,9 @@ update_send_buffer_size(tor_socket_t sock)
       &isb, sizeof(isb), &bytesReturned, NULL, NULL)) {
     setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (const char*)&isb, sizeof(isb));
   }
-#else
+#else /* !defined(_WIN32) */
   (void) sock;
-#endif
+#endif /* defined(_WIN32) */
 }
 
 /** Try to flush more bytes onto <b>conn</b>-\>s.
@@ -4080,6 +4217,7 @@ connection_handle_write_impl(connection_t *conn, int force)
     switch (result) {
       CASE_TOR_TLS_ERROR_ANY:
       case TOR_TLS_CLOSE:
+        or_conn->tls_error = result;
         log_info(LD_NET, result != TOR_TLS_CLOSE ?
                  "tls error. breaking.":"TLS connection closed on flush");
         /* Don't flush; connection is dead. */
@@ -4343,6 +4481,23 @@ connection_write_to_buf_impl_,(const char *string, size_t len,
   connection_write_to_buf_commit(conn, written);
 }
 
+/**
+ * Write a <b>string</b> (of size <b>len</b> to directory connection
+ * <b>dir_conn</b>. Apply compression if connection is configured to use
+ * it and finalize it if <b>done</b> is true.
+ */
+void
+connection_dir_buf_add(const char *string, size_t len,
+                       dir_connection_t *dir_conn, int done)
+{
+  if (dir_conn->compress_state != NULL) {
+    connection_buf_add_compress(string, len, dir_conn, done);
+    return;
+  }
+
+  connection_buf_add(string, len, TO_CONN(dir_conn));
+}
+
 void
 connection_buf_add_compress(const char *string, size_t len,
                             dir_connection_t *conn, int done)
@@ -4457,6 +4612,16 @@ connection_get_by_type_state(int type, int state)
   CONN_GET_TEMPLATE(conn, conn->type == type && conn->state == state);
 }
 
+/**
+ * Return a connection of type <b>type</b> that is not an internally linked
+ * connection, and is not marked for close.
+ **/
+MOCK_IMPL(connection_t *,
+connection_get_by_type_nonlinked,(int type))
+{
+  CONN_GET_TEMPLATE(conn, conn->type == type && !conn->linked);
+}
+
 /** Return a connection of type <b>type</b> that has rendquery equal
  * to <b>rendquery</b>, and that is not marked for close. If state
  * is non-zero, conn must be of that state too.
@@ -4837,10 +5002,10 @@ connection_finished_flushing(connection_t *conn)
   }
 }
 
-/** Called when our attempt to connect() to another server has just
- * succeeded.
+/** Called when our attempt to connect() to a server has just succeeded.
  *
- * This function just passes conn to the connection-specific
+ * This function checks if the interface address has changed (clients only),
+ * and then passes conn to the connection-specific
  * connection_*_finished_connecting() function.
  */
 static int
@@ -5299,7 +5464,7 @@ assert_connection_ok(connection_t *conn, time_t now)
         tor_assert(entry_conn->socks_request->has_finished);
         if (!conn->marked_for_close) {
           tor_assert(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
-          assert_cpath_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
+          cpath_assert_layer_ok(ENTRY_TO_EDGE_CONN(entry_conn)->cpath_layer);
         }
       }
     }
@@ -5353,17 +5518,20 @@ assert_connection_ok(connection_t *conn, time_t now)
 }
 
 /** Fills <b>addr</b> and <b>port</b> with the details of the global
- *  proxy server we are using.
- *  <b>conn</b> contains the connection we are using the proxy for.
+ *  proxy server we are using. Store a 1 to the int pointed to by
+ *  <b>is_put_out</b> if the connection is using a pluggable
+ *  transport; store 0 otherwise. <b>conn</b> contains the connection
+ *  we are using the proxy for.
  *
  *  Return 0 on success, -1 on failure.
  */
 int
 get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                   const connection_t *conn)
+                   int *is_pt_out, const connection_t *conn)
 {
   const or_options_t *options = get_options();
 
+  *is_pt_out = 0;
   /* Client Transport Plugins can use another proxy, but that should be hidden
    * from the rest of tor (as the plugin is responsible for dealing with the
    * proxy), check it first, then check the rest of the proxy types to allow
@@ -5379,6 +5547,7 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
       tor_addr_copy(addr, &transport->addr);
       *port = transport->port;
       *proxy_type = transport->socks_version;
+      *is_pt_out = 1;
       return 0;
     }
 
@@ -5400,6 +5569,13 @@ get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
     *port = options->Socks5ProxyPort;
     *proxy_type = PROXY_SOCKS5;
     return 0;
+  } else if (options->TCPProxy) {
+    tor_addr_copy(addr, &options->TCPProxyAddr);
+    *port = options->TCPProxyPort;
+    /* The only supported protocol in TCPProxy is haproxy. */
+    tor_assert(options->TCPProxyProtocol == TCP_PROXY_PROTOCOL_HAPROXY);
+    *proxy_type = PROXY_HAPROXY;
+    return 0;
   }
 
   tor_addr_make_unspec(addr);
@@ -5415,11 +5591,13 @@ log_failed_proxy_connection(connection_t *conn)
 {
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt;
 
-  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, conn) != 0)
+  if (get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         conn) != 0)
     return; /* if we have no proxy set up, leave this function. */
 
+  (void)is_pt;
   log_warn(LD_NET,
            "The connection to the %s proxy server at %s just failed. "
            "Make sure that the proxy server is up and running.",
@@ -5435,6 +5613,7 @@ proxy_type_to_string(int proxy_type)
   case PROXY_CONNECT:   return "HTTP";
   case PROXY_SOCKS4:    return "SOCKS4";
   case PROXY_SOCKS5:    return "SOCKS5";
+  case PROXY_HAPROXY:   return "HAPROXY";
   case PROXY_PLUGGABLE: return "pluggable transports SOCKS";
   case PROXY_NONE:      return "NULL";
   default:              tor_assert(0);
diff --git a/src/core/mainloop/connection.h b/src/core/mainloop/connection.h
index 8ecdd6f06f..bcd3d590a5 100644
--- a/src/core/mainloop/connection.h
+++ b/src/core/mainloop/connection.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,25 @@
 #ifndef TOR_CONNECTION_H
 #define TOR_CONNECTION_H
 
-listener_connection_t *TO_LISTENER_CONN(connection_t *);
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/log/log.h"
+
+#ifdef HAVE_SYS_SOCKET_H
+#include <sys/socket.h>
+#endif
+
+struct listener_connection_t;
+struct connection_t;
+struct dir_connection_t;
+struct or_connection_t;
+struct edge_connection_t;
+struct entry_connection_t;
+struct control_connection_t;
+struct port_cfg_t;
+struct tor_addr_t;
+struct or_options_t;
+
+struct listener_connection_t *TO_LISTENER_CONN(struct connection_t *);
 
 struct buf_t;
 
@@ -56,7 +74,7 @@ struct buf_t;
 
 #define CONN_TYPE_MAX_ 19
 /* !!!! If _CONN_TYPE_MAX is ever over 31, we must grow the type field in
- * connection_t. */
+ * struct connection_t. */
 
 /* Proxy client handshake states */
 /* We use a proxy but we haven't even connected to it yet. */
@@ -75,8 +93,10 @@ struct buf_t;
 #define PROXY_SOCKS5_WANT_AUTH_RFC1929_OK 6
 /* We use a SOCKS5 proxy and we just sent our CONNECT command. */
 #define PROXY_SOCKS5_WANT_CONNECT_OK 7
+/* We use an HAPROXY proxy and we just sent the proxy header. */
+#define PROXY_HAPROXY_WAIT_FOR_FLUSH 8
 /* We use a proxy and we CONNECTed successfully!. */
-#define PROXY_CONNECTED 8
+#define PROXY_CONNECTED 9
 
 /** State for any listener connection. */
 #define LISTENER_STATE_READY 0
@@ -88,34 +108,36 @@ struct buf_t;
  */
 typedef struct
 {
-  connection_t *old_conn; /* Old listener connection to be replaced */
-  const port_cfg_t *new_port; /* New port configuration */
+  struct connection_t *old_conn; /* Old listener connection to be replaced */
+  const struct port_cfg_t *new_port; /* New port configuration */
 } listener_replacement_t;
 
 const char *conn_type_to_string(int type);
 const char *conn_state_to_string(int type, int state);
 int conn_listener_type_supports_af_unix(int type);
 
-dir_connection_t *dir_connection_new(int socket_family);
-or_connection_t *or_connection_new(int type, int socket_family);
-edge_connection_t *edge_connection_new(int type, int socket_family);
-entry_connection_t *entry_connection_new(int type, int socket_family);
-control_connection_t *control_connection_new(int socket_family);
-listener_connection_t *listener_connection_new(int type, int socket_family);
-connection_t *connection_new(int type, int socket_family);
-int connection_init_accepted_conn(connection_t *conn,
-                                  const listener_connection_t *listener);
-void connection_link_connections(connection_t *conn_a, connection_t *conn_b);
-MOCK_DECL(void,connection_free_,(connection_t *conn));
+struct dir_connection_t *dir_connection_new(int socket_family);
+struct or_connection_t *or_connection_new(int type, int socket_family);
+struct edge_connection_t *edge_connection_new(int type, int socket_family);
+struct entry_connection_t *entry_connection_new(int type, int socket_family);
+struct control_connection_t *control_connection_new(int socket_family);
+struct listener_connection_t *listener_connection_new(int type,
+                                                      int socket_family);
+struct connection_t *connection_new(int type, int socket_family);
+int connection_init_accepted_conn(struct connection_t *conn,
+                                const struct listener_connection_t *listener);
+void connection_link_connections(struct connection_t *conn_a,
+                                 struct connection_t *conn_b);
+MOCK_DECL(void,connection_free_,(struct connection_t *conn));
 #define connection_free(conn) \
-  FREE_AND_NULL(connection_t, connection_free_, (conn))
+  FREE_AND_NULL(struct connection_t, connection_free_, (conn))
 void connection_free_all(void);
-void connection_about_to_close_connection(connection_t *conn);
-void connection_close_immediate(connection_t *conn);
-void connection_mark_for_close_(connection_t *conn,
+void connection_about_to_close_connection(struct connection_t *conn);
+void connection_close_immediate(struct connection_t *conn);
+void connection_mark_for_close_(struct connection_t *conn,
                                 int line, const char *file);
 MOCK_DECL(void, connection_mark_for_close_internal_,
-          (connection_t *conn, int line, const char *file));
+          (struct connection_t *conn, int line, const char *file));
 
 #define connection_mark_for_close(c) \
   connection_mark_for_close_((c), __LINE__, SHORT_FILE__)
@@ -130,11 +152,11 @@ MOCK_DECL(void, connection_mark_for_close_internal_,
  * connection_or_notify_error()), or you actually are the
  * connection_or_close_for_error() or connection_or_close_normally function.
  * For all other cases, use connection_mark_and_flush() instead, which
- * checks for or_connection_t properly, instead.  See below.
+ * checks for struct or_connection_t properly, instead.  See below.
  */
 #define connection_mark_and_flush_internal_(c,line,file)                \
   do {                                                                  \
-    connection_t *tmp_conn__ = (c);                                     \
+    struct connection_t *tmp_conn__ = (c);                              \
     connection_mark_for_close_internal_(tmp_conn__, (line), (file));    \
     tmp_conn__->hold_open_until_flushed = 1;                            \
   } while (0)
@@ -147,7 +169,7 @@ MOCK_DECL(void, connection_mark_for_close_internal_,
  */
 #define connection_mark_and_flush_(c,line,file)                           \
   do {                                                                    \
-    connection_t *tmp_conn_ = (c);                                        \
+    struct connection_t *tmp_conn_ = (c);                                 \
     if (tmp_conn_->type == CONN_TYPE_OR) {                                \
       log_warn(LD_CHANNEL | LD_BUG,                                       \
                "Something tried to close (and flush) an or_connection_t"  \
@@ -164,13 +186,13 @@ MOCK_DECL(void, connection_mark_for_close_internal_,
 
 void connection_expire_held_open(void);
 
-int connection_connect(connection_t *conn, const char *address,
-                       const tor_addr_t *addr,
+int connection_connect(struct connection_t *conn, const char *address,
+                       const struct tor_addr_t *addr,
                        uint16_t port, int *socket_error);
 
 #ifdef HAVE_SYS_UN_H
 
-int connection_connect_unix(connection_t *conn, const char *socket_path,
+int connection_connect_unix(struct connection_t *conn, const char *socket_path,
                             int *socket_error);
 
 #endif /* defined(HAVE_SYS_UN_H) */
@@ -183,75 +205,86 @@ int connection_connect_unix(connection_t *conn, const char *socket_path,
     username and password fields. */
 #define MAX_SOCKS5_AUTH_SIZE_TOTAL 2*MAX_SOCKS5_AUTH_FIELD_SIZE
 
-int connection_proxy_connect(connection_t *conn, int type);
-int connection_read_proxy_handshake(connection_t *conn);
-void log_failed_proxy_connection(connection_t *conn);
-int get_proxy_addrport(tor_addr_t *addr, uint16_t *port, int *proxy_type,
-                       const connection_t *conn);
+int connection_proxy_connect(struct connection_t *conn, int type);
+int connection_read_proxy_handshake(struct connection_t *conn);
+void log_failed_proxy_connection(struct connection_t *conn);
+int get_proxy_addrport(struct tor_addr_t *addr, uint16_t *port,
+                       int *proxy_type,
+                       int *is_pt_out, const struct connection_t *conn);
 
-int retry_all_listeners(smartlist_t *new_conns,
+int retry_all_listeners(struct smartlist_t *new_conns,
                         int close_all_noncontrol);
 
 void connection_mark_all_noncontrol_listeners(void);
 void connection_mark_all_noncontrol_connections(void);
 
-ssize_t connection_bucket_write_limit(connection_t *conn, time_t now);
-int global_write_bucket_low(connection_t *conn, size_t attempt, int priority);
+ssize_t connection_bucket_write_limit(struct connection_t *conn, time_t now);
+bool connection_dir_is_global_write_low(const struct connection_t *conn,
+                                        size_t attempt);
 void connection_bucket_init(void);
-void connection_bucket_adjust(const or_options_t *options);
+void connection_bucket_adjust(const struct or_options_t *options);
 void connection_bucket_refill_all(time_t now,
                                   uint32_t now_ts);
-void connection_read_bw_exhausted(connection_t *conn, bool is_global_bw);
-void connection_write_bw_exhausted(connection_t *conn, bool is_global_bw);
-void connection_consider_empty_read_buckets(connection_t *conn);
-void connection_consider_empty_write_buckets(connection_t *conn);
-
-int connection_handle_read(connection_t *conn);
-
-int connection_buf_get_bytes(char *string, size_t len, connection_t *conn);
-int connection_buf_get_line(connection_t *conn, char *data,
-                                   size_t *data_len);
-int connection_fetch_from_buf_http(connection_t *conn,
+void connection_read_bw_exhausted(struct connection_t *conn,
+                                  bool is_global_bw);
+void connection_write_bw_exhausted(struct connection_t *conn,
+                                   bool is_global_bw);
+void connection_consider_empty_read_buckets(struct connection_t *conn);
+void connection_consider_empty_write_buckets(struct connection_t *conn);
+
+int connection_handle_read(struct connection_t *conn);
+
+int connection_buf_get_bytes(char *string, size_t len,
+                             struct connection_t *conn);
+int connection_buf_get_line(struct connection_t *conn, char *data,
+                            size_t *data_len);
+int connection_fetch_from_buf_http(struct connection_t *conn,
                                char **headers_out, size_t max_headerlen,
                                char **body_out, size_t *body_used,
                                size_t max_bodylen, int force_complete);
 
-int connection_wants_to_flush(connection_t *conn);
-int connection_outbuf_too_full(connection_t *conn);
-int connection_handle_write(connection_t *conn, int force);
-int connection_flush(connection_t *conn);
+int connection_wants_to_flush(struct connection_t *conn);
+int connection_outbuf_too_full(struct connection_t *conn);
+int connection_handle_write(struct connection_t *conn, int force);
+int connection_flush(struct connection_t *conn);
 
 MOCK_DECL(void, connection_write_to_buf_impl_,
-          (const char *string, size_t len, connection_t *conn, int zlib));
+          (const char *string, size_t len, struct connection_t *conn,
+           int zlib));
 /* DOCDOC connection_write_to_buf */
 static void connection_buf_add(const char *string, size_t len,
-                                    connection_t *conn);
+                                    struct connection_t *conn);
+void connection_dir_buf_add(const char *string, size_t len,
+                            struct dir_connection_t *dir_conn, int done);
 static inline void
-connection_buf_add(const char *string, size_t len, connection_t *conn)
+connection_buf_add(const char *string, size_t len, struct connection_t *conn)
 {
   connection_write_to_buf_impl_(string, len, conn, 0);
 }
 void connection_buf_add_compress(const char *string, size_t len,
-                                 dir_connection_t *conn, int done);
-void connection_buf_add_buf(connection_t *conn, struct buf_t *buf);
-
-size_t connection_get_inbuf_len(connection_t *conn);
-size_t connection_get_outbuf_len(connection_t *conn);
-connection_t *connection_get_by_global_id(uint64_t id);
-
-connection_t *connection_get_by_type(int type);
-MOCK_DECL(connection_t *,connection_get_by_type_addr_port_purpose,(int type,
-                                                  const tor_addr_t *addr,
-                                                  uint16_t port, int purpose));
-connection_t *connection_get_by_type_state(int type, int state);
-connection_t *connection_get_by_type_state_rendquery(int type, int state,
+                                 struct dir_connection_t *conn, int done);
+void connection_buf_add_buf(struct connection_t *conn, struct buf_t *buf);
+
+size_t connection_get_inbuf_len(struct connection_t *conn);
+size_t connection_get_outbuf_len(struct connection_t *conn);
+struct connection_t *connection_get_by_global_id(uint64_t id);
+
+struct connection_t *connection_get_by_type(int type);
+MOCK_DECL(struct connection_t *,connection_get_by_type_nonlinked,(int type));
+MOCK_DECL(struct connection_t *,connection_get_by_type_addr_port_purpose,
+                                               (int type,
+                                                const struct tor_addr_t *addr,
+                                                uint16_t port, int purpose));
+struct connection_t *connection_get_by_type_state(int type, int state);
+struct connection_t *connection_get_by_type_state_rendquery(
+                                                     int type, int state,
                                                      const char *rendquery);
-smartlist_t *connection_list_by_type_state(int type, int state);
-smartlist_t *connection_list_by_type_purpose(int type, int purpose);
-smartlist_t *connection_dir_list_by_purpose_and_resource(
+struct smartlist_t *connection_list_by_type_state(int type, int state);
+struct smartlist_t *connection_list_by_type_purpose(int type, int purpose);
+struct smartlist_t *connection_dir_list_by_purpose_and_resource(
                                                   int purpose,
                                                   const char *resource);
-smartlist_t *connection_dir_list_by_purpose_resource_and_state(
+struct smartlist_t *connection_dir_list_by_purpose_resource_and_state(
                                                   int purpose,
                                                   const char *resource,
                                                   int state);
@@ -270,7 +303,7 @@ connection_dir_count_by_purpose_and_resource(
                                              int purpose,
                                              const char *resource)
 {
-  smartlist_t *conns = connection_dir_list_by_purpose_and_resource(
+  struct smartlist_t *conns = connection_dir_list_by_purpose_and_resource(
                                                                    purpose,
                                                                    resource);
   CONN_LEN_AND_FREE_TEMPLATE(conns);
@@ -284,7 +317,7 @@ connection_dir_count_by_purpose_resource_and_state(
                                                    const char *resource,
                                                    int state)
 {
-  smartlist_t *conns =
+  struct smartlist_t *conns =
     connection_dir_list_by_purpose_resource_and_state(
                                                       purpose,
                                                       resource,
@@ -294,26 +327,26 @@ connection_dir_count_by_purpose_resource_and_state(
 
 #undef CONN_LEN_AND_FREE_TEMPLATE
 
-int any_other_active_or_conns(const or_connection_t *this_conn);
+int any_other_active_or_conns(const struct or_connection_t *this_conn);
 
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
 #define connection_speaks_cells(conn) (((conn)->type == CONN_TYPE_OR) || 0)
-int connection_is_listener(connection_t *conn);
-int connection_state_is_open(connection_t *conn);
-int connection_state_is_connecting(connection_t *conn);
+int connection_is_listener(struct connection_t *conn);
+int connection_state_is_open(struct connection_t *conn);
+int connection_state_is_connecting(struct connection_t *conn);
 
 char *alloc_http_authenticator(const char *authenticator);
 
-void assert_connection_ok(connection_t *conn, time_t now);
-int connection_or_nonopen_was_started_here(or_connection_t *conn);
+void assert_connection_ok(struct connection_t *conn, time_t now);
+int connection_or_nonopen_was_started_here(struct or_connection_t *conn);
 void connection_dump_buffer_mem_stats(int severity);
 
 MOCK_DECL(void, clock_skew_warning,
-          (const connection_t *conn, long apparent_skew, int trusted,
+          (const struct connection_t *conn, long apparent_skew, int trusted,
            log_domain_mask_t domain, const char *received,
            const char *source));
 
-int connection_is_moribund(connection_t *conn);
+int connection_is_moribund(struct connection_t *conn);
 void connection_check_oos(int n_socks, int failed);
 
 /** Execute the statement <b>stmt</b>, which may log events concerning the
@@ -335,18 +368,18 @@ void connection_check_oos(int n_socks, int failed);
   STMT_END
 
 #ifdef CONNECTION_PRIVATE
-STATIC void connection_free_minimal(connection_t *conn);
+STATIC void connection_free_minimal(struct connection_t *conn);
 
 /* Used only by connection.c and test*.c */
 MOCK_DECL(STATIC int,connection_connect_sockaddr,
-                                            (connection_t *conn,
+                                            (struct connection_t *conn,
                                              const struct sockaddr *sa,
                                              socklen_t sa_len,
                                              const struct sockaddr *bindaddr,
                                              socklen_t bindaddr_len,
                                              int *socket_error));
-MOCK_DECL(STATIC void, kill_conn_list_for_oos, (smartlist_t *conns));
-MOCK_DECL(STATIC smartlist_t *, pick_oos_victims, (int n));
+MOCK_DECL(STATIC void, kill_conn_list_for_oos, (struct smartlist_t *conns));
+MOCK_DECL(STATIC struct smartlist_t *, pick_oos_victims, (int n));
 
 #endif /* defined(CONNECTION_PRIVATE) */
 
diff --git a/src/core/mainloop/core_mainloop.md b/src/core/mainloop/core_mainloop.md
new file mode 100644
index 0000000000..fee8a8179c
--- /dev/null
+++ b/src/core/mainloop/core_mainloop.md
@@ -0,0 +1,10 @@
+@dir /core/mainloop
+@brief core/mainloop: Non-onion-routing mainloop functionality
+
+This module uses the event-loop code of \refdir{lib/evloop} to implement an
+asynchronous connection-oriented protocol handler.
+
+The layering here is imperfect: the code here was split from \refdir{core/or}
+without refactoring how the two modules call one another.  Probably many
+functions should be moved and refactored.
+
diff --git a/src/core/mainloop/cpuworker.c b/src/core/mainloop/cpuworker.c
index e704d55642..485ddb9741 100644
--- a/src/core/mainloop/cpuworker.c
+++ b/src/core/mainloop/cpuworker.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,6 @@
  **/
 #include "core/or/or.h"
 #include "core/or/channel.h"
-#include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
@@ -27,6 +26,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "core/or/onion.h"
+#include "feature/relay/circuitbuild_relay.h"
 #include "feature/relay/onion_queue.h"
 #include "feature/stats/rephist.h"
 #include "feature/relay/router.h"
@@ -34,11 +34,10 @@
 #include "core/crypto/onion_crypto.h"
 
 #include "core/or/or_circuit_st.h"
-#include "lib/intmath/weakrng.h"
 
 static void queue_pending_tasks(void);
 
-typedef struct worker_state_s {
+typedef struct worker_state_t {
   int generation;
   server_onion_keys_t *onion_keys;
 } worker_state_t;
@@ -74,8 +73,6 @@ worker_state_free_void(void *arg)
 static replyqueue_t *replyqueue = NULL;
 static threadpool_t *threadpool = NULL;
 
-static tor_weak_rng_t request_sample_rng = TOR_WEAK_RNG_INIT;
-
 static int total_pending_tasks = 0;
 static int max_pending_tasks = 128;
 
@@ -109,7 +106,6 @@ cpu_init(void)
 
   /* Total voodoo. Can we make this more sensible? */
   max_pending_tasks = get_num_cpus(get_options()) * 64;
-  crypto_seed_weak_rng(&request_sample_rng);
 }
 
 /** Magic numbers to make sure our cpuworker_requests don't grow any
@@ -164,7 +160,7 @@ typedef struct cpuworker_reply_t {
   uint8_t rend_auth_material[DIGEST_LEN];
 } cpuworker_reply_t;
 
-typedef struct cpuworker_job_u {
+typedef struct cpuworker_job_u_t {
   or_circuit_t *circ;
   union {
     cpuworker_request_t request;
@@ -235,9 +231,10 @@ should_time_request(uint16_t onionskin_type)
    * sample */
   if (onionskins_n_processed[onionskin_type] < 4096)
     return 1;
+
   /** Otherwise, measure with P=1/128.  We avoid doing this for every
    * handshake, since the measurement itself can take a little time. */
-  return tor_weak_random_one_in_n(&request_sample_rng, 128);
+  return crypto_fast_rng_one_in_n(get_thread_fast_rng(), 128);
 }
 
 /** Return an estimate of how many microseconds we will need for a single
diff --git a/src/core/mainloop/cpuworker.h b/src/core/mainloop/cpuworker.h
index 77e2c42508..7e71961750 100644
--- a/src/core/mainloop/cpuworker.h
+++ b/src/core/mainloop/cpuworker.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,10 +14,10 @@
 
 void cpu_init(void);
 void cpuworkers_rotate_keyinfo(void);
-struct workqueue_entry_s;
+struct workqueue_entry_t;
 enum workqueue_reply_t;
 enum workqueue_priority_t;
-MOCK_DECL(struct workqueue_entry_s *, cpuworker_queue_work, (
+MOCK_DECL(struct workqueue_entry_t *, cpuworker_queue_work, (
                     enum workqueue_priority_t priority,
                     enum workqueue_reply_t (*fn)(void *, void *),
                     void (*reply_fn)(void *),
diff --git a/src/core/mainloop/include.am b/src/core/mainloop/include.am
new file mode 100644
index 0000000000..63643127f3
--- /dev/null
+++ b/src/core/mainloop/include.am
@@ -0,0 +1,22 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/mainloop/connection.c		\
+	src/core/mainloop/cpuworker.c		\
+	src/core/mainloop/mainloop.c		\
+	src/core/mainloop/mainloop_pubsub.c	\
+	src/core/mainloop/mainloop_sys.c	\
+	src/core/mainloop/netstatus.c		\
+	src/core/mainloop/periodic.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/mainloop/connection.h			\
+	src/core/mainloop/cpuworker.h			\
+	src/core/mainloop/mainloop.h			\
+	src/core/mainloop/mainloop_pubsub.h		\
+	src/core/mainloop/mainloop_state.inc    	\
+	src/core/mainloop/mainloop_state_st.h    	\
+	src/core/mainloop/mainloop_sys.h		\
+	src/core/mainloop/netstatus.h			\
+	src/core/mainloop/periodic.h
diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c
index e67e1299b2..b4dbedbfe4 100644
--- a/src/core/mainloop/mainloop.c
+++ b/src/core/mainloop/mainloop.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,12 +71,13 @@
 #include "feature/client/bridges.h"
 #include "feature/client/dnsserv.h"
 #include "feature/client/entrynodes.h"
+#include "feature/client/proxymode.h"
 #include "feature/client/transports.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
-#include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
-#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_cache.h"
@@ -95,7 +96,7 @@
 #include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/err/backtrace.h"
 #include "lib/tls/buffers_tls.h"
@@ -105,9 +106,6 @@
 
 #include <event2/event.h>
 
-#include "feature/dirauth/dirvote.h"
-#include "feature/dirauth/authmode.h"
-
 #include "core/or/cell_st.h"
 #include "core/or/entry_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -200,12 +198,10 @@ static int can_complete_circuits = 0;
 #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60)
 
 static int conn_close_if_marked(int i);
-static int run_main_loop_until_done(void);
 static void connection_start_reading_from_linked_conn(connection_t *conn);
 static int connection_should_read_from_linked_conn(connection_t *conn);
 static void conn_read_callback(evutil_socket_t fd, short event, void *_conn);
 static void conn_write_callback(evutil_socket_t fd, short event, void *_conn);
-static void second_elapsed_callback(periodic_timer_t *timer, void *args);
 static void shutdown_did_not_work_callback(evutil_socket_t fd, short event,
                                            void *arg) ATTR_NORETURN;
 
@@ -759,7 +755,7 @@ tor_shutdown_event_loop_for_restart_cb(
   tor_event_free(tor_shutdown_event_loop_for_restart_event);
   tor_shutdown_event_loop_and_exit(0);
 }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
 /**
  * After finishing the current callback (if any), shut down the main loop,
@@ -970,7 +966,6 @@ conn_close_if_marked(int i)
     return 0; /* nothing to see here, move along */
   now = time(NULL);
   assert_connection_ok(conn, now);
-  /* assert_all_pending_dns_resolves_ok(); */
 
   log_debug(LD_NET,"Cleaning up connection (fd "TOR_SOCKET_T_FORMAT").",
             conn->s);
@@ -1137,14 +1132,14 @@ directory_info_has_arrived(time_t now, int from_cache, int suppress_logs)
 
   if (!router_have_minimum_dir_info()) {
     int quiet = suppress_logs || from_cache ||
-                directory_too_idle_to_fetch_descriptors(options, now);
+                dirclient_too_idle_to_fetch_descriptors(options, now);
     tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR,
         "I learned some more directory information, but not enough to "
         "build a circuit: %s", get_dir_info_status_string());
     update_all_descriptor_downloads(now);
     return;
   } else {
-    if (directory_fetches_from_authorities(options)) {
+    if (dirclient_fetches_from_authorities(options)) {
       update_all_descriptor_downloads(now);
     }
 
@@ -1357,123 +1352,101 @@ get_signewnym_epoch(void)
 static int periodic_events_initialized = 0;
 
 /* Declare all the timer callback functions... */
+#ifndef COCCI
 #undef CALLBACK
 #define CALLBACK(name) \
   static int name ## _callback(time_t, const or_options_t *)
+
 CALLBACK(add_entropy);
-CALLBACK(check_authority_cert);
-CALLBACK(check_canonical_channels);
-CALLBACK(check_descriptor);
-CALLBACK(check_dns_honesty);
-CALLBACK(check_ed_keys);
 CALLBACK(check_expired_networkstatus);
-CALLBACK(check_for_reachability_bw);
-CALLBACK(check_onion_keys_expiry_time);
 CALLBACK(clean_caches);
 CALLBACK(clean_consdiffmgr);
-CALLBACK(dirvote);
-CALLBACK(downrate_stability);
-CALLBACK(expire_old_ciruits_serverside);
 CALLBACK(fetch_networkstatus);
 CALLBACK(heartbeat);
 CALLBACK(hs_service);
 CALLBACK(launch_descriptor_fetches);
-CALLBACK(launch_reachability_tests);
-CALLBACK(reachability_warnings);
+CALLBACK(prune_old_routers);
 CALLBACK(record_bridge_stats);
 CALLBACK(rend_cache_failure_clean);
 CALLBACK(reset_padding_counts);
-CALLBACK(retry_dns);
 CALLBACK(retry_listeners);
-CALLBACK(rotate_onion_key);
 CALLBACK(rotate_x509_certificate);
-CALLBACK(save_stability);
 CALLBACK(save_state);
-CALLBACK(write_bridge_ns);
 CALLBACK(write_stats_file);
+CALLBACK(control_per_second_events);
+CALLBACK(second_elapsed);
 
 #undef CALLBACK
 
 /* Now we declare an array of periodic_event_item_t for each periodic event */
-#define CALLBACK(name, r, f) PERIODIC_EVENT(name, r, f)
-
-STATIC periodic_event_item_t periodic_events[] = {
-  /* Everyone needs to run those. */
-  CALLBACK(add_entropy, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(check_expired_networkstatus, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(clean_caches, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(fetch_networkstatus, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(heartbeat, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(launch_descriptor_fetches, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reset_padding_counts, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(retry_listeners, PERIODIC_EVENT_ROLE_ALL,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_state, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(rotate_x509_certificate, PERIODIC_EVENT_ROLE_ALL, 0),
-  CALLBACK(write_stats_file, PERIODIC_EVENT_ROLE_ALL, 0),
-
-  /* Routers (bridge and relay) only. */
-  CALLBACK(check_descriptor, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_ed_keys, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(check_for_reachability_bw, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_onion_keys_expiry_time, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(expire_old_ciruits_serverside, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(reachability_warnings, PERIODIC_EVENT_ROLE_ROUTER,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(retry_dns, PERIODIC_EVENT_ROLE_ROUTER, 0),
-  CALLBACK(rotate_onion_key, PERIODIC_EVENT_ROLE_ROUTER, 0),
-
-  /* Authorities (bridge and directory) only. */
-  CALLBACK(downrate_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-  CALLBACK(launch_reachability_tests, PERIODIC_EVENT_ROLE_AUTHORITIES,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(save_stability, PERIODIC_EVENT_ROLE_AUTHORITIES, 0),
-
-  /* Directory authority only. */
-  CALLBACK(check_authority_cert, PERIODIC_EVENT_ROLE_DIRAUTH, 0),
-  CALLBACK(dirvote, PERIODIC_EVENT_ROLE_DIRAUTH, PERIODIC_EVENT_FLAG_NEED_NET),
-
-  /* Relay only. */
-  CALLBACK(check_canonical_channels, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
-  CALLBACK(check_dns_honesty, PERIODIC_EVENT_ROLE_RELAY,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+#define CALLBACK(name, r, f)                            \
+  PERIODIC_EVENT(name, PERIODIC_EVENT_ROLE_ ## r, f)
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+#endif /* !defined(COCCI) */
+
+STATIC periodic_event_item_t mainloop_periodic_events[] = {
+
+  /* Everyone needs to run these. They need to have very long timeouts for
+   * that to be safe. */
+  CALLBACK(add_entropy, ALL, 0),
+  CALLBACK(heartbeat, ALL, 0),
+  CALLBACK(reset_padding_counts, ALL, 0),
+
+  /* This is a legacy catch-all callback that runs once per second if
+   * we are online and active. */
+  CALLBACK(second_elapsed, NET_PARTICIPANT,
+           FL(RUN_ON_DISABLE)),
+
+  /* XXXX Do we have a reason to do this on a callback? Does it do any good at
+   * all?  For now, if we're dormant, we can let our listeners decay. */
+  CALLBACK(retry_listeners, NET_PARTICIPANT, FL(NEED_NET)),
+
+  /* We need to do these if we're participating in the Tor network. */
+  CALLBACK(check_expired_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(fetch_networkstatus, NET_PARTICIPANT, 0),
+  CALLBACK(launch_descriptor_fetches, NET_PARTICIPANT, FL(NEED_NET)),
+  CALLBACK(rotate_x509_certificate, NET_PARTICIPANT, 0),
+  CALLBACK(check_network_participation, NET_PARTICIPANT, 0),
+
+  /* We need to do these if we're participating in the Tor network, and
+   * immediately before we stop. */
+  CALLBACK(clean_caches, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(save_state, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(write_stats_file, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
+  CALLBACK(prune_old_routers, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Hidden Service service only. */
-  CALLBACK(hs_service, PERIODIC_EVENT_ROLE_HS_SERVICE,
-           PERIODIC_EVENT_FLAG_NEED_NET),
+  CALLBACK(hs_service, HS_SERVICE, FL(NEED_NET)), // XXXX break this down more
 
   /* Bridge only. */
-  CALLBACK(record_bridge_stats, PERIODIC_EVENT_ROLE_BRIDGE, 0),
+  CALLBACK(record_bridge_stats, BRIDGE, 0),
 
   /* Client only. */
-  CALLBACK(rend_cache_failure_clean, PERIODIC_EVENT_ROLE_CLIENT, 0),
-
-  /* Bridge Authority only. */
-  CALLBACK(write_bridge_ns, PERIODIC_EVENT_ROLE_BRIDGEAUTH, 0),
+  /* XXXX this could be restricted to CLIENT+NET_PARTICIPANT */
+  CALLBACK(rend_cache_failure_clean, NET_PARTICIPANT, FL(RUN_ON_DISABLE)),
 
   /* Directory server only. */
-  CALLBACK(clean_consdiffmgr, PERIODIC_EVENT_ROLE_DIRSERVER, 0),
+  CALLBACK(clean_consdiffmgr, DIRSERVER, 0),
+
+  /* Controller with per-second events only. */
+  CALLBACK(control_per_second_events, CONTROLEV, 0),
 
   END_OF_PERIODIC_EVENTS
 };
+#ifndef COCCI
 #undef CALLBACK
+#undef FL
+#endif
 
 /* These are pointers to members of periodic_events[] that are used to
  * implement particular callbacks.  We keep them separate here so that we
  * can access them by name.  We also keep them inside periodic_events[]
  * so that we can implement "reset all timers" in a reasonable way. */
-static periodic_event_item_t *check_descriptor_event=NULL;
-static periodic_event_item_t *dirvote_event=NULL;
 static periodic_event_item_t *fetch_networkstatus_event=NULL;
 static periodic_event_item_t *launch_descriptor_fetches_event=NULL;
 static periodic_event_item_t *check_dns_honesty_event=NULL;
 static periodic_event_item_t *save_state_event=NULL;
+static periodic_event_item_t *prune_old_routers_event=NULL;
 
 /** Reset all the periodic events so we'll do all our actions again as if we
  * just started up.
@@ -1483,24 +1456,7 @@ static periodic_event_item_t *save_state_event=NULL;
 void
 reset_all_main_loop_timers(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_reschedule(&periodic_events[i]);
-  }
-}
-
-/** Return the member of periodic_events[] whose name is <b>name</b>.
- * Return NULL if no such event is found.
- */
-static periodic_event_item_t *
-find_periodic_event(const char *name)
-{
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    if (strcmp(name, periodic_events[i].name) == 0)
-      return &periodic_events[i];
-  }
-  return NULL;
+  periodic_events_reset_all();
 }
 
 /** Return a bitmask of the roles this tor instance is configured for using
@@ -1510,7 +1466,7 @@ get_my_roles(const or_options_t *options)
 {
   tor_assert(options);
 
-  int roles = 0;
+  int roles = PERIODIC_EVENT_ROLE_ALL;
   int is_bridge = options->BridgeRelay;
   int is_relay = server_mode(options);
   int is_dirauth = authdir_mode_v3(options);
@@ -1518,6 +1474,8 @@ get_my_roles(const or_options_t *options)
   int is_hidden_service = !!hs_service_get_num_services() ||
                           !!rend_num_services();
   int is_dirserver = dir_server_mode(options);
+  int sending_control_events = control_any_per_second_event_enabled();
+
   /* We also consider tor to have the role of a client if the ControlPort is
    * set because a lot of things can be done over the control port which
    * requires tor to have basic functionnalities. */
@@ -1525,6 +1483,9 @@ get_my_roles(const or_options_t *options)
                   options->ControlPort_set ||
                   options->OwningControllerFD != UINT64_MAX;
 
+  int is_net_participant = is_participating_on_network() ||
+    is_relay || is_hidden_service;
+
   if (is_bridge) roles |= PERIODIC_EVENT_ROLE_BRIDGE;
   if (is_client) roles |= PERIODIC_EVENT_ROLE_CLIENT;
   if (is_relay) roles |= PERIODIC_EVENT_ROLE_RELAY;
@@ -1532,6 +1493,8 @@ get_my_roles(const or_options_t *options)
   if (is_bridgeauth) roles |= PERIODIC_EVENT_ROLE_BRIDGEAUTH;
   if (is_hidden_service) roles |= PERIODIC_EVENT_ROLE_HS_SERVICE;
   if (is_dirserver) roles |= PERIODIC_EVENT_ROLE_DIRSERVER;
+  if (is_net_participant) roles |= PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  if (sending_control_events) roles |= PERIODIC_EVENT_ROLE_CONTROLEV;
 
   return roles;
 }
@@ -1556,9 +1519,9 @@ initialize_periodic_events_cb(evutil_socket_t fd, short events, void *data)
   rescan_periodic_events(get_options());
 }
 
-/** Set up all the members of periodic_events[], and configure them all to be
- * launched from a callback. */
-STATIC void
+/** Set up all the members of mainloop_periodic_events[], and configure them
+ * all to be launched from a callback. */
+void
 initialize_periodic_events(void)
 {
   if (periodic_events_initialized)
@@ -1566,39 +1529,60 @@ initialize_periodic_events(void)
 
   periodic_events_initialized = 1;
 
-  /* Set up all periodic events. We'll launch them by roles. */
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_setup(&periodic_events[i]);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_events_register(&mainloop_periodic_events[i]);
   }
 
+  /* Set up all periodic events. We'll launch them by roles. */
+
+#ifndef COCCI
 #define NAMED_CALLBACK(name) \
-  STMT_BEGIN name ## _event = find_periodic_event( #name ); STMT_END
+  STMT_BEGIN name ## _event = periodic_events_find( #name ); STMT_END
+#endif
 
-  NAMED_CALLBACK(check_descriptor);
-  NAMED_CALLBACK(dirvote);
+  NAMED_CALLBACK(prune_old_routers);
   NAMED_CALLBACK(fetch_networkstatus);
   NAMED_CALLBACK(launch_descriptor_fetches);
   NAMED_CALLBACK(check_dns_honesty);
   NAMED_CALLBACK(save_state);
-
-  struct timeval one_second = { 1, 0 };
-  initialize_periodic_events_event = tor_evtimer_new(
-                  tor_libevent_get_base(),
-                  initialize_periodic_events_cb, NULL);
-  event_add(initialize_periodic_events_event, &one_second);
 }
 
 STATIC void
 teardown_periodic_events(void)
 {
-  int i;
-  for (i = 0; periodic_events[i].name; ++i) {
-    periodic_event_destroy(&periodic_events[i]);
-  }
+  periodic_events_disconnect_all();
+  fetch_networkstatus_event = NULL;
+  launch_descriptor_fetches_event = NULL;
+  check_dns_honesty_event = NULL;
+  save_state_event = NULL;
+  prune_old_routers_event = NULL;
   periodic_events_initialized = 0;
 }
 
+static mainloop_event_t *rescan_periodic_events_ev = NULL;
+
+/** Callback: rescan the periodic event list. */
+static void
+rescan_periodic_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void)event;
+  (void)arg;
+  rescan_periodic_events(get_options());
+}
+
+/**
+ * Schedule an event that will rescan which periodic events should run.
+ **/
+MOCK_IMPL(void,
+schedule_rescan_periodic_events,(void))
+{
+  if (!rescan_periodic_events_ev) {
+    rescan_periodic_events_ev =
+      mainloop_event_new(rescan_periodic_events_cb, NULL);
+  }
+  mainloop_event_activate(rescan_periodic_events_ev);
+}
+
 /** Do a pass at all our periodic events, disable those we don't need anymore
  * and enable those we need now using the given options. */
 void
@@ -1606,36 +1590,7 @@ rescan_periodic_events(const or_options_t *options)
 {
   tor_assert(options);
 
-  /* Avoid scanning the event list if we haven't initialized it yet. This is
-   * particularly useful for unit tests in order to avoid initializing main
-   * loop events everytime. */
-  if (!periodic_events_initialized) {
-    return;
-  }
-
-  int roles = get_my_roles(options);
-
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-
-    int enable = !!(item->roles & roles);
-
-    /* Handle the event flags. */
-    if (net_is_disabled() &&
-        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
-      enable = 0;
-    }
-
-    /* Enable the event if needed. It is safe to enable an event that was
-     * already enabled. Same goes for disabling it. */
-    if (enable) {
-      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
-      periodic_event_enable(item);
-    } else {
-      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
-      periodic_event_disable(item);
-    }
-  }
+  periodic_events_rescan_by_roles(get_my_roles(options), net_is_disabled());
 }
 
 /* We just got new options globally set, see if we need to enabled or disable
@@ -1643,26 +1598,7 @@ rescan_periodic_events(const or_options_t *options)
 void
 periodic_events_on_new_options(const or_options_t *options)
 {
-  /* Only if we've already initialized the events, rescan the list which will
-   * enable or disable events depending on our roles. This will be called at
-   * bootup and we don't want this function to initialize the events because
-   * they aren't set up at this stage. */
-  if (periodic_events_initialized) {
-    rescan_periodic_events(options);
-  }
-}
-
-/**
- * Update our schedule so that we'll check whether we need to update our
- * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
- * seconds.
- */
-void
-reschedule_descriptor_update_check(void)
-{
-  if (check_descriptor_event) {
-    periodic_event_reschedule(check_descriptor_event);
-  }
+  rescan_periodic_events(options);
 }
 
 /**
@@ -1708,40 +1644,41 @@ mainloop_schedule_postloop_cleanup(void)
   mainloop_event_activate(postloop_cleanup_ev);
 }
 
-#define LONGEST_TIMER_PERIOD (30 * 86400)
-/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
- * clipped to the range [1 second, LONGEST_TIMER_PERIOD]. */
-static inline int
-safe_timer_diff(time_t now, time_t next)
-{
-  if (next > now) {
-    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
-     * and nothing that could run Tor. It's a bug if 'next' is around then.
-     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
-     * Bang. We cannot extrapolate past a singularity, but there was probably
-     * nothing that could run Tor then, either.
-     **/
-    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
-
-    if (next - LONGEST_TIMER_PERIOD > now)
-      return LONGEST_TIMER_PERIOD;
-    return (int)(next - now);
-  } else {
-    return 1;
+/** Event to run 'scheduled_shutdown_cb' */
+static mainloop_event_t *scheduled_shutdown_ev=NULL;
+
+/** Callback: run a scheduled shutdown */
+static void
+scheduled_shutdown_cb(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  (void)arg;
+  log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+  tor_shutdown_event_loop_and_exit(0);
+}
+
+/** Schedule the mainloop to exit after <b>delay_sec</b> seconds. */
+void
+mainloop_schedule_shutdown(int delay_sec)
+{
+  const struct timeval delay_tv = { delay_sec, 0 };
+  if (! scheduled_shutdown_ev) {
+    scheduled_shutdown_ev = mainloop_event_new(scheduled_shutdown_cb, NULL);
   }
+  mainloop_event_schedule(scheduled_shutdown_ev, &delay_tv);
 }
 
 /** Perform regular maintenance tasks.  This function gets run once per
- * second by second_elapsed_callback().
+ * second.
  */
-static void
-run_scheduled_events(time_t now)
+static int
+second_elapsed_callback(time_t now, const or_options_t *options)
 {
-  const or_options_t *options = get_options();
-
-  /* 0. See if we've been asked to shut down and our timeout has
-   * expired; or if our bandwidth limits are exhausted and we
-   * should hibernate; or if it's time to wake up from hibernation.
+  /* 0. See if our bandwidth limits are exhausted and we should hibernate
+   *
+   * Note: we have redundant mechanisms to handle the case where it's
+   * time to wake up from hibernation; or where we have a scheduled
+   * shutdown and it's time to run it, but this will also handle those.
    */
   consider_hibernation(now);
 
@@ -1751,10 +1688,13 @@ run_scheduled_events(time_t now)
   if (options->UseBridges && !net_is_disabled()) {
     /* Note: this check uses net_is_disabled(), not should_delay_dir_fetches()
      * -- the latter is only for fetching consensus-derived directory info. */
+    // TODO: client
+    //     Also, schedule this rather than probing 1x / sec
     fetch_bridge_descriptors(options, now);
   }
 
   if (accounting_is_enabled(options)) {
+    // TODO: refactor or rewrite?
     accounting_run_housekeeping(now);
   }
 
@@ -1765,6 +1705,7 @@ run_scheduled_events(time_t now)
    */
   /* (If our circuit build timeout can ever become lower than a second (which
    * it can't, currently), we should do this more often.) */
+  // TODO: All expire stuff can become NET_PARTICIPANT, RUN_ON_DISABLE
   circuit_expire_building();
   circuit_expire_waiting_for_better_guard();
 
@@ -1798,80 +1739,8 @@ run_scheduled_events(time_t now)
     run_connection_housekeeping(i, now);
   }
 
-  /* 11b. check pending unconfigured managed proxies */
-  if (!net_is_disabled() && pt_proxies_configuration_pending())
-    pt_configure_remaining_proxies();
-}
-
-/* Periodic callback: rotate the onion keys after the period defined by the
- * "onion-key-rotation-days" consensus parameter, shut down and restart all
- * cpuworkers, and update our descriptor if necessary.
- */
-static int
-rotate_onion_key_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_lifetime = get_onion_key_lifetime();
-    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
-    if (rotation_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL,"Rotating onion key.");
-    rotate_onion_key();
-    cpuworkers_rotate_keyinfo();
-    if (router_rebuild_descriptor(1)<0) {
-      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
-    }
-    if (advertised_server_mode() && !net_is_disabled())
-      router_upload_dir_desc_to_dirservers(0);
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Period callback: Check if our old onion keys are still valid after the
- * period of time defined by the consensus parameter
- * "onion-key-grace-period-days", otherwise expire them by setting them to
- * NULL.
- */
-static int
-check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    int onion_key_grace_period = get_onion_key_grace_period();
-    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
-    if (expiry_time > now) {
-      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-    }
-
-    log_info(LD_GENERAL, "Expiring old onion keys.");
-    expire_old_onion_keys();
-    cpuworkers_rotate_keyinfo();
-    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
-  }
-
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
-/* Periodic callback: Every 30 seconds, check whether it's time to make new
- * Ed25519 subkeys.
- */
-static int
-check_ed_keys_callback(time_t now, const or_options_t *options)
-{
-  if (server_mode(options)) {
-    if (should_make_new_ed_keys(options, now)) {
-      int new_signing_key = load_ed_keys(options, now);
-      if (new_signing_key < 0 ||
-          generate_ed_link_cert(options, now, new_signing_key > 0)) {
-        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
-        tor_shutdown_event_loop_and_exit(1);
-      }
-    }
-    return 30;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
+  /* Run again in a second. */
+  return 1;
 }
 
 /**
@@ -1947,100 +1816,53 @@ add_entropy_callback(time_t now, const or_options_t *options)
   return ENTROPY_INTERVAL;
 }
 
-/**
- * Periodic callback: if we're an authority, make sure we test
- * the routers on the network for reachability.
- */
-static int
-launch_reachability_tests_callback(time_t now, const or_options_t *options)
+/** Periodic callback: if there has been no network usage in a while,
+ * enter a dormant state. */
+STATIC int
+check_network_participation_callback(time_t now, const or_options_t *options)
 {
-  if (authdir_mode_tests_reachability(options) &&
-      !net_is_disabled()) {
-    /* try to determine reachability of the other Tor relays */
-    dirserv_test_reachability(now);
+  /* If we're a server, we can't become dormant. */
+  if (server_mode(options)) {
+    goto found_activity;
   }
-  return REACHABILITY_TEST_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, discount the stability
- * information (and other rephist information) that's older.
- */
-static int
-downrate_stability_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* 1d. Periodically, we discount older stability information so that new
-   * stability info counts more, and save the stability information to disk as
-   * appropriate. */
-  time_t next = rep_hist_downrate_old_runs(now);
-  return safe_timer_diff(now, next);
-}
+  /* If we're running an onion service, we can't become dormant. */
+  /* XXXX this would be nice to change, so that we can be dormant with a
+   * service. */
+  if (hs_service_get_num_services() || rend_num_services()) {
+    goto found_activity;
+  }
 
-/**
- * Periodic callback: if we're an authority, record our measured stability
- * information from rephist in an mtbf file.
- */
-static int
-save_stability_callback(time_t now, const or_options_t *options)
-{
-  if (authdir_mode_tests_reachability(options)) {
-    if (rep_hist_record_mtbf_data(now, 1)<0) {
-      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+  /* If we have any currently open entry streams other than "linked"
+   * connections used for directory requests, those count as user activity.
+   */
+  if (options->DormantTimeoutDisabledByIdleStreams) {
+    if (connection_get_by_type_nonlinked(CONN_TYPE_AP) != NULL) {
+      goto found_activity;
     }
   }
-#define SAVE_STABILITY_INTERVAL (30*60)
-  return SAVE_STABILITY_INTERVAL;
-}
 
-/**
- * Periodic callback: if we're an authority, check on our authority
- * certificate (the one that authenticates our authority signing key).
- */
-static int
-check_authority_cert_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  (void)options;
-  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
-   * close to expiring and warn the admin if it is. */
-  v3_authority_check_key_expiry();
-#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
-  return CHECK_V3_CERTIFICATE_INTERVAL;
-}
+  /* XXXX Make this configurable? */
+/** How often do we check whether we have had network activity? */
+#define CHECK_PARTICIPATION_INTERVAL (5*60)
 
-/**
- * Scheduled callback: Run directory-authority voting functionality.
- *
- * The schedule is a bit complicated here, so dirvote_act() manages the
- * schedule itself.
- **/
-static int
-dirvote_callback(time_t now, const or_options_t *options)
-{
-  if (!authdir_mode_v3(options)) {
-    tor_assert_nonfatal_unreached();
-    return 3600;
+  /* Become dormant if there has been no user activity in a long time.
+   * (The funny checks below are in order to prevent overflow.) */
+  time_t time_since_last_activity = 0;
+  if (get_last_user_activity_time() < now)
+    time_since_last_activity = now - get_last_user_activity_time();
+  if (time_since_last_activity >= options->DormantClientTimeout) {
+    log_notice(LD_GENERAL, "No user activity in a long time: becoming"
+               " dormant.");
+    set_network_participation(false);
+    rescan_periodic_events(options);
   }
 
-  time_t next = dirvote_act(options, now);
-  if (BUG(next == TIME_MAX)) {
-    /* This shouldn't be returned unless we called dirvote_act() without
-     * being an authority.  If it happens, maybe our configuration will
-     * fix itself in an hour or so? */
-    return 3600;
-  }
-  return safe_timer_diff(now, next);
-}
+  return CHECK_PARTICIPATION_INTERVAL;
 
-/** Reschedule the directory-authority voting event.  Run this whenever the
- * schedule has changed. */
-void
-reschedule_dirvote(const or_options_t *options)
-{
-  if (periodic_events_initialized && authdir_mode_v3(options)) {
-    periodic_event_reschedule(dirvote_event);
-  }
+ found_activity:
+  note_user_activity(now);
+  return CHECK_PARTICIPATION_INTERVAL;
 }
 
 /**
@@ -2053,11 +1875,9 @@ check_expired_networkstatus_callback(time_t now, const or_options_t *options)
   (void)options;
   /* Check whether our networkstatus has expired. */
   networkstatus_t *ns = networkstatus_get_latest_consensus();
-  /*XXXX RD: This value needs to be the same as REASONABLY_LIVE_TIME in
-   * networkstatus_get_reasonably_live_consensus(), but that value is way
-   * way too high.  Arma: is the bridge issue there resolved yet? -NM */
-#define NS_EXPIRY_SLOP (24*60*60)
-  if (ns && ns->valid_until < (now - NS_EXPIRY_SLOP) &&
+  /* Use reasonably live consensuses until they are no longer reasonably live.
+   */
+  if (ns && !networkstatus_consensus_reasonably_live(ns, now) &&
       router_have_minimum_dir_info()) {
     router_dir_info_changed();
   }
@@ -2143,17 +1963,6 @@ write_stats_file_callback(time_t now, const or_options_t *options)
   return safe_timer_diff(now, next_time_to_write_stats_files);
 }
 
-#define CHANNEL_CHECK_INTERVAL (60*60)
-static int
-check_canonical_channels_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  if (public_server_mode(options))
-    channel_check_for_duplicates();
-
-  return CHANNEL_CHECK_INTERVAL;
-}
-
 static int
 reset_padding_counts_callback(time_t now, const or_options_t *options)
 {
@@ -2228,87 +2037,24 @@ rend_cache_failure_clean_callback(time_t now, const or_options_t *options)
 }
 
 /**
- * Periodic callback: If we're a server and initializing dns failed, retry.
+ * Periodic callback: prune routerlist of old information about Tor network.
  */
 static int
-retry_dns_callback(time_t now, const or_options_t *options)
+prune_old_routers_callback(time_t now, const or_options_t *options)
 {
+#define ROUTERLIST_PRUNING_INTERVAL (60*60) // 1 hour.
   (void)now;
-#define RETRY_DNS_INTERVAL (10*60)
-  if (server_mode(options) && has_dns_init_failed())
-    dns_init();
-  return RETRY_DNS_INTERVAL;
-}
-
-/** Periodic callback: consider rebuilding or and re-uploading our descriptor
- * (if we've passed our internal checks). */
-static int
-check_descriptor_callback(time_t now, const or_options_t *options)
-{
-/** How often do we check whether part of our router info has changed in a
- * way that would require an upload? That includes checking whether our IP
- * address has changed. */
-#define CHECK_DESCRIPTOR_INTERVAL (60)
-
   (void)options;
 
-  /* 2b. Once per minute, regenerate and upload the descriptor if the old
-   * one is inaccurate. */
   if (!net_is_disabled()) {
-    check_descriptor_bandwidth_changed(now);
-    check_descriptor_ipaddress_changed(now);
-    mark_my_descriptor_dirty_if_too_old(now);
-    consider_publishable_server(0);
     /* If any networkstatus documents are no longer recent, we need to
      * update all the descriptors' running status. */
     /* Remove dead routers. */
-    /* XXXX This doesn't belong here, but it was here in the pre-
-     * XXXX refactoring code. */
+    log_debug(LD_GENERAL, "Pruning routerlist...");
     routerlist_remove_old_routers();
   }
 
-  return CHECK_DESCRIPTOR_INTERVAL;
-}
-
-/**
- * Periodic callback: check whether we're reachable (as a relay), and
- * whether our bandwidth has changed enough that we need to
- * publish a new descriptor.
- */
-static int
-check_for_reachability_bw_callback(time_t now, const or_options_t *options)
-{
-  /* XXXX This whole thing was stuck in the middle of what is now
-   * XXXX check_descriptor_callback.  I'm not sure it's right. */
-
-  static int dirport_reachability_count = 0;
-  /* also, check religiously for reachability, if it's within the first
-   * 20 minutes of our uptime. */
-  if (server_mode(options) &&
-      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
-      !net_is_disabled()) {
-    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-      router_do_reachability_checks(1, dirport_reachability_count==0);
-      if (++dirport_reachability_count > 5)
-        dirport_reachability_count = 0;
-      return 1;
-    } else {
-      /* If we haven't checked for 12 hours and our bandwidth estimate is
-       * low, do another bandwidth test. This is especially important for
-       * bridges, since they might go long periods without much use. */
-      const routerinfo_t *me = router_get_my_routerinfo();
-      static int first_time = 1;
-      if (!first_time && me &&
-          me->bandwidthcapacity < me->bandwidthrate &&
-          me->bandwidthcapacity < 51200) {
-        reset_bandwidth_test();
-      }
-      first_time = 0;
-#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
-      return BANDWIDTH_RECHECK_INTERVAL;
-    }
-  }
-  return CHECK_DESCRIPTOR_INTERVAL;
+  return ROUTERLIST_PRUNING_INTERVAL;
 }
 
 /**
@@ -2322,7 +2068,7 @@ fetch_networkstatus_callback(time_t now, const or_options_t *options)
    * documents? */
   const int we_are_bootstrapping = networkstatus_consensus_is_bootstrapping(
                                                                         now);
-  const int prefer_mirrors = !directory_fetches_from_authorities(
+  const int prefer_mirrors = !dirclient_fetches_from_authorities(
                                                               get_options());
   int networkstatus_dl_check_interval = 60;
   /* check more often when testing, or when bootstrapping from mirrors
@@ -2353,109 +2099,6 @@ retry_listeners_callback(time_t now, const or_options_t *options)
   return PERIODIC_EVENT_NO_UPDATE;
 }
 
-/**
- * Periodic callback: as a server, see if we have any old unused circuits
- * that should be expired */
-static int
-expire_old_ciruits_serverside_callback(time_t now, const or_options_t *options)
-{
-  (void)options;
-  /* every 11 seconds, so not usually the same second as other such events */
-  circuit_expire_old_circuits_serverside(now);
-  return 11;
-}
-
-/**
- * Callback: Send warnings if Tor doesn't find its ports reachable.
- */
-static int
-reachability_warnings_callback(time_t now, const or_options_t *options)
-{
-  (void) now;
-
-  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
-    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
-  }
-
-  if (server_mode(options) &&
-      !net_is_disabled() &&
-      have_completed_a_circuit()) {
-    /* every 20 minutes, check and complain if necessary */
-    const routerinfo_t *me = router_get_my_routerinfo();
-    if (me && !check_whether_orport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,"Your server (%s:%d) has not managed to confirm that "
-               "its ORPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->or_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED ORADDRESS=%s:%d",
-                                  address, me->or_port);
-      tor_free(address);
-    }
-
-    if (me && !check_whether_dirport_reachable(options)) {
-      char *address = tor_dup_ip(me->addr);
-      log_warn(LD_CONFIG,
-               "Your server (%s:%d) has not managed to confirm that its "
-               "DirPort is reachable. Relays do not publish descriptors "
-               "until their ORPort and DirPort are reachable. Please check "
-               "your firewalls, ports, address, /etc/hosts file, etc.",
-               address, me->dir_port);
-      control_event_server_status(LOG_WARN,
-                                  "REACHABILITY_FAILED DIRADDRESS=%s:%d",
-                                  address, me->dir_port);
-      tor_free(address);
-    }
-  }
-
-  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
-}
-
-static int dns_honesty_first_time = 1;
-
-/**
- * Periodic event: if we're an exit, see if our DNS server is telling us
- * obvious lies.
- */
-static int
-check_dns_honesty_callback(time_t now, const or_options_t *options)
-{
-  (void)now;
-  /* 9. and if we're an exit node, check whether our DNS is telling stories
-   * to us. */
-  if (net_is_disabled() ||
-      ! public_server_mode(options) ||
-      router_my_exit_policy_is_reject_star())
-    return PERIODIC_EVENT_NO_UPDATE;
-
-  if (dns_honesty_first_time) {
-    /* Don't launch right when we start */
-    dns_honesty_first_time = 0;
-    return crypto_rand_int_range(60, 180);
-  }
-
-  dns_launch_correctness_checks();
-  return 12*3600 + crypto_rand_int(12*3600);
-}
-
-/**
- * Periodic callback: if we're the bridge authority, write a networkstatus
- * file to disk.
- */
-static int
-write_bridge_ns_callback(time_t now, const or_options_t *options)
-{
-  /* 10. write bridge networkstatus file to disk */
-  if (options->BridgeAuthoritativeDir) {
-    networkstatus_dump_bridge_status_to_file(now);
-#define BRIDGE_STATUSFILE_INTERVAL (30*60)
-     return BRIDGE_STATUSFILE_INTERVAL;
-  }
-  return PERIODIC_EVENT_NO_UPDATE;
-}
-
 static int heartbeat_callback_first_time = 1;
 
 /**
@@ -2523,36 +2166,19 @@ hs_service_callback(time_t now, const or_options_t *options)
   return 1;
 }
 
-/** Timer: used to invoke second_elapsed_callback() once per second. */
-static periodic_timer_t *second_timer = NULL;
-
-/**
- * Enable or disable the per-second timer as appropriate, creating it if
- * necessary.
+/*
+ * Periodic callback: Send once-per-second events to the controller(s).
+ * This is called every second.
  */
-void
-reschedule_per_second_timer(void)
+static int
+control_per_second_events_callback(time_t now, const or_options_t *options)
 {
-  struct timeval one_second;
-  one_second.tv_sec = 1;
-  one_second.tv_usec = 0;
-
-  if (! second_timer) {
-    second_timer = periodic_timer_new(tor_libevent_get_base(),
-                                      &one_second,
-                                      second_elapsed_callback,
-                                      NULL);
-    tor_assert(second_timer);
-  }
+  (void) options;
+  (void) now;
 
-  const bool run_per_second_events =
-    control_any_per_second_event_enabled() || ! net_is_completely_disabled();
+  control_per_second_events();
 
-  if (run_per_second_events) {
-    periodic_timer_launch(second_timer, &one_second);
-  } else {
-    periodic_timer_disable(second_timer);
-  }
+  return 1;
 }
 
 /** Last time that update_current_time was called. */
@@ -2582,6 +2208,17 @@ update_current_time(time_t now)
   memcpy(&last_updated, &current_second_last_changed, sizeof(last_updated));
   monotime_coarse_get(&current_second_last_changed);
 
+  /** How much clock jumping means that we should adjust our idea of when
+   * to go dormant? */
+#define NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE 20
+
+  /* Don't go dormant early or late just because we jumped in time. */
+  if (ABS(seconds_elapsed) >= NUM_JUMPED_SECONDS_BEFORE_NETSTATUS_UPDATE) {
+    if (is_participating_on_network()) {
+      netstatus_note_clock_jumped(seconds_elapsed);
+    }
+  }
+
   /** How much clock jumping do we tolerate? */
 #define NUM_JUMPED_SECONDS_BEFORE_WARN 100
 
@@ -2591,6 +2228,7 @@ update_current_time(time_t now)
   if (seconds_elapsed < -NUM_JUMPED_SECONDS_BEFORE_WARN) {
     // moving back in time is always a bad sign.
     circuit_note_clock_jumped(seconds_elapsed, false);
+
   } else if (seconds_elapsed >= NUM_JUMPED_SECONDS_BEFORE_WARN) {
     /* Compare the monotonic clock to the result of time(). */
     const int32_t monotime_msec_passed =
@@ -2620,31 +2258,6 @@ update_current_time(time_t now)
   current_second = now;
 }
 
-/** Libevent callback: invoked once every second. */
-static void
-second_elapsed_callback(periodic_timer_t *timer, void *arg)
-{
-  /* XXXX This could be sensibly refactored into multiple callbacks, and we
-   * could use Libevent's timers for this rather than checking the current
-   * time against a bunch of timeouts every second. */
-  time_t now;
-  (void)timer;
-  (void)arg;
-
-  now = time(NULL);
-
-  /* We don't need to do this once-per-second any more: time-updating is
-   * only in this callback _because it is a callback_. It should be fine
-   * to disable this callback, and the time will still get updated.
-   */
-  update_current_time(now);
-
-  /* Maybe some controller events are ready to fire */
-  control_per_second_events();
-
-  run_scheduled_events(now);
-}
-
 #ifdef HAVE_SYSTEMD_209
 static periodic_timer_t *systemd_watchdog_timer = NULL;
 
@@ -2660,18 +2273,23 @@ systemd_watchdog_callback(periodic_timer_t *timer, void *arg)
 
 #define UPTIME_CUTOFF_FOR_NEW_BANDWIDTH_TEST (6*60*60)
 
-/** Called when our IP address seems to have changed. <b>at_interface</b>
- * should be true if we detected a change in our interface, and false if we
- * detected a change in our published address. */
+/** Called when our IP address seems to have changed. <b>on_client_conn</b>
+ * should be true if:
+ *   - we detected a change in our interface address, using an outbound
+ *     connection, and therefore
+ *   - our client TLS keys need to be rotated.
+ * Otherwise, it should be false, and:
+ *   - we detected a change in our published address
+ *     (using some other method), and therefore
+ *   - the published addresses in our descriptor need to change.
+ */
 void
-ip_address_changed(int at_interface)
+ip_address_changed(int on_client_conn)
 {
   const or_options_t *options = get_options();
   int server = server_mode(options);
-  int exit_reject_interfaces = (server && options->ExitRelay
-                                && options->ExitPolicyRejectLocalInterfaces);
 
-  if (at_interface) {
+  if (on_client_conn) {
     if (! server) {
       /* Okay, change our keys. */
       if (init_keys_client() < 0)
@@ -2683,15 +2301,14 @@ ip_address_changed(int at_interface)
         reset_bandwidth_test();
       reset_uptime();
       router_reset_reachability();
+      /* All relays include their IP addresses as their ORPort addresses in
+       * their descriptor.
+       * Exit relays also incorporate interface addresses in their exit
+       * policies, when ExitPolicyRejectLocalInterfaces is set. */
+      mark_my_descriptor_dirty("IP address changed");
     }
   }
 
-  /* Exit relays incorporate interface addresses in their exit policies when
-   * ExitPolicyRejectLocalInterfaces is set */
-  if (exit_reject_interfaces || (server && !at_interface)) {
-    mark_my_descriptor_dirty("IP address changed");
-  }
-
   dns_servers_relaunch_checks();
 }
 
@@ -2702,8 +2319,7 @@ dns_servers_relaunch_checks(void)
 {
   if (server_mode(get_options())) {
     dns_reset_correctness_checks();
-    if (periodic_events_initialized) {
-      tor_assert(check_dns_honesty_event);
+    if (check_dns_honesty_event) {
       periodic_event_reschedule(check_dns_honesty_event);
     }
   }
@@ -2713,8 +2329,6 @@ dns_servers_relaunch_checks(void)
 void
 initialize_mainloop_events(void)
 {
-  initialize_periodic_events();
-
   if (!schedule_active_linked_connections_event) {
     schedule_active_linked_connections_event =
       mainloop_event_postloop_new(schedule_active_linked_connections_cb, NULL);
@@ -2732,11 +2346,16 @@ do_main_loop(void)
   /* initialize the periodic events first, so that code that depends on the
    * events being present does not assert.
    */
-  initialize_periodic_events();
+  tor_assert(periodic_events_initialized);
   initialize_mainloop_events();
 
-  /* set up once-a-second callback. */
-  reschedule_per_second_timer();
+  periodic_events_connect_all();
+
+  struct timeval one_second = { 1, 0 };
+  initialize_periodic_events_event = tor_evtimer_new(
+                  tor_libevent_get_base(),
+                  initialize_periodic_events_cb, NULL);
+  event_add(initialize_periodic_events_event, &one_second);
 
 #ifdef HAVE_SYSTEMD_209
   uint64_t watchdog_delay;
@@ -2759,10 +2378,6 @@ do_main_loop(void)
     }
   }
 #endif /* defined(HAVE_SYSTEMD_209) */
-
-  main_loop_should_exit = 0;
-  main_loop_exit_value = 0;
-
 #ifdef ENABLE_RESTART_DEBUGGING
   {
     static int first_time = 1;
@@ -2786,7 +2401,7 @@ do_main_loop(void)
       event_add(tor_shutdown_event_loop_for_restart_event, &restart_after);
     }
   }
-#endif
+#endif /* defined(ENABLE_RESTART_DEBUGGING) */
 
   return run_main_loop_until_done();
 }
@@ -2888,10 +2503,14 @@ run_main_loop_once(void)
  *
  * Shadow won't invoke this function, so don't fill it up with things.
  */
-static int
+STATIC int
 run_main_loop_until_done(void)
 {
   int loop_result = 1;
+
+  main_loop_should_exit = 0;
+  main_loop_exit_value = 0;
+
   do {
     loop_result = run_main_loop_once();
   } while (loop_result == 1);
@@ -2922,7 +2541,6 @@ tor_mainloop_free_all(void)
   smartlist_free(connection_array);
   smartlist_free(closeable_connection_lst);
   smartlist_free(active_linked_connection_lst);
-  periodic_timer_free(second_timer);
   teardown_periodic_events();
   tor_event_free(shutdown_did_not_work_event);
   tor_event_free(initialize_periodic_events_event);
@@ -2930,6 +2548,8 @@ tor_mainloop_free_all(void)
   mainloop_event_free(schedule_active_linked_connections_event);
   mainloop_event_free(postloop_cleanup_ev);
   mainloop_event_free(handle_deferred_signewnym_ev);
+  mainloop_event_free(scheduled_shutdown_ev);
+  mainloop_event_free(rescan_periodic_events_ev);
 
 #ifdef HAVE_SYSTEMD_209
   periodic_timer_free(systemd_watchdog_timer);
@@ -2949,7 +2569,6 @@ tor_mainloop_free_all(void)
   can_complete_circuits = 0;
   quiet_level = 0;
   should_init_bridge_stats = 1;
-  dns_honesty_first_time = 1;
   heartbeat_callback_first_time = 1;
   current_second = 0;
   memset(&current_second_last_changed, 0,
diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h
index c5669fc4e0..1ddfec2162 100644
--- a/src/core/mainloop/mainloop.h
+++ b/src/core/mainloop/mainloop.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,15 +56,14 @@ MOCK_DECL(int, connection_count_moribund, (void));
 void directory_all_unreachable(time_t now);
 void directory_info_has_arrived(time_t now, int from_cache, int suppress_logs);
 
-void ip_address_changed(int at_interface);
+void ip_address_changed(int on_client_conn);
 void dns_servers_relaunch_checks(void);
 void reset_all_main_loop_timers(void);
-void reschedule_descriptor_update_check(void);
 void reschedule_directory_downloads(void);
 void reschedule_or_state_save(void);
-void reschedule_dirvote(const or_options_t *options);
 void mainloop_schedule_postloop_cleanup(void);
 void rescan_periodic_events(const or_options_t *options);
+MOCK_DECL(void, schedule_rescan_periodic_events,(void));
 
 void update_current_time(time_t now);
 
@@ -81,34 +80,38 @@ uint64_t get_main_loop_error_count(void);
 uint64_t get_main_loop_idle_count(void);
 
 void periodic_events_on_new_options(const or_options_t *options);
-void reschedule_per_second_timer(void);
 
 void do_signewnym(time_t);
 time_t get_last_signewnym_time(void);
 
+void mainloop_schedule_shutdown(int delay_sec);
+
 void tor_init_connection_lists(void);
 void initialize_mainloop_events(void);
+void initialize_periodic_events(void);
 void tor_mainloop_free_all(void);
 
 struct token_bucket_rw_t;
 
 extern time_t time_of_process_start;
-extern int quiet_level;
 extern struct token_bucket_rw_t global_bucket;
 extern struct token_bucket_rw_t global_relayed_bucket;
 
 #ifdef MAINLOOP_PRIVATE
+STATIC int run_main_loop_until_done(void);
 STATIC void close_closeable_connections(void);
-STATIC void initialize_periodic_events(void);
 STATIC void teardown_periodic_events(void);
 STATIC int get_my_roles(const or_options_t *);
+STATIC int check_network_participation_callback(time_t now,
+                                                const or_options_t *options);
+
 #ifdef TOR_UNIT_TESTS
 extern smartlist_t *connection_array;
 
 /* We need the periodic_event_item_t definition. */
 #include "core/mainloop/periodic.h"
-extern periodic_event_item_t periodic_events[];
-#endif
-#endif /* defined(MAIN_PRIVATE) */
+extern periodic_event_item_t mainloop_periodic_events[];
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* defined(MAINLOOP_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_MAINLOOP_H) */
diff --git a/src/core/mainloop/mainloop_pubsub.c b/src/core/mainloop/mainloop_pubsub.c
new file mode 100644
index 0000000000..0e982d4c40
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.c
@@ -0,0 +1,179 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_pubsub.c
+ * @brief Connect the publish-subscribe code to the main-loop.
+ *
+ * This module is responsible for instantiating all the channels used by the
+ * publish-subscribe code, and making sure that each one's messages are
+ * processed when appropriate.
+ **/
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_pubsub.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/pubsub/pubsub_build.h"
+
+/**
+ * Dispatcher to use for delivering messages.
+ **/
+static dispatch_t *the_dispatcher = NULL;
+static pubsub_items_t *the_pubsub_items = NULL;
+/**
+ * A list of mainloop_event_t, indexed by channel ID, to flush the messages
+ * on a channel.
+ **/
+static smartlist_t *alert_events = NULL;
+
+/**
+ * Mainloop event callback: flush all the messages in a channel.
+ *
+ * The channel is encoded as a pointer, and passed via arg.
+ **/
+static void
+flush_channel_event(mainloop_event_t *ev, void *arg)
+{
+  (void)ev;
+  if (!the_dispatcher)
+    return;
+
+  channel_id_t chan = (channel_id_t)(uintptr_t)(arg);
+  dispatch_flush(the_dispatcher, chan, INT_MAX);
+}
+
+/**
+ * Construct our global pubsub object from <b>builder</b>. Return 0 on
+ * success, -1 on failure. */
+int
+tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder)
+{
+  int rv = -1;
+  tor_mainloop_disconnect_pubsub();
+
+  the_dispatcher = pubsub_builder_finalize(builder, &the_pubsub_items);
+  if (! the_dispatcher)
+    goto err;
+
+  rv = 0;
+  goto done;
+ err:
+  tor_mainloop_disconnect_pubsub();
+ done:
+  return rv;
+}
+
+/**
+ * Install libevent events for all of the pubsub channels.
+ *
+ * Invoke this after tor_mainloop_connect_pubsub, and after libevent has been
+ * initialized.
+ */
+void
+tor_mainloop_connect_pubsub_events(void)
+{
+  tor_assert(the_dispatcher);
+  tor_assert(! alert_events);
+
+  const size_t num_channels = get_num_channel_ids();
+  alert_events = smartlist_new();
+  for (size_t i = 0; i < num_channels; ++i) {
+    smartlist_add(alert_events,
+                  mainloop_event_postloop_new(flush_channel_event,
+                                              (void*)(uintptr_t)(i)));
+  }
+}
+
+/**
+ * Dispatch alertfn callback: do nothing. Implements DELIV_NEVER.
+ **/
+static void
+alertfn_never(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  (void)arg;
+}
+
+/**
+ * Dispatch alertfn callback: activate a mainloop event. Implements
+ * DELIV_PROMPT.
+ **/
+static void
+alertfn_prompt(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void)d;
+  (void)chan;
+  mainloop_event_t *event = arg;
+  mainloop_event_activate(event);
+}
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Set the strategy to be used for delivering messages on the named channel.
+ *
+ * This function needs to be called once globally for each channel, to
+ * set up how messages are delivered.
+ **/
+int
+tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                   deliv_strategy_t strategy)
+{
+  channel_id_t chan = get_channel_id(msg_channel_name);
+  if (BUG(chan == ERROR_ID) ||
+      BUG(chan >= smartlist_len(alert_events)))
+    return -1;
+
+  switch (strategy) {
+    case DELIV_NEVER:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_never, NULL);
+      break;
+    case DELIV_PROMPT:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_prompt,
+                            smartlist_get(alert_events, chan));
+      break;
+    case DELIV_IMMEDIATE:
+      dispatch_set_alert_fn(the_dispatcher, chan, alertfn_immediate, NULL);
+      break;
+  }
+  return 0;
+}
+
+/**
+ * Remove all pubsub dispatchers and events from the mainloop.
+ **/
+void
+tor_mainloop_disconnect_pubsub(void)
+{
+  if (the_pubsub_items) {
+    pubsub_items_clear_bindings(the_pubsub_items);
+    pubsub_items_free(the_pubsub_items);
+  }
+  if (alert_events) {
+    SMARTLIST_FOREACH(alert_events, mainloop_event_t *, ev,
+                      mainloop_event_free(ev));
+    smartlist_free(alert_events);
+  }
+  dispatch_free(the_dispatcher);
+}
diff --git a/src/core/mainloop/mainloop_pubsub.h b/src/core/mainloop/mainloop_pubsub.h
new file mode 100644
index 0000000000..3698fd8d03
--- /dev/null
+++ b/src/core/mainloop/mainloop_pubsub.h
@@ -0,0 +1,61 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_pubsub.h
+ * @brief Header for mainloop_pubsub.c
+ **/
+
+#ifndef TOR_MAINLOOP_PUBSUB_H
+#define TOR_MAINLOOP_PUBSUB_H
+
+struct pubsub_builder_t;
+
+/**
+ * Describe when and how messages are delivered on message channel.
+ *
+ * Every message channel must be associated with one of these strategies.
+ **/
+typedef enum {
+   /**
+    * Never deliver messages automatically.
+    *
+    * If a message channel uses this strategy, then no matter now many
+    * messages are published on it, they are not delivered until something
+    * manually calls dispatch_flush() for that channel
+    **/
+   DELIV_NEVER=0,
+   /**
+    * Deliver messages promptly, via the event loop.
+    *
+    * If a message channel uses this strategy, then publishing a messages
+    * that channel activates an event that causes messages to be handled
+    * later in the mainloop.  The messages will be processed at some point
+    * very soon, delaying only for pending IO events and the like.
+    *
+    * Generally this is the best choice for a delivery strategy, since
+    * it avoids stack explosion.
+    **/
+   DELIV_PROMPT,
+   /**
+    * Deliver messages immediately, skipping the event loop.
+    *
+    * Every event on this channel is flushed immediately after it is queued,
+    * using the stack.
+    *
+    * This delivery type should be used with caution, since it can cause
+    * unexpected call chains, resource starvation, and the like.
+    **/
+   DELIV_IMMEDIATE,
+} deliv_strategy_t;
+
+int tor_mainloop_connect_pubsub(struct pubsub_builder_t *builder);
+void tor_mainloop_connect_pubsub_events(void);
+int tor_mainloop_set_delivery_strategy(const char *msg_channel_name,
+                                        deliv_strategy_t strategy);
+void tor_mainloop_disconnect_pubsub(void);
+
+#endif /* !defined(TOR_MAINLOOP_PUBSUB_H) */
diff --git a/src/core/mainloop/mainloop_state.inc b/src/core/mainloop/mainloop_state.inc
new file mode 100644
index 0000000000..34a37caaa2
--- /dev/null
+++ b/src/core/mainloop/mainloop_state.inc
@@ -0,0 +1,19 @@
+
+/**
+ * @file mainloop_state.inc
+ * @brief Declare configuration options for the crypto_ops module.
+ **/
+
+/** Holds state for the mainloop, corresponding to part of the state
+ * file in Tor's DataDirectory. */
+BEGIN_CONF_STRUCT(mainloop_state_t)
+
+/** Number of minutes since the last user-initiated request (as defined by
+ * the dormant net-status system.) Set to zero if we are dormant. */
+CONF_VAR(MinutesSinceUserActivity, POSINT, 0, NULL)
+
+/** True if we were dormant when we last wrote the file; false if we
+ * weren't.  "auto" on initial startup. */
+CONF_VAR(Dormant, AUTOBOOL, 0, "auto")
+
+END_CONF_STRUCT(mainloop_state_t)
diff --git a/src/core/mainloop/mainloop_state_st.h b/src/core/mainloop/mainloop_state_st.h
new file mode 100644
index 0000000000..5649b536f9
--- /dev/null
+++ b/src/core/mainloop/mainloop_state_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_state_st.h
+ * @brief Declare a state structure for mainloop-relevant fields
+ **/
+
+#ifndef TOR_CORE_MAINLOOP_MAINLOOP_STATE_ST_H
+#define TOR_CORE_MAINLOOP_MAINLOOP_STATE_ST_H
+
+#include "lib/conf/confdecl.h"
+
+#define CONF_CONTEXT STRUCT
+#include "core/mainloop/mainloop_state.inc"
+#undef CONF_CONTEXT
+
+typedef struct mainloop_state_t mainloop_state_t;
+
+#endif /* !defined(TOR_CORE_MAINLOOP_MAINLOOP_STATE_ST_H) */
diff --git a/src/core/mainloop/mainloop_sys.c b/src/core/mainloop/mainloop_sys.c
new file mode 100644
index 0000000000..884bae1c59
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_sys.c
+ * @brief Declare the "mainloop" subsystem.
+ **/
+
+#include "core/or/or.h"
+#include "core/mainloop/mainloop_sys.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_state_st.h"
+#include "core/mainloop/netstatus.h"
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confdecl.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_mainloop_initialize(void)
+{
+  initialize_periodic_events();
+  return 0;
+}
+
+static void
+subsys_mainloop_shutdown(void)
+{
+  tor_mainloop_free_all();
+}
+
+/** Declare a list of state variables for mainloop state. */
+#define CONF_CONTEXT TABLE
+#include "core/mainloop/mainloop_state.inc"
+#undef CONF_CONTEXT
+
+/** Magic number for mainloop state objects */
+#define MAINLOOP_STATE_MAGIC 0x59455449
+
+/**
+ * Format object for mainloop state.
+ **/
+static config_format_t mainloop_state_fmt = {
+  .size = sizeof(mainloop_state_t),
+  .magic = { "mainloop_state",
+             MAINLOOP_STATE_MAGIC,
+             offsetof(mainloop_state_t, magic)
+            },
+  .vars = mainloop_state_t_vars,
+};
+
+/**
+ */
+static int
+mainloop_set_state(void *arg)
+{
+  const mainloop_state_t *state = arg;
+  tor_assert(state->magic == MAINLOOP_STATE_MAGIC);
+
+  netstatus_load_from_state(state, approx_time());
+
+  return 0;
+}
+
+static int
+mainloop_flush_state(void *arg)
+{
+  mainloop_state_t *state = arg;
+  tor_assert(state->magic == MAINLOOP_STATE_MAGIC);
+
+  netstatus_flush_to_state(state, approx_time());
+
+  return 0;
+}
+
+const struct subsys_fns_t sys_mainloop = {
+  .name = "mainloop",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = 5,
+  .initialize = subsys_mainloop_initialize,
+  .shutdown = subsys_mainloop_shutdown,
+
+  .state_format = &mainloop_state_fmt,
+  .set_state = mainloop_set_state,
+  .flush_state = mainloop_flush_state,
+};
diff --git a/src/core/mainloop/mainloop_sys.h b/src/core/mainloop/mainloop_sys.h
new file mode 100644
index 0000000000..b3ade33cd1
--- /dev/null
+++ b/src/core/mainloop/mainloop_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file mainloop_sys.h
+ * @brief Header for mainloop_sys.c
+ **/
+
+#ifndef MAINLOOP_SYS_H
+#define MAINLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_mainloop;
+
+#endif /* !defined(MAINLOOP_SYS_H) */
diff --git a/src/core/mainloop/netstatus.c b/src/core/mainloop/netstatus.c
index 1444ca5db2..61a3469eaa 100644
--- a/src/core/mainloop/netstatus.c
+++ b/src/core/mainloop/netstatus.c
@@ -1,14 +1,23 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file netstatus.c
+ * @brief Track whether the network is disabled, dormant, etc.
+ **/
+
 #include "core/or/or.h"
 #include "core/mainloop/netstatus.h"
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_state_st.h"
 #include "app/config/config.h"
 #include "feature/hibernate/hibernate.h"
 
+#include "app/config/or_state_st.h"
+
 /** Return true iff our network is in some sense disabled or shutting down:
  * either we're hibernating, entering hibernation, or the network is turned
  * off with DisableNetwork. */
@@ -26,3 +35,136 @@ net_is_completely_disabled(void)
 {
   return get_options()->DisableNetwork || we_are_fully_hibernating();
 }
+
+/**
+ * The time at which we've last seen "user activity" -- that is, any activity
+ * that should keep us as a participant on the network.
+ *
+ * This is not actually the true time.  We will adjust this forward if
+ * our clock jumps, or if Tor is shut down for a while, so that the time
+ * since our last activity remains as it was before the jump or shutdown.
+ */
+static time_t last_user_activity_seen = 0;
+
+/**
+ * True iff we are currently a "network participant" -- that is, we
+ * are building circuits, fetching directory information, and so on.
+ **/
+static bool participating_on_network = false;
+
+/**
+ * Record the fact that we have seen "user activity" at the time now.  Move
+ * "last activity seen" time forwards, but never backwards.
+ *
+ * If we were previously not participating on the network, set our
+ * participation status to true, and launch periodic events as appropriate.
+ **/
+void
+note_user_activity(time_t now)
+{
+  last_user_activity_seen = MAX(now, last_user_activity_seen);
+
+  if (! participating_on_network) {
+    log_notice(LD_GENERAL, "Tor is no longer dormant.");
+    set_network_participation(true);
+    schedule_rescan_periodic_events();
+  }
+}
+
+/**
+ * Change the time at which "user activitiy" was last seen to <b>now</b>.
+ *
+ * Unlike note_user_actity, this function sets the time without checking
+ * whether it is in the past, and without causing any rescan of periodic events
+ * or change in participation status.
+ */
+void
+reset_user_activity(time_t now)
+{
+  last_user_activity_seen = now;
+}
+
+/**
+ * Return the most recent time at which we recorded "user activity".
+ **/
+time_t
+get_last_user_activity_time(void)
+{
+  return last_user_activity_seen;
+}
+
+/**
+ * Set the field that remembers whether we are currently participating on the
+ * network.  Does not schedule or un-schedule periodic events.
+ **/
+void
+set_network_participation(bool participation)
+{
+  participating_on_network = participation;
+}
+
+/**
+ * Return true iff we are currently participating on the network.
+ **/
+bool
+is_participating_on_network(void)
+{
+  return participating_on_network;
+}
+
+/**
+ * Update 'state' with the last time at which we were active on the network.
+ **/
+void
+netstatus_flush_to_state(mainloop_state_t *state, time_t now)
+{
+  state->Dormant = ! participating_on_network;
+  if (participating_on_network) {
+    time_t sec_since_activity = MAX(0, now - last_user_activity_seen);
+    state->MinutesSinceUserActivity = (int)(sec_since_activity / 60);
+  } else {
+    state->MinutesSinceUserActivity = 0;
+  }
+}
+
+/**
+ * Update our current view of network participation from an or_state_t object.
+ **/
+void
+netstatus_load_from_state(const mainloop_state_t *state, time_t now)
+{
+  time_t last_activity;
+  if (state->Dormant == -1) { // Initial setup.
+    if (get_options()->DormantOnFirstStartup) {
+      last_activity = 0;
+      participating_on_network = false;
+    } else {
+      // Start up as active, treat activity as happening now.
+      last_activity = now;
+      participating_on_network = true;
+    }
+  } else if (state->Dormant) {
+    last_activity = 0;
+    participating_on_network = false;
+  } else {
+    last_activity = now - 60 * state->MinutesSinceUserActivity;
+    participating_on_network = true;
+  }
+  if (get_options()->DormantCanceledByStartup) {
+    last_activity = now;
+    participating_on_network = true;
+  }
+  reset_user_activity(last_activity);
+}
+
+/**
+ * Adjust the time at which the user was last active by <b>seconds_diff</b>
+ * in response to a clock jump.
+ */
+void
+netstatus_note_clock_jumped(time_t seconds_diff)
+{
+  time_t last_active = get_last_user_activity_time();
+  if (last_active)
+    reset_user_activity(last_active + seconds_diff);
+}
diff --git a/src/core/mainloop/netstatus.h b/src/core/mainloop/netstatus.h
index ae8547b30d..5f54e54553 100644
--- a/src/core/mainloop/netstatus.h
+++ b/src/core/mainloop/netstatus.h
@@ -1,13 +1,32 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file netstatus.h
+ * @brief Header for netstatus.c
+ **/
+
 #ifndef TOR_NETSTATUS_H
 #define TOR_NETSTATUS_H
 
 int net_is_disabled(void);
 int net_is_completely_disabled(void);
 
-#endif
+void note_user_activity(time_t now);
+void reset_user_activity(time_t now);
+time_t get_last_user_activity_time(void);
+
+void set_network_participation(bool participation);
+bool is_participating_on_network(void);
+
+struct mainloop_state_t;
+
+void netstatus_flush_to_state(struct mainloop_state_t *state, time_t now);
+void netstatus_load_from_state(const struct mainloop_state_t *state,
+                               time_t now);
+void netstatus_note_clock_jumped(time_t seconds_diff);
+
+#endif /* !defined(TOR_NETSTATUS_H) */
diff --git a/src/core/mainloop/periodic.c b/src/core/mainloop/periodic.c
index 2651bbbc89..b5fd8fab61 100644
--- a/src/core/mainloop/periodic.c
+++ b/src/core/mainloop/periodic.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,9 +6,22 @@
  *
  * \brief Generic backend for handling periodic events.
  *
- * The events in this module are used by main.c to track items that need
+ * The events in this module are used to track items that need
  * to fire once every N seconds, possibly picking a new interval each time
- * that they fire.  See periodic_events[] in main.c for examples.
+ * that they fire.  See periodic_events[] in mainloop.c for examples.
+ *
+ * This module manages a global list of periodic_event_item_t objects,
+ * each corresponding to a single event.  To register an event, pass it to
+ * periodic_events_register() when initializing your subsystem.
+ *
+ * Registering an event makes the periodic event subsystem know about it, but
+ * doesn't cause the event to get created immediately.  Before the event can
+ * be started, periodic_event_connect_all() must be called by mainloop.c to
+ * connect all the events to Libevent.
+ *
+ * We expect that periodic_event_item_t objects will be statically allocated;
+ * we set them up and tear them down here, but we don't take ownership of
+ * them.
  */
 
 #include "core/or/or.h"
@@ -16,7 +29,6 @@
 #include "app/config/config.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/periodic.h"
-#include "lib/evloop/compat_libevent.h"
 
 /** We disable any interval greater than this number of seconds, on the
  * grounds that it is probably an absolute time mistakenly passed in as a
@@ -24,6 +36,12 @@
  */
 static const int MAX_INTERVAL = 10 * 365 * 86400;
 
+/**
+ * Global list of periodic events that have been registered with
+ * <b>periodic_event_register</b>.
+ **/
+static smartlist_t *the_periodic_events = NULL;
+
 /** Set the event <b>event</b> to run in <b>next_interval</b> seconds from
  * now. */
 static void
@@ -45,10 +63,6 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   periodic_event_item_t *event = data;
   tor_assert(ev == event->ev);
 
-  if (BUG(!periodic_event_is_enabled(event))) {
-    return;
-  }
-
   time_t now = time(NULL);
   update_current_time(now);
   const or_options_t *options = get_options();
@@ -57,7 +71,7 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
   int next_interval = 0;
 
   if (!periodic_event_is_enabled(event)) {
-    /* The event got disabled from inside its callback; no need to
+    /* The event got disabled from inside its callback, or before: no need to
      * reschedule. */
     return;
   }
@@ -91,15 +105,16 @@ periodic_event_dispatch(mainloop_event_t *ev, void *data)
 void
 periodic_event_reschedule(periodic_event_item_t *event)
 {
-  /* Don't reschedule a disabled event. */
-  if (periodic_event_is_enabled(event)) {
+  /* Don't reschedule a disabled or uninitialized event. */
+  if (event->ev && periodic_event_is_enabled(event)) {
     periodic_event_set_interval(event, 1);
   }
 }
 
-/** Initializes the libevent backend for a periodic event. */
+/** Connects a periodic event to the Libevent backend.  Does not launch the
+ * event immediately. */
 void
-periodic_event_setup(periodic_event_item_t *event)
+periodic_event_connect(periodic_event_item_t *event)
 {
   if (event->ev) { /* Already setup? This is a bug */
     log_err(LD_BUG, "Initial dispatch should only be done once.");
@@ -117,7 +132,7 @@ void
 periodic_event_launch(periodic_event_item_t *event)
 {
   if (! event->ev) { /* Not setup? This is a bug */
-    log_err(LD_BUG, "periodic_event_launch without periodic_event_setup");
+    log_err(LD_BUG, "periodic_event_launch without periodic_event_connect");
     tor_assert(0);
   }
   /* Event already enabled? This is a bug */
@@ -131,9 +146,9 @@ periodic_event_launch(periodic_event_item_t *event)
   periodic_event_dispatch(event->ev, event);
 }
 
-/** Release all storage associated with <b>event</b> */
-void
-periodic_event_destroy(periodic_event_item_t *event)
+/** Disconnect and unregister the periodic event in <b>event</b> */
+static void
+periodic_event_disconnect(periodic_event_item_t *event)
 {
   if (!event)
     return;
@@ -177,3 +192,177 @@ periodic_event_disable(periodic_event_item_t *event)
   mainloop_event_cancel(event->ev);
   event->enabled = 0;
 }
+
+/**
+ * Disable an event, then schedule it to run once.
+ * Do nothing if the event was already disabled.
+ */
+void
+periodic_event_schedule_and_disable(periodic_event_item_t *event)
+{
+  tor_assert(event);
+  if (!periodic_event_is_enabled(event))
+    return;
+
+  periodic_event_disable(event);
+
+  mainloop_event_activate(event->ev);
+}
+
+/**
+ * Add <b>item</b> to the list of periodic events.
+ *
+ * Note that <b>item</b> should be statically allocated: we do not
+ * take ownership of it.
+ **/
+void
+periodic_events_register(periodic_event_item_t *item)
+{
+  if (!the_periodic_events)
+    the_periodic_events = smartlist_new();
+
+  if (BUG(smartlist_contains(the_periodic_events, item)))
+    return;
+
+  smartlist_add(the_periodic_events, item);
+}
+
+/**
+ * Make all registered periodic events connect to the libevent backend.
+ */
+void
+periodic_events_connect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (item->ev)
+      continue;
+    periodic_event_connect(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Reset all the registered periodic events so we'll do all our actions again
+ * as if we just started up.
+ *
+ * Useful if our clock just moved back a long time from the future,
+ * so we don't wait until that future arrives again before acting.
+ */
+void
+periodic_events_reset_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    periodic_event_reschedule(item);
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Return the registered periodic event whose name is <b>name</b>.
+ * Return NULL if no such event is found.
+ */
+periodic_event_item_t *
+periodic_events_find(const char *name)
+{
+  if (! the_periodic_events)
+    return NULL;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (strcmp(name, item->name) == 0)
+      return item;
+  } SMARTLIST_FOREACH_END(item);
+  return NULL;
+}
+
+/**
+ * Start or stop registered periodic events, depending on our current set of
+ * roles.
+ *
+ * Invoked when our list of roles, or the net_disabled flag has changed.
+ **/
+void
+periodic_events_rescan_by_roles(int roles, bool net_disabled)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    if (!item->ev)
+      continue;
+
+    int enable = !!(item->roles & roles);
+
+    /* Handle the event flags. */
+    if (net_disabled &&
+        (item->flags & PERIODIC_EVENT_FLAG_NEED_NET)) {
+      enable = 0;
+    }
+
+    /* Enable the event if needed. It is safe to enable an event that was
+     * already enabled. Same goes for disabling it. */
+    if (enable) {
+      log_debug(LD_GENERAL, "Launching periodic event %s", item->name);
+      periodic_event_enable(item);
+    } else {
+      log_debug(LD_GENERAL, "Disabling periodic event %s", item->name);
+      if (item->flags & PERIODIC_EVENT_FLAG_RUN_ON_DISABLE) {
+        periodic_event_schedule_and_disable(item);
+      } else {
+        periodic_event_disable(item);
+      }
+    }
+  } SMARTLIST_FOREACH_END(item);
+}
+
+/**
+ * Invoked at shutdown: disconnect and unregister all periodic events.
+ *
+ * Does not free the periodic_event_item_t object themselves, because we do
+ * not own them.
+ */
+void
+periodic_events_disconnect_all(void)
+{
+  if (! the_periodic_events)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(the_periodic_events, periodic_event_item_t *, item) {
+    periodic_event_disconnect(item);
+  } SMARTLIST_FOREACH_END(item);
+
+  smartlist_free(the_periodic_events);
+}
+
+#define LONGEST_TIMER_PERIOD (30 * 86400)
+/** Helper: Return the number of seconds between <b>now</b> and <b>next</b>,
+ * clipped to the range [1 second, LONGEST_TIMER_PERIOD].
+ *
+ * We use this to answer the question, "how many seconds is it from now until
+ * next" in periodic timer callbacks.  Don't use it for other purposes
+ **/
+int
+safe_timer_diff(time_t now, time_t next)
+{
+  if (next > now) {
+    /* There were no computers at signed TIME_MIN (1902 on 32-bit systems),
+     * and nothing that could run Tor. It's a bug if 'next' is around then.
+     * On 64-bit systems with signed TIME_MIN, TIME_MIN is before the Big
+     * Bang. We cannot extrapolate past a singularity, but there was probably
+     * nothing that could run Tor then, either.
+     **/
+    tor_assert(next > TIME_MIN + LONGEST_TIMER_PERIOD);
+
+    if (next - LONGEST_TIMER_PERIOD > now)
+      return LONGEST_TIMER_PERIOD;
+    return (int)(next - now);
+  } else {
+    return 1;
+  }
+}
diff --git a/src/core/mainloop/periodic.h b/src/core/mainloop/periodic.h
index e49fafd174..de556a6bdb 100644
--- a/src/core/mainloop/periodic.h
+++ b/src/core/mainloop/periodic.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file periodic.h
+ * @brief Header for periodic.c
+ **/
+
 #ifndef TOR_PERIODIC_H
 #define TOR_PERIODIC_H
 
@@ -15,6 +20,10 @@
 #define PERIODIC_EVENT_ROLE_BRIDGEAUTH  (1U << 4)
 #define PERIODIC_EVENT_ROLE_HS_SERVICE  (1U << 5)
 #define PERIODIC_EVENT_ROLE_DIRSERVER   (1U << 6)
+#define PERIODIC_EVENT_ROLE_CONTROLEV   (1U << 7)
+
+#define PERIODIC_EVENT_ROLE_NET_PARTICIPANT (1U << 8)
+#define PERIODIC_EVENT_ROLE_ALL         (1U << 9)
 
 /* Helper macro to make it a bit less annoying to defined groups of roles that
  * are often used. */
@@ -25,10 +34,6 @@
 /* Authorities that is both bridge and directory. */
 #define PERIODIC_EVENT_ROLE_AUTHORITIES \
   (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_DIRAUTH)
-/* All roles. */
-#define PERIODIC_EVENT_ROLE_ALL \
-  (PERIODIC_EVENT_ROLE_AUTHORITIES | PERIODIC_EVENT_ROLE_CLIENT | \
-   PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_ROUTER)
 
 /*
  * Event flags which can change the behavior of an event.
@@ -39,6 +44,11 @@
  * the net_is_disabled() check. */
 #define PERIODIC_EVENT_FLAG_NEED_NET  (1U << 0)
 
+/* Indicate that if the event is enabled, it needs to be run once before
+ * it becomes disabled.
+ */
+#define PERIODIC_EVENT_FLAG_RUN_ON_DISABLE  (1U << 1)
+
 /** Callback function for a periodic event to take action.  The return value
 * influences the next time the function will get called.  Return
 * PERIODIC_EVENT_NO_UPDATE to not update <b>last_action_time</b> and be polled
@@ -66,8 +76,10 @@ typedef struct periodic_event_item_t {
 } periodic_event_item_t;
 
 /** events will get their interval from first execution */
+#ifndef COCCI
 #define PERIODIC_EVENT(fn, r, f) { fn##_callback, 0, NULL, #fn, r, f, 0 }
 #define END_OF_PERIODIC_EVENTS { NULL, 0, NULL, NULL, 0, 0, 0 }
+#endif
 
 /* Return true iff the given event was setup before thus is enabled to be
  * scheduled. */
@@ -78,11 +90,20 @@ periodic_event_is_enabled(const periodic_event_item_t *item)
 }
 
 void periodic_event_launch(periodic_event_item_t *event);
-void periodic_event_setup(periodic_event_item_t *event);
-void periodic_event_destroy(periodic_event_item_t *event);
+void periodic_event_connect(periodic_event_item_t *event);
+//void periodic_event_disconnect(periodic_event_item_t *event);
 void periodic_event_reschedule(periodic_event_item_t *event);
 void periodic_event_enable(periodic_event_item_t *event);
 void periodic_event_disable(periodic_event_item_t *event);
+void periodic_event_schedule_and_disable(periodic_event_item_t *event);
 
-#endif /* !defined(TOR_PERIODIC_H) */
+void periodic_events_register(periodic_event_item_t *item);
+void periodic_events_connect_all(void);
+void periodic_events_reset_all(void);
+periodic_event_item_t *periodic_events_find(const char *name);
+void periodic_events_rescan_by_roles(int roles, bool net_disabled);
+void periodic_events_disconnect_all(void);
+
+int safe_timer_diff(time_t now, time_t next);
 
+#endif /* !defined(TOR_PERIODIC_H) */
diff --git a/src/core/or/.may_include b/src/core/or/.may_include
new file mode 100644
index 0000000000..beb12f155d
--- /dev/null
+++ b/src/core/or/.may_include
@@ -0,0 +1,40 @@
+!advisory
+
+orconfig.h
+
+lib/arch/*.h
+lib/buf/*.h
+lib/cc/*.h
+lib/compress/*.h
+lib/container/*.h
+lib/crypt_ops/*.h
+lib/ctime/*.h
+lib/defs/*.h
+lib/encoding/*.h
+lib/err/*.h
+lib/evloop/*.h
+lib/fs/*.h
+lib/geoip/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/math/*.h
+lib/net/*.h
+lib/pubsub/*.h
+lib/string/*.h
+lib/subsys/*.h
+lib/test/*.h
+lib/testsupport/*.h
+lib/thread/*.h
+lib/time/*.h
+lib/tls/*.h
+lib/wallclock/*.h
+
+trunnel/*.h
+
+core/mainloop/*.h
+core/proto/*.h
+core/crypto/*.h
+core/or/*.h
+
+ext/*.h
diff --git a/src/core/or/addr_policy_st.h b/src/core/or/addr_policy_st.h
index a75f1a731d..08d16ee616 100644
--- a/src/core/or/addr_policy_st.h
+++ b/src/core/or/addr_policy_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file addr_policy_st.h
+ * @brief Address policy structures.
+ **/
+
 #ifndef TOR_ADDR_POLICY_ST_H
 #define TOR_ADDR_POLICY_ST_H
 
@@ -33,7 +38,7 @@ struct addr_policy_t {
   /** Base address to accept or reject.
    *
    * Note that wildcards are treated
-   * differntly depending on address family. An AF_UNSPEC address means
+   * differently depending on address family. An AF_UNSPEC address means
    * "All addresses, IPv4 or IPv6." An AF_INET address with maskbits==0 means
    * "All IPv4 addresses" and an AF_INET6 address with maskbits == 0 means
    * "All IPv6 addresses".
@@ -43,4 +48,4 @@ struct addr_policy_t {
   uint16_t prt_max; /**< Highest port number to accept/reject. */
 };
 
-#endif
+#endif /* !defined(TOR_ADDR_POLICY_ST_H) */
diff --git a/src/core/or/address_set.c b/src/core/or/address_set.c
index 7ada4446c4..9bd3cc0f2d 100644
--- a/src/core/or/address_set.c
+++ b/src/core/or/address_set.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/crypt_ops/crypto_rand.h"
 
-/* Wrap our hash function to have the signature that the bloom filter
+/** Wrap our hash function to have the signature that the bloom filter
  * needs. */
 static uint64_t
 bloomfilt_addr_hash(const struct sipkey *key,
diff --git a/src/core/or/address_set.h b/src/core/or/address_set.h
index a505d31628..b4d94b65a9 100644
--- a/src/core/or/address_set.h
+++ b/src/core/or/address_set.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/or/cell_queue_st.h b/src/core/or/cell_queue_st.h
index 130b95a011..0681dba1b8 100644
--- a/src/core/or/cell_queue_st.h
+++ b/src/core/or/cell_queue_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cell_queue_st.h
+ * @brief Cell queue structures
+ **/
+
 #ifndef PACKED_CELL_ST_H
 #define PACKED_CELL_ST_H
 
@@ -22,8 +27,8 @@ struct packed_cell_t {
  * or_connection_t's outbuf. */
 struct cell_queue_t {
   /** Linked list of packed_cell_t*/
-  TOR_SIMPLEQ_HEAD(cell_simpleq, packed_cell_t) head;
+  TOR_SIMPLEQ_HEAD(cell_simpleq_t, packed_cell_t) head;
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
+#endif /* !defined(PACKED_CELL_ST_H) */
diff --git a/src/core/or/cell_st.h b/src/core/or/cell_st.h
index 7ab7eceb50..a640d6a456 100644
--- a/src/core/or/cell_st.h
+++ b/src/core/or/cell_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cell_st.h
+ * @brief Fixed-size cell structure.
+ **/
+
 #ifndef CELL_ST_H
 #define CELL_ST_H
 
@@ -16,5 +21,4 @@ struct cell_t {
   uint8_t payload[CELL_PAYLOAD_SIZE]; /**< Cell body. */
 };
 
-#endif
-
+#endif /* !defined(CELL_ST_H) */
diff --git a/src/core/or/channel.c b/src/core/or/channel.c
index 9649bdf278..50c03de846 100644
--- a/src/core/or/channel.c
+++ b/src/core/or/channel.c
@@ -1,5 +1,4 @@
-
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -52,10 +51,10 @@
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 
 /* This one's for stuff only channel.c and the test suite should see */
-#define CHANNEL_PRIVATE_
+#define CHANNEL_FILE_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -85,6 +84,13 @@
 
 #include "core/or/cell_queue_st.h"
 
+/* Static function prototypes */
+
+static bool channel_matches_target_addr_for_extend(
+                                          channel_t *chan,
+                                          const tor_addr_t *target_ipv4_addr,
+                                          const tor_addr_t *target_ipv6_addr);
+
 /* Global lists of channels */
 
 /* All channel_t instances */
@@ -107,7 +113,7 @@ static smartlist_t *finished_listeners = NULL;
 
 /** Map from channel->global_identifier to channel.  Contains the same
  * elements as all_channels. */
-static HT_HEAD(channel_gid_map, channel_s) channel_gid_map = HT_INITIALIZER();
+static HT_HEAD(channel_gid_map, channel_t) channel_gid_map = HT_INITIALIZER();
 
 static unsigned
 channel_id_hash(const channel_t *chan)
@@ -119,13 +125,13 @@ channel_id_eq(const channel_t *a, const channel_t *b)
 {
   return a->global_identifier == b->global_identifier;
 }
-HT_PROTOTYPE(channel_gid_map, channel_s, gidmap_node,
-             channel_id_hash, channel_id_eq)
-HT_GENERATE2(channel_gid_map, channel_s, gidmap_node,
+HT_PROTOTYPE(channel_gid_map, channel_t, gidmap_node,
+             channel_id_hash, channel_id_eq);
+HT_GENERATE2(channel_gid_map, channel_t, gidmap_node,
              channel_id_hash, channel_id_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
-HANDLE_IMPL(channel, channel_s,)
+HANDLE_IMPL(channel, channel_t,)
 
 /* Counter for ID numbers */
 static uint64_t n_channels_allocated = 0;
@@ -138,13 +144,13 @@ static uint64_t n_channels_allocated = 0;
  * If more than one channel exists, follow the next_with_same_id pointer
  * as a linked list.
  */
-static HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map =
+static HT_HEAD(channel_idmap, channel_idmap_entry_t) channel_identity_map =
   HT_INITIALIZER();
 
-typedef struct channel_idmap_entry_s {
-  HT_ENTRY(channel_idmap_entry_s) node;
+typedef struct channel_idmap_entry_t {
+  HT_ENTRY(channel_idmap_entry_t) node;
   uint8_t digest[DIGEST_LEN];
-  TOR_LIST_HEAD(channel_list_s, channel_s) channel_list;
+  TOR_LIST_HEAD(channel_list_t, channel_t) channel_list;
 } channel_idmap_entry_t;
 
 static inline unsigned
@@ -160,10 +166,10 @@ channel_idmap_eq(const channel_idmap_entry_t *a,
   return tor_memeq(a->digest, b->digest, DIGEST_LEN);
 }
 
-HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
-             channel_idmap_eq)
-HT_GENERATE2(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash,
-             channel_idmap_eq, 0.5,  tor_reallocarray_, tor_free_)
+HT_PROTOTYPE(channel_idmap, channel_idmap_entry_t, node, channel_idmap_hash,
+             channel_idmap_eq);
+HT_GENERATE2(channel_idmap, channel_idmap_entry_t, node, channel_idmap_hash,
+             channel_idmap_eq, 0.5,  tor_reallocarray_, tor_free_);
 
 /* Functions to maintain the digest map */
 static void channel_remove_from_digest_map(channel_t *chan);
@@ -1092,23 +1098,6 @@ channel_get_cell_handler(channel_t *chan)
 }
 
 /**
- * Return the variable-length cell handler for a channel.
- *
- * This function gets the handler for incoming variable-length cells
- * installed on a channel.
- */
-channel_var_cell_handler_fn_ptr
-channel_get_var_cell_handler(channel_t *chan)
-{
-  tor_assert(chan);
-
-  if (CHANNEL_CAN_HANDLE_CELLS(chan))
-    return chan->var_cell_handler;
-
-  return NULL;
-}
-
-/**
  * Set both cell handlers for a channel.
  *
  * This function sets both the fixed-length and variable length cell handlers
@@ -1116,9 +1105,7 @@ channel_get_var_cell_handler(channel_t *chan)
  */
 void
 channel_set_cell_handlers(channel_t *chan,
-                          channel_cell_handler_fn_ptr cell_handler,
-                          channel_var_cell_handler_fn_ptr
-                            var_cell_handler)
+                          channel_cell_handler_fn_ptr cell_handler)
 {
   tor_assert(chan);
   tor_assert(CHANNEL_CAN_HANDLE_CELLS(chan));
@@ -1126,13 +1113,9 @@ channel_set_cell_handlers(channel_t *chan,
   log_debug(LD_CHANNEL,
            "Setting cell_handler callback for channel %p to %p",
            chan, cell_handler);
-  log_debug(LD_CHANNEL,
-           "Setting var_cell_handler callback for channel %p to %p",
-           chan, var_cell_handler);
 
   /* Change them */
   chan->cell_handler = cell_handler;
-  chan->var_cell_handler = var_cell_handler;
 }
 
 /*
@@ -1443,6 +1426,7 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
 {
   int ret = -1;
   size_t cell_bytes;
+  uint8_t command = packed_cell_get_command(cell, chan->wide_circ_ids);
 
   tor_assert(chan);
   tor_assert(cell);
@@ -1477,6 +1461,16 @@ write_packed_cell(channel_t *chan, packed_cell_t *cell)
   /* Successfully sent the cell. */
   ret = 0;
 
+  /* Update padding statistics for the packed codepath.. */
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
+  if (command == CELL_PADDING)
+    rep_hist_padding_count_write(PADDING_TYPE_CELL);
+  if (chan->padding_enabled) {
+    rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
+    if (command == CELL_PADDING)
+      rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
+  }
+
  done:
   return ret;
 }
@@ -1893,11 +1887,11 @@ channel_do_open_actions(channel_t *chan)
         geoip_note_client_seen(GEOIP_CLIENT_CONNECT,
                                &remote_addr, transport_name,
                                now);
-        tor_free(transport_name);
         /* Notify the DoS subsystem of a new client. */
         if (tlschan && tlschan->conn) {
           dos_new_client_conn(tlschan->conn, transport_name);
         }
+        tor_free(transport_name);
       }
       /* Otherwise the underlying transport can't tell us this, so skip it */
     }
@@ -2372,7 +2366,7 @@ channel_is_better(channel_t *a, channel_t *b)
   if (!a->is_canonical_to_peer && b->is_canonical_to_peer) return 0;
 
   /*
-   * Okay, if we're here they tied on canonicity, the prefer the older
+   * Okay, if we're here they tied on canonicity. Prefer the older
    * connection, so that the adversary can't create a new connection
    * and try to switch us over to it (which will leak information
    * about long-lived circuits). Additionally, switching connections
@@ -2397,19 +2391,23 @@ channel_is_better(channel_t *a, channel_t *b)
 /**
  * Get a channel to extend a circuit.
  *
- * Pick a suitable channel to extend a circuit to given the desired digest
- * the address we believe is correct for that digest; this tries to see
- * if we already have one for the requested endpoint, but if there is no good
- * channel, set *msg_out to a message describing the channel's state
- * and our next action, and set *launch_out to a boolean indicated whether
- * the caller should try to launch a new channel with channel_connect().
+ * Given the desired relay identity, pick a suitable channel to extend a
+ * circuit to the target IPv4 or IPv6 address requsted by the client. Search
+ * for an existing channel for the requested endpoint. Make sure the channel
+ * is usable for new circuits, and matches one of the target addresses.
+ *
+ * Try to return the best channel. But if there is no good channel, set
+ * *msg_out to a message describing the channel's state and our next action,
+ * and set *launch_out to a boolean indicated whether the caller should try to
+ * launch a new channel with channel_connect().
  */
-channel_t *
-channel_get_for_extend(const char *rsa_id_digest,
-                       const ed25519_public_key_t *ed_id,
-                       const tor_addr_t *target_addr,
-                       const char **msg_out,
-                       int *launch_out)
+MOCK_IMPL(channel_t *,
+channel_get_for_extend,(const char *rsa_id_digest,
+                        const ed25519_public_key_t *ed_id,
+                        const tor_addr_t *target_ipv4_addr,
+                        const tor_addr_t *target_ipv6_addr,
+                        const char **msg_out,
+                        int *launch_out))
 {
   channel_t *chan, *best = NULL;
   int n_inprogress_goodaddr = 0, n_old = 0;
@@ -2420,9 +2418,7 @@ channel_get_for_extend(const char *rsa_id_digest,
 
   chan = channel_find_by_remote_identity(rsa_id_digest, ed_id);
 
-  /* Walk the list, unrefing the old one and refing the new at each
-   * iteration.
-   */
+  /* Walk the list of channels */
   for (; chan; chan = channel_next_with_rsa_identity(chan)) {
     tor_assert(tor_memeq(chan->identity_digest,
                          rsa_id_digest, DIGEST_LEN));
@@ -2441,11 +2437,15 @@ channel_get_for_extend(const char *rsa_id_digest,
       continue;
     }
 
+    const bool matches_target =
+      channel_matches_target_addr_for_extend(chan,
+                                             target_ipv4_addr,
+                                             target_ipv6_addr);
     /* Never return a non-open connection. */
     if (!CHANNEL_IS_OPEN(chan)) {
       /* If the address matches, don't launch a new connection for this
        * circuit. */
-      if (channel_matches_target_addr_for_extend(chan, target_addr))
+      if (matches_target)
         ++n_inprogress_goodaddr;
       continue;
     }
@@ -2458,8 +2458,7 @@ channel_get_for_extend(const char *rsa_id_digest,
 
     /* Only return canonical connections or connections where the address
      * is the address we wanted. */
-    if (!channel_is_canonical(chan) &&
-        !channel_matches_target_addr_for_extend(chan, target_addr)) {
+    if (!channel_is_canonical(chan) && !matches_target) {
       ++n_noncanonical;
       continue;
     }
@@ -2841,8 +2840,8 @@ channel_get_actual_remote_address(channel_t *chan)
  * Subsequent calls to channel_get_{actual,canonical}_remote_{address,descr}
  * may invalidate the return value from this function.
  */
-const char *
-channel_get_canonical_remote_descr(channel_t *chan)
+MOCK_IMPL(const char *,
+channel_get_canonical_remote_descr,(channel_t *chan))
 {
   tor_assert(chan);
   tor_assert(chan->get_remote_descr);
@@ -3303,20 +3302,33 @@ channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info)
 }
 
 /**
- * Check if a channel matches a given target address; return true iff we do.
+ * Check if a channel matches the given target IPv4 or IPv6 addresses.
+ * If either address matches, return true. If neither address matches,
+ * return false.
+ *
+ * Both addresses can't be NULL.
  *
  * This function calls into the lower layer and asks if this channel thinks
- * it matches a given target address for circuit extension purposes.
+ * it matches the target addresses for circuit extension purposes.
  */
-int
+static bool
 channel_matches_target_addr_for_extend(channel_t *chan,
-                                       const tor_addr_t *target)
+                                       const tor_addr_t *target_ipv4_addr,
+                                       const tor_addr_t *target_ipv6_addr)
 {
   tor_assert(chan);
   tor_assert(chan->matches_target);
-  tor_assert(target);
 
-  return chan->matches_target(chan, target);
+  IF_BUG_ONCE(!target_ipv4_addr && !target_ipv6_addr)
+    return false;
+
+  if (target_ipv4_addr && chan->matches_target(chan, target_ipv4_addr))
+    return true;
+
+  if (target_ipv6_addr && chan->matches_target(chan, target_ipv6_addr))
+    return true;
+
+  return false;
 }
 
 /**
@@ -3389,7 +3401,7 @@ channel_sort_by_ed25519_identity(const void **a_, const void **b_)
  * all of which MUST have the same RSA ID.  (They MAY have different
  * Ed25519 IDs.) */
 static void
-channel_rsa_id_group_set_badness(struct channel_list_s *lst, int force)
+channel_rsa_id_group_set_badness(struct channel_list_t *lst, int force)
 {
   /*XXXX This function should really be about channels. 15056 */
   channel_t *chan = TOR_LIST_FIRST(lst);
diff --git a/src/core/or/channel.h b/src/core/or/channel.h
index d41f0d70bb..fa4ce4f703 100644
--- a/src/core/or/channel.h
+++ b/src/core/or/channel.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 #include "lib/container/handles.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 
+#include "ext/ht.h"
 #include "tor_queue.h"
 
 #define tor_timer_t timeout
@@ -22,7 +23,6 @@ struct tor_timer_t;
 /* Channel handler function pointer typedefs */
 typedef void (*channel_listener_fn_ptr)(channel_listener_t *, channel_t *);
 typedef void (*channel_cell_handler_fn_ptr)(channel_t *, cell_t *);
-typedef void (*channel_var_cell_handler_fn_ptr)(channel_t *, var_cell_t *);
 
 /**
  * This enum is used by channelpadding to decide when to pad channels.
@@ -48,7 +48,7 @@ typedef enum {
 /* channel states for channel_t */
 
 typedef enum {
-  /*
+  /**
    * Closed state - channel is inactive
    *
    * Permitted transitions from:
@@ -57,7 +57,7 @@ typedef enum {
    *   - CHANNEL_STATE_OPENING
    */
   CHANNEL_STATE_CLOSED = 0,
-  /*
+  /**
    * Opening state - channel is trying to connect
    *
    * Permitted transitions from:
@@ -68,7 +68,7 @@ typedef enum {
    *   - CHANNEL_STATE_OPEN
    */
   CHANNEL_STATE_OPENING,
-  /*
+  /**
    * Open state - channel is active and ready for use
    *
    * Permitted transitions from:
@@ -80,7 +80,7 @@ typedef enum {
    *   - CHANNEL_STATE_MAINT
    */
   CHANNEL_STATE_OPEN,
-  /*
+  /**
    * Maintenance state - channel is temporarily offline for subclass specific
    *   maintenance activities such as TLS renegotiation.
    *
@@ -92,7 +92,7 @@ typedef enum {
    *   - CHANNEL_STATE_OPEN
    */
   CHANNEL_STATE_MAINT,
-  /*
+  /**
    * Closing state - channel is shutting down
    *
    * Permitted transitions from:
@@ -103,7 +103,7 @@ typedef enum {
    *   - CHANNEL_STATE_ERROR
    */
   CHANNEL_STATE_CLOSING,
-  /*
+  /**
    * Error state - channel has experienced a permanent error
    *
    * Permitted transitions from:
@@ -115,7 +115,7 @@ typedef enum {
    *   - None
    */
   CHANNEL_STATE_ERROR,
-  /*
+  /**
    * Placeholder for maximum state value
    */
   CHANNEL_STATE_LAST
@@ -124,7 +124,7 @@ typedef enum {
 /* channel listener states for channel_listener_t */
 
 typedef enum {
-  /*
+  /**
    * Closed state - channel listener is inactive
    *
    * Permitted transitions from:
@@ -133,7 +133,7 @@ typedef enum {
    *   - CHANNEL_LISTENER_STATE_LISTENING
    */
   CHANNEL_LISTENER_STATE_CLOSED = 0,
-  /*
+  /**
    * Listening state - channel listener is listening for incoming
    * connections
    *
@@ -144,7 +144,7 @@ typedef enum {
    *   - CHANNEL_LISTENER_STATE_ERROR
    */
   CHANNEL_LISTENER_STATE_LISTENING,
-  /*
+  /**
    * Closing state - channel listener is shutting down
    *
    * Permitted transitions from:
@@ -154,7 +154,7 @@ typedef enum {
    *   - CHANNEL_LISTENER_STATE_ERROR
    */
   CHANNEL_LISTENER_STATE_CLOSING,
-  /*
+  /**
    * Error state - channel listener has experienced a permanent error
    *
    * Permitted transitions from:
@@ -164,7 +164,7 @@ typedef enum {
    *   - None
    */
   CHANNEL_LISTENER_STATE_ERROR,
-  /*
+  /**
    * Placeholder for maximum state value
    */
   CHANNEL_LISTENER_STATE_LAST
@@ -178,15 +178,15 @@ typedef enum {
  * to a particular node, and once constructed support the abstract operations
  * defined below.
  */
-struct channel_s {
+struct channel_t {
   /** Magic number for type-checking cast macros */
   uint32_t magic;
 
   /** List entry for hashtable for global-identifier lookup. */
-  HT_ENTRY(channel_s) gidmap_node;
+  HT_ENTRY(channel_t) gidmap_node;
 
   /** Handle entry for handle-based lookup */
-  HANDLE_ENTRY(channel, channel_s);
+  HANDLE_ENTRY(channel, channel_t);
 
   /** Current channel state */
   channel_state_t state;
@@ -267,21 +267,21 @@ struct channel_s {
 
   /** State variable for use by the scheduler */
   enum {
-    /*
+    /**
      * The channel is not open, or it has a full output buffer but no queued
      * cells.
      */
     SCHED_CHAN_IDLE = 0,
-    /*
+    /**
      * The channel has space on its output buffer to write, but no queued
      * cells.
      */
     SCHED_CHAN_WAITING_FOR_CELLS,
-    /*
+    /**
      * The scheduler has queued cells but no output buffer space to write.
      */
     SCHED_CHAN_WAITING_TO_WRITE,
-    /*
+    /**
      * The scheduler has both queued cells and output buffer space, and is
      * eligible for the scheduler loop.
      */
@@ -320,7 +320,6 @@ struct channel_s {
 
   /** Registered handlers for incoming cells */
   channel_cell_handler_fn_ptr cell_handler;
-  channel_var_cell_handler_fn_ptr var_cell_handler;
 
   /* Methods implemented by the lower layer */
 
@@ -395,7 +394,7 @@ struct channel_s {
    * Linked list of channels with the same RSA identity digest, for use with
    * the digest->channel map
    */
-  TOR_LIST_ENTRY(channel_s) next_with_same_id;
+  TOR_LIST_ENTRY(channel_t) next_with_same_id;
 
   /** Circuit mux for circuits sending on this channel */
   circuitmux_t *cmux;
@@ -442,9 +441,9 @@ struct channel_s {
   ratelim_t last_warned_circ_ids_exhausted;
 
   /** Channel timestamps for cell channels */
-  time_t timestamp_client; /* Client used this, according to relay.c */
-  time_t timestamp_recv; /* Cell received from lower layer */
-  time_t timestamp_xmit; /* Cell sent to lower layer */
+  time_t timestamp_client; /*(< Client used this, according to relay.c */
+  time_t timestamp_recv; /**< Cell received from lower layer */
+  time_t timestamp_xmit; /**< Cell sent to lower layer */
 
   /** Timestamp for run_connection_housekeeping(). We update this once a
    * second when we run housekeeping and find a circuit on this channel, and
@@ -456,16 +455,17 @@ struct channel_s {
    * distinct namespace. */
   uint64_t dirreq_id;
 
-  /** Channel counters for cell channels */
+  /** Channel counters for cells and bytes we have received. */
   uint64_t n_cells_recved, n_bytes_recved;
+  /** Channel counters for cells and bytes we have sent. */
   uint64_t n_cells_xmitted, n_bytes_xmitted;
 };
 
-struct channel_listener_s {
-  /* Current channel listener state */
+struct channel_listener_t {
+  /** Current channel listener state */
   channel_listener_state_t state;
 
-  /* Globally unique ID number for a channel over the lifetime of a Tor
+  /** Globally unique ID number for a channel over the lifetime of a Tor
    * process.
    */
   uint64_t global_identifier;
@@ -539,13 +539,8 @@ void channel_listener_set_listener_fn(channel_listener_t *chan,
 /* Incoming cell callbacks */
 channel_cell_handler_fn_ptr channel_get_cell_handler(channel_t *chan);
 
-channel_var_cell_handler_fn_ptr
-channel_get_var_cell_handler(channel_t *chan);
-
 void channel_set_cell_handlers(channel_t *chan,
-                               channel_cell_handler_fn_ptr cell_handler,
-                               channel_var_cell_handler_fn_ptr
-                                 var_cell_handler);
+                               channel_cell_handler_fn_ptr cell_handler);
 
 /* Clean up closed channels and channel listeners periodically; these are
  * called from run_scheduled_events() in main.c.
@@ -560,13 +555,13 @@ void channel_free_all(void);
 void channel_dumpstats(int severity);
 void channel_listener_dumpstats(int severity);
 
-#ifdef TOR_CHANNEL_INTERNAL_
+#ifdef CHANNEL_OBJECT_PRIVATE
 
-#ifdef CHANNEL_PRIVATE_
+#ifdef CHANNEL_FILE_PRIVATE
 
 STATIC void channel_add_to_digest_map(channel_t *chan);
 
-#endif /* defined(CHANNEL_PRIVATE_) */
+#endif /* defined(CHANNEL_FILE_PRIVATE) */
 
 /* Channel operations for subclasses and internal use only */
 
@@ -645,7 +640,7 @@ void channel_notify_flushed(channel_t *chan);
 /* Handle stuff we need to do on open like notifying circuits */
 void channel_do_open_actions(channel_t *chan);
 
-#endif /* defined(TOR_CHANNEL_INTERNAL_) */
+#endif /* defined(CHANNEL_OBJECT_PRIVATE) */
 
 /* Helper functions to perform operations on channels */
 
@@ -661,11 +656,13 @@ channel_t * channel_connect(const tor_addr_t *addr, uint16_t port,
                             const char *rsa_id_digest,
                             const struct ed25519_public_key_t *ed_id);
 
-channel_t * channel_get_for_extend(const char *rsa_id_digest,
+MOCK_DECL(channel_t *, channel_get_for_extend,(
+                                   const char *rsa_id_digest,
                                    const struct ed25519_public_key_t *ed_id,
-                                   const tor_addr_t *target_addr,
+                                   const tor_addr_t *target_ipv4_addr,
+                                   const tor_addr_t *target_ipv6_addr,
                                    const char **msg_out,
-                                   int *launch_out);
+                                   int *launch_out));
 
 /* Ask which of two channels is better for circuit-extension purposes */
 int channel_is_better(channel_t *a, channel_t *b);
@@ -726,7 +723,7 @@ const char * channel_get_actual_remote_descr(channel_t *chan);
 const char * channel_get_actual_remote_address(channel_t *chan);
 MOCK_DECL(int, channel_get_addr_if_possible, (channel_t *chan,
                                               tor_addr_t *addr_out));
-const char * channel_get_canonical_remote_descr(channel_t *chan);
+MOCK_DECL(const char *, channel_get_canonical_remote_descr,(channel_t *chan));
 int channel_has_queued_writes(channel_t *chan);
 int channel_is_bad_for_new_circs(channel_t *chan);
 void channel_mark_bad_for_new_circs(channel_t *chan);
@@ -741,8 +738,6 @@ int channel_matches_extend_info(channel_t *chan, extend_info_t *extend_info);
 int channel_remote_identity_matches(const channel_t *chan,
                                     const char *rsa_id_digest,
                                     const ed25519_public_key_t *ed_id);
-int channel_matches_target_addr_for_extend(channel_t *chan,
-                                           const tor_addr_t *target);
 unsigned int channel_num_circuits(channel_t *chan);
 MOCK_DECL(void,channel_set_circid_type,(channel_t *chan,
                                         crypto_pk_t *identity_rcvd,
@@ -772,7 +767,7 @@ int packed_cell_is_destroy(channel_t *chan,
                            circid_t *circid_out);
 
 /* Declare the handle helpers */
-HANDLE_DECL(channel, channel_s,)
+HANDLE_DECL(channel, channel_t,)
 #define channel_handle_free(h)    \
   FREE_AND_NULL(channel_handle_t, channel_handle_free_, (h))
 #undef tor_timer_t
diff --git a/src/core/or/channelpadding.c b/src/core/or/channelpadding.c
index 4a0f0e00da..be2ce78a17 100644
--- a/src/core/or/channelpadding.c
+++ b/src/core/or/channelpadding.c
@@ -1,12 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-/* TOR_CHANNEL_INTERNAL_ define needed for an O(1) implementation of
+/**
+ * @file channelpadding.c
+ * @brief Link-level padding code.
+ **/
+
+/* CHANNEL_OBJECT_PRIVATE define needed for an O(1) implementation of
  * channelpadding_channel_to_channelinfo() */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 
 #include "core/or/or.h"
 #include "core/or/channel.h"
diff --git a/src/core/or/channelpadding.h b/src/core/or/channelpadding.h
index 48002eedb7..d1c7192ffd 100644
--- a/src/core/or/channelpadding.h
+++ b/src/core/or/channelpadding.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/or/channeltls.c b/src/core/or/channeltls.c
index 18025ff73a..f9eb67c399 100644
--- a/src/core/or/channeltls.c
+++ b/src/core/or/channeltls.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,7 @@
  * Define this so channel.h gives us things only channel_t subclasses
  * should touch.
  */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 
 #define CHANNELTLS_PRIVATE
 
@@ -45,8 +45,10 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/command.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
+#include "feature/relay/relay_handshake.h"
 #include "feature/control/control.h"
 #include "feature/client/entrynodes.h"
 #include "trunnel/link_handshake.h"
@@ -59,6 +61,7 @@
 #include "feature/nodelist/torcert.h"
 #include "feature/nodelist/networkstatus.h"
 #include "trunnel/channelpadding_negotiation.h"
+#include "trunnel/netinfo.h"
 #include "core/or/channelpadding.h"
 
 #include "core/or/cell_st.h"
@@ -562,9 +565,7 @@ channel_tls_get_transport_name_method(channel_t *chan, char **transport_out)
 static const char *
 channel_tls_get_remote_descr_method(channel_t *chan, int flags)
 {
-#define MAX_DESCR_LEN 32
-
-  static char buf[MAX_DESCR_LEN + 1];
+  static char buf[TOR_ADDRPORT_BUF_LEN];
   channel_tls_t *tlschan = BASE_CHAN_TO_TLS(chan);
   connection_t *conn;
   const char *answer = NULL;
@@ -577,15 +578,14 @@ channel_tls_get_remote_descr_method(channel_t *chan, int flags)
     switch (flags) {
       case 0:
         /* Canonical address with port*/
-        tor_snprintf(buf, MAX_DESCR_LEN + 1,
+        tor_snprintf(buf, TOR_ADDRPORT_BUF_LEN,
                      "%s:%u", conn->address, conn->port);
         answer = buf;
         break;
       case GRD_FLAG_ORIGINAL:
         /* Actual address with port */
         addr_str = tor_addr_to_str_dup(&(tlschan->conn->real_addr));
-        tor_snprintf(buf, MAX_DESCR_LEN + 1,
-                     "%s:%u", addr_str, conn->port);
+        tor_snprintf(buf, TOR_ADDRPORT_BUF_LEN, "%s:%u", addr_str, conn->port);
         tor_free(addr_str);
         answer = buf;
         break;
@@ -724,10 +724,13 @@ channel_tls_matches_target_method(channel_t *chan,
    * base_.addr is updated by connection_or_init_conn_from_address()
    * to be the address in the descriptor. It may be tempting to
    * allow either address to be allowed, but if we did so, it would
-   * enable someone who steals a relay's keys to impersonate/MITM it
+   * enable someone who steals a relay's keys to covertly impersonate/MITM it
    * from anywhere on the Internet! (Because they could make long-lived
    * TLS connections from anywhere to all relays, and wait for them to
    * be used for extends).
+   *
+   * An adversary who has stolen a relay's keys could also post a fake relay
+   * descriptor, but that attack is easier to detect.
    */
   return tor_addr_eq(&(tlschan->conn->real_addr), target);
 }
@@ -937,7 +940,6 @@ channel_tls_listener_describe_transport_method(channel_listener_t *chan_l)
 void
 channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                           or_connection_t *conn,
-                                          uint8_t old_state,
                                           uint8_t state)
 {
   channel_t *base_chan;
@@ -946,8 +948,6 @@ channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
   tor_assert(conn);
   tor_assert(conn->chan == chan);
   tor_assert(chan->conn == conn);
-  /* Shut the compiler up without triggering -Wtautological-compare */
-  (void)old_state;
 
   base_chan = TLS_CHAN_TO_BASE(chan);
 
@@ -1017,6 +1017,16 @@ channel_tls_time_process_cell(cell_t *cell, channel_tls_t *chan, int *time,
 }
 #endif /* defined(KEEP_TIMING_STATS) */
 
+#ifdef KEEP_TIMING_STATS
+#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
+    ++num ## tp;                                                \
+    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
+                             channel_tls_process_ ## tp ## _cell);  \
+    } STMT_END
+#else /* !defined(KEEP_TIMING_STATS) */
+#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
+#endif /* defined(KEEP_TIMING_STATS) */
+
 /**
  * Handle an incoming cell on a channel_tls_t.
  *
@@ -1036,16 +1046,6 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   channel_tls_t *chan;
   int handshaking;
 
-#ifdef KEEP_TIMING_STATS
-#define PROCESS_CELL(tp, cl, cn) STMT_BEGIN {                   \
-    ++num ## tp;                                                \
-    channel_tls_time_process_cell(cl, cn, & tp ## time ,            \
-                             channel_tls_process_ ## tp ## _cell);  \
-    } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
-#define PROCESS_CELL(tp, cl, cn) channel_tls_process_ ## tp ## _cell(cl, cn)
-#endif /* defined(KEEP_TIMING_STATS) */
-
   tor_assert(cell);
   tor_assert(conn);
 
@@ -1063,7 +1063,8 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
     return;
 
   /* Reject all but VERSIONS and NETINFO when handshaking. */
-  /* (VERSIONS should actually be impossible; it's variable-length.) */
+  /* (VERSIONS actually indicates a protocol warning: it's variable-length,
+   * so if it reaches this function, we're on a v1 connection.) */
   if (handshaking && cell->command != CELL_VERSIONS &&
       cell->command != CELL_NETINFO) {
     log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
@@ -1084,13 +1085,13 @@ channel_tls_handle_cell(cell_t *cell, or_connection_t *conn)
   entry_guards_note_internet_connectivity(get_guard_selection_info());
   rep_hist_padding_count_read(PADDING_TYPE_TOTAL);
 
-  if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+  if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
     rep_hist_padding_count_read(PADDING_TYPE_ENABLED_TOTAL);
 
   switch (cell->command) {
     case CELL_PADDING:
       rep_hist_padding_count_read(PADDING_TYPE_CELL);
-      if (TLS_CHAN_TO_BASE(chan)->currently_padding)
+      if (TLS_CHAN_TO_BASE(chan)->padding_enabled)
         rep_hist_padding_count_read(PADDING_TYPE_ENABLED_CELL);
       ++stats_n_padding_cells_processed;
       /* do nothing */
@@ -1317,6 +1318,8 @@ channel_tls_handle_var_cell(var_cell_t *var_cell, or_connection_t *conn)
   }
 }
 
+#undef PROCESS_CELL
+
 /**
  * Update channel marks after connection_or.c has changed an address.
  *
@@ -1632,6 +1635,35 @@ channel_tls_process_padding_negotiate_cell(cell_t *cell, channel_tls_t *chan)
 }
 
 /**
+ * Convert <b>netinfo_addr</b> into corresponding <b>tor_addr</b>.
+ * Return 0 on success; on failure, return -1 and log a warning.
+ */
+static int
+tor_addr_from_netinfo_addr(tor_addr_t *tor_addr,
+                           const netinfo_addr_t *netinfo_addr) {
+  tor_assert(tor_addr);
+  tor_assert(netinfo_addr);
+
+  uint8_t type = netinfo_addr_get_addr_type(netinfo_addr);
+  uint8_t len = netinfo_addr_get_len(netinfo_addr);
+
+  if (type == NETINFO_ADDR_TYPE_IPV4 && len == 4)  {
+    uint32_t ipv4 = netinfo_addr_get_addr_ipv4(netinfo_addr);
+    tor_addr_from_ipv4h(tor_addr, ipv4);
+  } else if (type == NETINFO_ADDR_TYPE_IPV6 && len == 16) {
+    const uint8_t *ipv6_bytes = netinfo_addr_getconstarray_addr_ipv6(
+                                  netinfo_addr);
+    tor_addr_from_ipv6_bytes(tor_addr, ipv6_bytes);
+  } else {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR, "Cannot read address from NETINFO "
+                                     "- wrong type/length.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/**
  * Helper: compute the absolute value of a time_t.
  *
  * (we need this because labs() doesn't always work for time_t, since
@@ -1655,8 +1687,6 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   time_t timestamp;
   uint8_t my_addr_type;
   uint8_t my_addr_len;
-  const uint8_t *my_addr_ptr;
-  const uint8_t *cp, *end;
   uint8_t n_other_addrs;
   time_t now = time(NULL);
   const routerinfo_t *me = router_get_my_routerinfo();
@@ -1704,7 +1734,7 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
         tor_assert(tor_digest_is_zero(
                   (const char*)(chan->conn->handshake_state->
                       authenticated_rsa_peer_id)));
-        tor_assert(tor_mem_is_zero(
+        tor_assert(fast_mem_is_zero(
                   (const char*)(chan->conn->handshake_state->
                                 authenticated_ed25519_peer_id.pubkey), 32));
         /* If the client never authenticated, it's a tor client or bridge
@@ -1727,38 +1757,48 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
   }
 
   /* Decode the cell. */
-  timestamp = ntohl(get_uint32(cell->payload));
-  const time_t sent_versions_at =
-    chan->conn->handshake_state->sent_versions_at;
-  if (now > sent_versions_at && (now - sent_versions_at) < 180) {
-    /* If we have gotten the NETINFO cell reasonably soon after having
-     * sent our VERSIONS cell, maybe we can learn skew information from it. */
-    apparent_skew = now - timestamp;
+  netinfo_cell_t *netinfo_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&netinfo_cell, cell->payload,
+                                      CELL_PAYLOAD_SIZE);
+
+  if (parsed < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_OR,
+           "Failed to parse NETINFO cell - closing connection.");
+    connection_or_close_for_error(chan->conn, 0);
+    return;
   }
 
-  my_addr_type = (uint8_t) cell->payload[4];
-  my_addr_len = (uint8_t) cell->payload[5];
-  my_addr_ptr = (uint8_t*) cell->payload + 6;
-  end = cell->payload + CELL_PAYLOAD_SIZE;
-  cp = cell->payload + 6 + my_addr_len;
+  timestamp = netinfo_cell_get_timestamp(netinfo_cell);
 
+  const netinfo_addr_t *my_addr =
+    netinfo_cell_getconst_other_addr(netinfo_cell);
+
+  my_addr_type = netinfo_addr_get_addr_type(my_addr);
+  my_addr_len = netinfo_addr_get_len(my_addr);
+
+  if ((now - chan->conn->handshake_state->sent_versions_at) < 180) {
+    apparent_skew = now - timestamp;
+  }
   /* We used to check:
    *    if (my_addr_len >= CELL_PAYLOAD_SIZE - 6) {
    *
    * This is actually never going to happen, since my_addr_len is at most 255,
    * and CELL_PAYLOAD_LEN - 6 is 503.  So we know that cp is < end. */
 
-  if (my_addr_type == RESOLVED_TYPE_IPV4 && my_addr_len == 4) {
-    tor_addr_from_ipv4n(&my_apparent_addr, get_uint32(my_addr_ptr));
+  if (tor_addr_from_netinfo_addr(&my_apparent_addr, my_addr) == -1) {
+    connection_or_close_for_error(chan->conn, 0);
+    netinfo_cell_free(netinfo_cell);
+    return;
+  }
 
+  if (my_addr_type == NETINFO_ADDR_TYPE_IPV4 && my_addr_len == 4) {
     if (!get_options()->BridgeRelay && me &&
-        get_uint32(my_addr_ptr) == htonl(me->addr)) {
+        tor_addr_eq_ipv4h(&my_apparent_addr, me->addr)) {
       TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer = 1;
     }
-
-  } else if (my_addr_type == RESOLVED_TYPE_IPV6 && my_addr_len == 16) {
-    tor_addr_from_ipv6_bytes(&my_apparent_addr, (const char *) my_addr_ptr);
-
+  } else if (my_addr_type == NETINFO_ADDR_TYPE_IPV6 &&
+             my_addr_len == 16) {
     if (!get_options()->BridgeRelay && me &&
         !tor_addr_is_null(&me->ipv6_addr) &&
         tor_addr_eq(&my_apparent_addr, &me->ipv6_addr)) {
@@ -1766,18 +1806,20 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
     }
   }
 
-  n_other_addrs = (uint8_t) *cp++;
-  while (n_other_addrs && cp < end-2) {
+  n_other_addrs = netinfo_cell_get_n_my_addrs(netinfo_cell);
+  for (uint8_t i = 0; i < n_other_addrs; i++) {
     /* Consider all the other addresses; if any matches, this connection is
      * "canonical." */
+
+    const netinfo_addr_t *netinfo_addr =
+      netinfo_cell_getconst_my_addrs(netinfo_cell, i);
+
     tor_addr_t addr;
-    const uint8_t *next =
-      decode_address_from_payload(&addr, cp, (int)(end-cp));
-    if (next == NULL) {
+
+    if (tor_addr_from_netinfo_addr(&addr, netinfo_addr) == -1) {
       log_fn(LOG_PROTOCOL_WARN,  LD_OR,
-             "Bad address in netinfo cell; closing connection.");
-      connection_or_close_for_error(chan->conn, 0);
-      return;
+             "Bad address in netinfo cell; Skipping.");
+      continue;
     }
     /* A relay can connect from anywhere and be canonical, so
      * long as it tells you from where it came. This may sound a bit
@@ -1790,10 +1832,10 @@ channel_tls_process_netinfo_cell(cell_t *cell, channel_tls_t *chan)
       connection_or_set_canonical(chan->conn, 1);
       break;
     }
-    cp = next;
-    --n_other_addrs;
   }
 
+  netinfo_cell_free(netinfo_cell);
+
   if (me && !TLS_CHAN_TO_BASE(chan)->is_canonical_to_peer &&
       channel_is_canonical(TLS_CHAN_TO_BASE(chan))) {
     const char *descr =
diff --git a/src/core/or/channeltls.h b/src/core/or/channeltls.h
index a2648ff537..f04ce0fa9c 100644
--- a/src/core/or/channeltls.h
+++ b/src/core/or/channeltls.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,16 +22,16 @@ struct curve25519_public_key_t;
 
 #define TLS_CHAN_MAGIC 0x8a192427U
 
-#ifdef TOR_CHANNEL_INTERNAL_
+#ifdef CHANNEL_OBJECT_PRIVATE
 
-struct channel_tls_s {
+struct channel_tls_t {
   /* Base channel_t struct */
   channel_t base_;
   /* or_connection_t pointer */
   or_connection_t *conn;
 };
 
-#endif /* defined(TOR_CHANNEL_INTERNAL_) */
+#endif /* defined(CHANNEL_OBJECT_PRIVATE) */
 
 channel_t * channel_tls_connect(const tor_addr_t *addr, uint16_t port,
                                 const char *id_digest,
@@ -49,7 +49,6 @@ channel_tls_t * channel_tls_from_base(channel_t *chan);
 void channel_tls_handle_cell(cell_t *cell, or_connection_t *conn);
 void channel_tls_handle_state_change_on_orconn(channel_tls_t *chan,
                                                or_connection_t *conn,
-                                               uint8_t old_state,
                                                uint8_t state);
 void channel_tls_handle_var_cell(var_cell_t *var_cell,
                                  or_connection_t *conn);
diff --git a/src/core/or/circuit_st.h b/src/core/or/circuit_st.h
index d4339ff50d..4baafb1848 100644
--- a/src/core/or/circuit_st.h
+++ b/src/core/or/circuit_st.h
@@ -1,17 +1,30 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file circuit_st.h
+ * @brief Base circuit structure.
+ **/
+
 #ifndef CIRCUIT_ST_H
 #define CIRCUIT_ST_H
 
 #include "core/or/or.h"
 
+#include "lib/container/handles.h"
+
 #include "core/or/cell_queue_st.h"
+#include "ext/ht.h"
 
 struct hs_token_t;
+struct circpad_machine_spec_t;
+struct circpad_machine_runtime_t;
+
+/** Number of padding state machines on a circuit. */
+#define CIRCPAD_MAX_MACHINES (2)
 
 /** "magic" value for an origin_circuit_t */
 #define ORIGIN_CIRCUIT_MAGIC 0x35315243u
@@ -49,6 +62,9 @@ struct circuit_t {
   uint32_t magic; /**< For memory and type debugging: must equal
                    * ORIGIN_CIRCUIT_MAGIC or OR_CIRCUIT_MAGIC. */
 
+  /** Handle entry for handle-based lookup */
+  HANDLE_ENTRY(circuit, circuit_t);
+
   /** The channel that is next in this circuit. */
   channel_t *n_chan;
 
@@ -61,12 +77,6 @@ struct circuit_t {
    */
   circid_t n_circ_id;
 
-  /**
-   * Circuit mux associated with n_chan to which this circuit is attached;
-   * NULL if we have no n_chan.
-   */
-  circuitmux_t *n_mux;
-
   /** Queue of cells waiting to be transmitted on n_chan */
   cell_queue_t n_chan_cells;
 
@@ -93,6 +103,10 @@ struct circuit_t {
   /** True iff this circuit has received a DESTROY cell in either direction */
   unsigned int received_destroy : 1;
 
+  /** True iff we have sent a sufficiently random data cell since last
+   * we reset send_randomness_after_n_cells. */
+  unsigned int have_sent_sufficiently_random_cell : 1;
+
   uint8_t state; /**< Current status of this circuit. */
   uint8_t purpose; /**< Why are we creating this circuit? */
 
@@ -105,6 +119,32 @@ struct circuit_t {
    * circuit-level sendme cells to indicate that we're willing to accept
    * more. */
   int deliver_window;
+  /**
+   * How many cells do we have until we need to send one that contains
+   * sufficient randomness?  Used to ensure that authenticated SENDME cells
+   * will reflect some unpredictable information.
+   **/
+  uint16_t send_randomness_after_n_cells;
+
+  /** FIFO containing the digest of the cells that are just before a SENDME is
+   * sent by the client. It is done at the last cell before our package_window
+   * goes down to 0 which is when we expect a SENDME.
+   *
+   * Our current circuit package window is capped to 1000
+   * (CIRCWINDOW_START_MAX) which is also the start value. The increment is
+   * set to 100 (CIRCWINDOW_INCREMENT) which means we don't allow more than
+   * 1000/100 = 10 outstanding SENDME cells worth of data. Meaning that this
+   * list can not contain more than 10 digests of DIGEST_LEN bytes (20).
+   *
+   * At position i in the list, the digest corresponds to the
+   * (CIRCWINDOW_INCREMENT * i)-nth cell received since we expect a SENDME to
+   * be received containing that cell digest.
+   *
+   * For example, position 2 (starting at 0) means that we've received 300
+   * cells so the 300th cell digest is kept at index 2.
+   *
+   * At maximum, this list contains 200 bytes plus the smartlist overhead. */
+  smartlist_t *sendme_last_digests;
 
   /** Temporary field used during circuits_handle_oom. */
   uint32_t age_tmp;
@@ -177,6 +217,27 @@ struct circuit_t {
   /** Hashtable node: used to look up the circuit by its HS token using the HS
       circuitmap. */
   HT_ENTRY(circuit_t) hs_circuitmap_node;
+
+  /** Adaptive Padding state machines: these are immutable. The state machines
+   *  that come from the consensus are saved to a global structure, to avoid
+   *  per-circuit allocations. This merely points to the global copy in
+   *  origin_padding_machines or relay_padding_machines that should never
+   *  change or get deallocated.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  const struct circpad_machine_spec_t *padding_machine[CIRCPAD_MAX_MACHINES];
+
+  /** Adaptive Padding machine runtime info for above machines. This is
+   *  the per-circuit mutable information, such as the current state and
+   *  histogram token counts. Some of it is optional (aka NULL).
+   *  If a machine is being shut down, these indexes can be NULL
+   *  without the corresponding padding_machine being NULL, while we
+   *  wait for the other end to respond to our shutdown request.
+   *
+   *  Each element of this array corresponds to a different padding machine,
+   *  and we can have up to CIRCPAD_MAX_MACHINES such machines. */
+  struct circpad_machine_runtime_t *padding_info[CIRCPAD_MAX_MACHINES];
 };
 
-#endif
+#endif /* !defined(CIRCUIT_ST_H) */
diff --git a/src/core/or/circuitbuild.c b/src/core/or/circuitbuild.c
index 70b5d8215a..ec61b4a455 100644
--- a/src/core/or/circuitbuild.c
+++ b/src/core/or/circuitbuild.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,20 +21,19 @@
  * cells arrive, the client will invoke circuit_send_next_onion_skin() to send
  * CREATE or RELAY_EXTEND cells.
  *
- * On the server side, this module also handles the logic of responding to
- * RELAY_EXTEND requests, using circuit_extend().
+ * The server side is handled in feature/relay/circuitbuild_relay.c.
  **/
 
 #define CIRCUITBUILD_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/crypto/hs_ntor.h"
 #include "core/crypto/onion_crypto.h"
 #include "core/crypto/onion_fast.h"
 #include "core/crypto/onion_tap.h"
-#include "core/crypto/relay_crypto.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
 #include "core/or/channel.h"
@@ -42,17 +41,20 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/command.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/onion.h"
+#include "core/or/ocirc_event.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
@@ -80,15 +82,6 @@
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
-static channel_t * channel_connect_for_circuit(const tor_addr_t *addr,
-                                            uint16_t port,
-                                            const char *id_digest,
-                                            const ed25519_public_key_t *ed_id);
-static int circuit_deliver_create_cell(circuit_t *circ,
-                                       const create_cell_t *create_cell,
-                                       int relayed);
-static crypt_path_t *onion_next_hop_in_cpath(crypt_path_t *cpath);
-STATIC int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 static int circuit_send_first_onion_skin(origin_circuit_t *circ);
 static int circuit_build_no_more_hops(origin_circuit_t *circ);
 static int circuit_send_intermediate_onion_skin(origin_circuit_t *circ,
@@ -102,10 +95,10 @@ static const node_t *choose_good_middle_server(uint8_t purpose,
  * and then calls command_setup_channel() to give it the right
  * callbacks.
  */
-static channel_t *
-channel_connect_for_circuit(const tor_addr_t *addr, uint16_t port,
-                            const char *id_digest,
-                            const ed25519_public_key_t *ed_id)
+MOCK_IMPL(channel_t *,
+channel_connect_for_circuit,(const tor_addr_t *addr, uint16_t port,
+                             const char *id_digest,
+                             const struct ed25519_public_key_t *ed_id))
 {
   channel_t *chan;
 
@@ -492,7 +485,7 @@ circuit_establish_circuit(uint8_t purpose, extend_info_t *exit_ei, int flags)
     return NULL;
   }
 
-  control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
+  circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
 
   if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
     circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
@@ -508,6 +501,27 @@ origin_circuit_get_guard_state(origin_circuit_t *circ)
   return circ->guard_state;
 }
 
+/**
+ * Helper function to publish a channel association message
+ *
+ * circuit_handle_first_hop() calls this to notify subscribers about a
+ * channel launch event, which associates a circuit with a channel.
+ * This doesn't always correspond to an assignment of the circuit's
+ * n_chan field, because that seems to be only for fully-open
+ * channels.
+ **/
+static void
+circuit_chan_publish(const origin_circuit_t *circ, const channel_t *chan)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = circ->global_identifier;
+  msg->chan = chan->global_identifier;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_chan_publish(msg);
+}
+
 /** Start establishing the first hop of our circuit. Figure out what
  * OR we should connect to, and if necessary start the connection to
  * it. If we're already connected, then send the 'create' cell.
@@ -522,7 +536,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   int should_launch = 0;
   const or_options_t *options = get_options();
 
-  firsthop = onion_next_hop_in_cpath(circ->cpath);
+  firsthop = cpath_get_next_non_open_hop(circ->cpath);
   tor_assert(firsthop);
   tor_assert(firsthop->extend_info);
 
@@ -545,11 +559,17 @@ circuit_handle_first_hop(origin_circuit_t *circ)
             fmt_addrport(&firsthop->extend_info->addr,
                          firsthop->extend_info->port));
 
-  n_chan = channel_get_for_extend(firsthop->extend_info->identity_digest,
-                                  &firsthop->extend_info->ed_identity,
-                                  &firsthop->extend_info->addr,
-                                  &msg,
-                                  &should_launch);
+  /* We'll cleanup this code in #33220, when we add an IPv6 address to
+   * extend_info_t. */
+  const bool addr_is_ipv4 =
+    (tor_addr_family(&firsthop->extend_info->addr) == AF_INET);
+  n_chan = channel_get_for_extend(
+                          firsthop->extend_info->identity_digest,
+                          &firsthop->extend_info->ed_identity,
+                          addr_is_ipv4 ? &firsthop->extend_info->addr : NULL,
+                          addr_is_ipv4 ? NULL : &firsthop->extend_info->addr,
+                          &msg,
+                          &should_launch);
 
   if (!n_chan) {
     /* not currently connected in a useful way. */
@@ -559,8 +579,6 @@ circuit_handle_first_hop(origin_circuit_t *circ)
     circ->base_.n_hop = extend_info_dup(firsthop->extend_info);
 
     if (should_launch) {
-      if (circ->build_state->onehop_tunnel)
-        control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
       n_chan = channel_connect_for_circuit(
           &firsthop->extend_info->addr,
           firsthop->extend_info->port,
@@ -570,6 +588,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
         log_info(LD_CIRC,"connect to firsthop failed. Closing.");
         return -END_CIRC_REASON_CONNECTFAILED;
       }
+      circuit_chan_publish(circ, n_chan);
     }
 
     log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
@@ -581,6 +600,7 @@ circuit_handle_first_hop(origin_circuit_t *circ)
   } else { /* it's already open. use it. */
     tor_assert(!circ->base_.n_hop);
     circ->base_.n_chan = n_chan;
+    circuit_chan_publish(circ, n_chan);
     log_debug(LD_CIRC,"Conn open. Delivering first onion skin.");
     if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
       log_info(LD_CIRC,"circuit_send_next_onion_skin failed.");
@@ -700,9 +720,10 @@ circuit_n_chan_done(channel_t *chan, int status, int close_origin_circuits)
  * gave us via an EXTEND cell, so we shouldn't worry if we don't understand
  * it. Return -1 if we failed to find a suitable circid, else return 0.
  */
-static int
-circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell,
-                            int relayed)
+MOCK_IMPL(int,
+circuit_deliver_create_cell,(circuit_t *circ,
+                             const struct create_cell_t *create_cell,
+                             int relayed))
 {
   cell_t cell;
   circid_t id;
@@ -762,40 +783,6 @@ circuit_deliver_create_cell(circuit_t *circ, const create_cell_t *create_cell,
   return -1;
 }
 
-/** We've decided to start our reachability testing. If all
- * is set, log this to the user. Return 1 if we did, or 0 if
- * we chose not to log anything. */
-int
-inform_testing_reachability(void)
-{
-  char dirbuf[128];
-  char *address;
-  const routerinfo_t *me = router_get_my_routerinfo();
-  if (!me)
-    return 0;
-  address = tor_dup_ip(me->addr);
-  control_event_server_status(LOG_NOTICE,
-                              "CHECKING_REACHABILITY ORADDRESS=%s:%d",
-                              address, me->or_port);
-  if (me->dir_port) {
-    tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d",
-                 address, me->dir_port);
-    control_event_server_status(LOG_NOTICE,
-                                "CHECKING_REACHABILITY DIRADDRESS=%s:%d",
-                                address, me->dir_port);
-  }
-  log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... "
-                         "(this may take up to %d minutes -- look for log "
-                         "messages indicating success)",
-      address, me->or_port,
-      me->dir_port ? dirbuf : "",
-      me->dir_port ? "are" : "is",
-      TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60);
-
-  tor_free(address);
-  return 1;
-}
-
 /** Return true iff we should send a create_fast cell to start building a given
  * circuit */
 static inline int
@@ -941,15 +928,18 @@ circuit_send_next_onion_skin(origin_circuit_t *circ)
   tor_assert(circ->cpath->state == CPATH_STATE_OPEN);
   tor_assert(circ->base_.state == CIRCUIT_STATE_BUILDING);
 
-  crypt_path_t *hop = onion_next_hop_in_cpath(circ->cpath);
+  crypt_path_t *hop = cpath_get_next_non_open_hop(circ->cpath);
   circuit_build_times_handle_completed_hop(circ);
 
+  circpad_machine_event_circ_added_hop(circ);
+
   if (hop) {
     /* Case two: we're on a hop after the first. */
     return circuit_send_intermediate_onion_skin(circ, hop);
   }
 
   /* Case three: the circuit is finished. Do housekeeping tasks on it. */
+  circpad_machine_event_circ_built(circ);
   return circuit_build_no_more_hops(circ);
 }
 
@@ -1192,192 +1182,6 @@ circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle)
   }
 }
 
-/** Take the 'extend' <b>cell</b>, pull out addr/port plus the onion
- * skin and identity digest for the next hop. If we're already connected,
- * pass the onion skin to the next hop using a create cell; otherwise
- * launch a new OR connection, and <b>circ</b> will notice when the
- * connection succeeds or fails.
- *
- * Return -1 if we want to warn and tear down the circuit, else return 0.
- */
-int
-circuit_extend(cell_t *cell, circuit_t *circ)
-{
-  channel_t *n_chan;
-  relay_header_t rh;
-  extend_cell_t ec;
-  const char *msg = NULL;
-  int should_launch = 0;
-
-  if (circ->n_chan) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "n_chan already set. Bug/attack. Closing.");
-    return -1;
-  }
-  if (circ->n_hop) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "conn to next hop already launched. Bug/attack. Closing.");
-    return -1;
-  }
-
-  if (!server_mode(get_options())) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Got an extend cell, but running as a client. Closing.");
-    return -1;
-  }
-
-  relay_header_unpack(&rh, cell->payload);
-
-  if (extend_cell_parse(&ec, rh.command,
-                        cell->payload+RELAY_HEADER_SIZE,
-                        rh.length) < 0) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Can't parse extend cell. Closing circuit.");
-    return -1;
-  }
-
-  if (!ec.orport_ipv4.port || tor_addr_is_null(&ec.orport_ipv4.addr)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend to zero destination port or addr.");
-    return -1;
-  }
-
-  if (tor_addr_is_internal(&ec.orport_ipv4.addr, 0) &&
-      !get_options()->ExtendAllowPrivateAddresses) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend to a private address");
-    return -1;
-  }
-
-  /* Check if they asked us for 0000..0000. We support using
-   * an empty fingerprint for the first hop (e.g. for a bridge relay),
-   * but we don't want to let clients send us extend cells for empty
-   * fingerprints -- a) because it opens the user up to a mitm attack,
-   * and b) because it lets an attacker force the relay to hold open a
-   * new TLS connection for each extend request. */
-  if (tor_digest_is_zero((const char*)ec.node_id)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend without specifying an id_digest.");
-    return -1;
-  }
-
-  /* Fill in ed_pubkey if it was not provided and we can infer it from
-   * our networkstatus */
-  if (ed25519_public_key_is_zero(&ec.ed_pubkey)) {
-    const node_t *node = node_get_by_id((const char*)ec.node_id);
-    const ed25519_public_key_t *node_ed_id = NULL;
-    if (node &&
-        node_supports_ed25519_link_authentication(node, 1) &&
-        (node_ed_id = node_get_ed25519_id(node))) {
-      ed25519_pubkey_copy(&ec.ed_pubkey, node_ed_id);
-    }
-  }
-
-  /* Next, check if we're being asked to connect to the hop that the
-   * extend cell came from. There isn't any reason for that, and it can
-   * assist circular-path attacks. */
-  if (tor_memeq(ec.node_id,
-                TO_OR_CIRCUIT(circ)->p_chan->identity_digest,
-                DIGEST_LEN)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend back to the previous hop.");
-    return -1;
-  }
-
-  /* Check the previous hop Ed25519 ID too */
-  if (! ed25519_public_key_is_zero(&ec.ed_pubkey) &&
-      ed25519_pubkey_eq(&ec.ed_pubkey,
-                        &TO_OR_CIRCUIT(circ)->p_chan->ed25519_identity)) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Client asked me to extend back to the previous hop "
-           "(by Ed25519 ID).");
-    return -1;
-  }
-
-  n_chan = channel_get_for_extend((const char*)ec.node_id,
-                                  &ec.ed_pubkey,
-                                  &ec.orport_ipv4.addr,
-                                  &msg,
-                                  &should_launch);
-
-  if (!n_chan) {
-    log_debug(LD_CIRC|LD_OR,"Next router (%s): %s",
-              fmt_addrport(&ec.orport_ipv4.addr,ec.orport_ipv4.port),
-              msg?msg:"????");
-
-    circ->n_hop = extend_info_new(NULL /*nickname*/,
-                                  (const char*)ec.node_id,
-                                  &ec.ed_pubkey,
-                                  NULL, /*onion_key*/
-                                  NULL, /*curve25519_key*/
-                                  &ec.orport_ipv4.addr,
-                                  ec.orport_ipv4.port);
-
-    circ->n_chan_create_cell = tor_memdup(&ec.create_cell,
-                                          sizeof(ec.create_cell));
-
-    circuit_set_state(circ, CIRCUIT_STATE_CHAN_WAIT);
-
-    if (should_launch) {
-      /* we should try to open a connection */
-      n_chan = channel_connect_for_circuit(&ec.orport_ipv4.addr,
-                                           ec.orport_ipv4.port,
-                                           (const char*)ec.node_id,
-                                           &ec.ed_pubkey);
-      if (!n_chan) {
-        log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
-        circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
-        return 0;
-      }
-      log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
-    }
-    /* return success. The onion/circuit/etc will be taken care of
-     * automatically (may already have been) whenever n_chan reaches
-     * OR_CONN_STATE_OPEN.
-     */
-    return 0;
-  }
-
-  tor_assert(!circ->n_hop); /* Connection is already established. */
-  circ->n_chan = n_chan;
-  log_debug(LD_CIRC,
-            "n_chan is %s",
-            channel_get_canonical_remote_descr(n_chan));
-
-  if (circuit_deliver_create_cell(circ, &ec.create_cell, 1) < 0)
-    return -1;
-
-  return 0;
-}
-
-/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
- *
- * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
- * service circuits and <b>key_data</b> must be at least
- * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
- *
- * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
- * bytes, which are used as follows:
- *   - 20 to initialize f_digest
- *   - 20 to initialize b_digest
- *   - 16 to key f_crypto
- *   - 16 to key b_crypto
- *
- * (If 'reverse' is true, then f_XX and b_XX are swapped.)
- *
- * Return 0 if init was successful, else -1 if it failed.
- */
-int
-circuit_init_cpath_crypto(crypt_path_t *cpath,
-                          const char *key_data, size_t key_data_len,
-                          int reverse, int is_hs_v3)
-{
-
-  tor_assert(cpath);
-  return relay_crypto_init(&cpath->crypto, key_data, key_data_len, reverse,
-                           is_hs_v3);
-}
-
 /** A "created" cell <b>reply</b> came back to us on circuit <b>circ</b>.
  * (The body of <b>reply</b> varies depending on what sort of handshake
  * this is.)
@@ -1403,7 +1207,7 @@ circuit_finish_handshake(origin_circuit_t *circ,
   if (circ->cpath->state == CPATH_STATE_AWAITING_KEYS) {
     hop = circ->cpath;
   } else {
-    hop = onion_next_hop_in_cpath(circ->cpath);
+    hop = cpath_get_next_non_open_hop(circ->cpath);
     if (!hop) { /* got an extended when we're all done? */
       log_warn(LD_PROTOCOL,"got extended when circ already built? Closing.");
       return - END_CIRC_REASON_TORPROTOCOL;
@@ -1427,14 +1231,14 @@ circuit_finish_handshake(origin_circuit_t *circ,
 
   onion_handshake_state_release(&hop->handshake_state);
 
-  if (circuit_init_cpath_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
+  if (cpath_init_circuit_crypto(hop, keys, sizeof(keys), 0, 0)<0) {
     return -END_CIRC_REASON_TORPROTOCOL;
   }
 
   hop->state = CPATH_STATE_OPEN;
   log_info(LD_CIRC,"Finished building circuit hop:");
   circuit_log_path(LOG_INFO,LD_CIRC,circ);
-  control_event_circuit_status(circ, CIRC_EVENT_EXTENDED, 0);
+  circuit_event_status(circ, CIRC_EVENT_EXTENDED, 0);
 
   return 0;
 }
@@ -1479,7 +1283,7 @@ circuit_truncated(origin_circuit_t *circ, int reason)
     }
 
     layer->next = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
   log_info(LD_CIRC, "finished");
@@ -1487,61 +1291,6 @@ circuit_truncated(origin_circuit_t *circ, int reason)
 #endif /* 0 */
 }
 
-/** Given a response payload and keys, initialize, then send a created
- * cell back.
- */
-int
-onionskin_answer(or_circuit_t *circ,
-                 const created_cell_t *created_cell,
-                 const char *keys, size_t keys_len,
-                 const uint8_t *rend_circ_nonce)
-{
-  cell_t cell;
-
-  tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
-
-  if (created_cell_format(&cell, created_cell) < 0) {
-    log_warn(LD_BUG,"couldn't format created cell (type=%d, len=%d)",
-             (int)created_cell->cell_type, (int)created_cell->handshake_len);
-    return -1;
-  }
-  cell.circ_id = circ->p_circ_id;
-
-  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
-
-  log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
-            (unsigned int)get_uint32(keys),
-            (unsigned int)get_uint32(keys+20));
-  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
-    log_warn(LD_BUG,"Circuit initialization failed");
-    return -1;
-  }
-
-  memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
-
-  int used_create_fast = (created_cell->cell_type == CELL_CREATED_FAST);
-
-  append_cell_to_circuit_queue(TO_CIRCUIT(circ),
-                               circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
-  log_debug(LD_CIRC,"Finished sending '%s' cell.",
-            used_create_fast ? "created_fast" : "created");
-
-  /* Ignore the local bit when ExtendAllowPrivateAddresses is set:
-   * it violates the assumption that private addresses are local.
-   * Also, many test networks run on local addresses, and
-   * TestingTorNetwork sets ExtendAllowPrivateAddresses. */
-  if ((!channel_is_local(circ->p_chan)
-       || get_options()->ExtendAllowPrivateAddresses)
-      && !channel_is_outgoing(circ->p_chan)) {
-    /* record that we could process create cells from a non-local conn
-     * that we didn't initiate; presumably this means that create cells
-     * can reach us too. */
-    router_orport_found_reachable();
-  }
-
-  return 0;
-}
-
 /** Helper for new_route_len().  Choose a circuit length for purpose
  * <b>purpose</b>: DEFAULT_ROUTE_LEN (+ 1 if someone else chose the
  * exit).  If someone else chose the exit, they could be colluding
@@ -1673,24 +1422,28 @@ route_len_for_purpose(uint8_t purpose, extend_info_t *exit_ei)
  * to handle the desired path length, return -1.
  */
 STATIC int
-new_route_len(uint8_t purpose, extend_info_t *exit_ei, smartlist_t *nodes)
+new_route_len(uint8_t purpose, extend_info_t *exit_ei,
+              const smartlist_t *nodes)
 {
-  int num_acceptable_routers;
   int routelen;
 
   tor_assert(nodes);
 
   routelen = route_len_for_purpose(purpose, exit_ei);
 
-  num_acceptable_routers = count_acceptable_nodes(nodes);
+  int num_acceptable_direct = count_acceptable_nodes(nodes, 1);
+  int num_acceptable_indirect = count_acceptable_nodes(nodes, 0);
 
-  log_debug(LD_CIRC,"Chosen route length %d (%d/%d routers suitable).",
-            routelen, num_acceptable_routers, smartlist_len(nodes));
+  log_debug(LD_CIRC,"Chosen route length %d (%d direct and %d indirect "
+             "routers suitable).", routelen, num_acceptable_direct,
+             num_acceptable_indirect);
 
-  if (num_acceptable_routers < routelen) {
+  if (num_acceptable_direct < 1 || num_acceptable_indirect < routelen - 1) {
     log_info(LD_CIRC,
-             "Not enough acceptable routers (%d/%d). Discarding this circuit.",
-             num_acceptable_routers, routelen);
+             "Not enough acceptable routers (%d/%d direct and %d/%d "
+             "indirect routers suitable). Discarding this circuit.",
+             num_acceptable_direct, routelen,
+             num_acceptable_indirect, routelen);
     return -1;
   }
 
@@ -2294,7 +2047,7 @@ circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
   state->chosen_exit = extend_info_dup(exit_ei);
 
   ++circ->build_state->desired_path_len;
-  onion_append_hop(&circ->cpath, exit_ei);
+  cpath_append_hop(&circ->cpath, exit_ei);
   return 0;
 }
 
@@ -2332,7 +2085,7 @@ circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *exit_ei)
  * particular router. See bug #25885.)
  */
 MOCK_IMPL(STATIC int,
-count_acceptable_nodes, (smartlist_t *nodes))
+count_acceptable_nodes, (const smartlist_t *nodes, int direct))
 {
   int num=0;
 
@@ -2346,7 +2099,7 @@ count_acceptable_nodes, (smartlist_t *nodes))
     if (! node->is_valid)
 //      log_debug(LD_CIRC,"Nope, the directory says %d is not valid.",i);
       continue;
-    if (! node_has_any_descriptor(node))
+    if (! node_has_preferred_descriptor(node, direct))
       continue;
     /* The node has a descriptor, so we can just check the ntor key directly */
     if (!node_has_curve25519_onion_key(node))
@@ -2359,47 +2112,6 @@ count_acceptable_nodes, (smartlist_t *nodes))
   return num;
 }
 
-/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
- * This function is used to extend cpath by another hop.
- */
-void
-onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop)
-{
-  if (*head_ptr) {
-    new_hop->next = (*head_ptr);
-    new_hop->prev = (*head_ptr)->prev;
-    (*head_ptr)->prev->next = new_hop;
-    (*head_ptr)->prev = new_hop;
-  } else {
-    *head_ptr = new_hop;
-    new_hop->prev = new_hop->next = new_hop;
-  }
-}
-
-#ifdef TOR_UNIT_TESTS
-
-/** Unittest helper function: Count number of hops in cpath linked list. */
-unsigned int
-cpath_get_n_hops(crypt_path_t **head_ptr)
-{
-  unsigned int n_hops = 0;
-  crypt_path_t *tmp;
-
-  if (!*head_ptr) {
-    return 0;
-  }
-
-  tmp = *head_ptr;
-  do {
-    n_hops++;
-    tmp = tmp->next;
-  } while (tmp != *head_ptr);
-
-  return n_hops;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 /**
  * Build the exclude list for vanguard circuits.
  *
@@ -2597,7 +2309,24 @@ choose_good_middle_server(uint8_t purpose,
     return choice;
   }
 
-  choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  if (options->MiddleNodes) {
+    smartlist_t *sl = smartlist_new();
+    routerset_get_all_nodes(sl, options->MiddleNodes,
+                            options->ExcludeNodes, 1);
+
+    smartlist_subtract(sl, excluded);
+
+    choice = node_sl_choose_by_bandwidth(sl, WEIGHT_FOR_MID);
+    smartlist_free(sl);
+    if (choice) {
+      log_fn(LOG_INFO, LD_CIRC, "Chose fixed middle node: %s",
+          hex_str(choice->identity, DIGEST_LEN));
+    } else {
+      log_fn(LOG_NOTICE, LD_CIRC, "Restricted middle not available");
+    }
+  } else {
+    choice = router_choose_random_node(excluded, options->ExcludeNodes, flags);
+  }
   smartlist_free(excluded);
   return choice;
 }
@@ -2657,20 +2386,6 @@ choose_good_entry_server(uint8_t purpose, cpath_build_state_t *state,
   return choice;
 }
 
-/** Return the first non-open hop in cpath, or return NULL if all
- * hops are open. */
-static crypt_path_t *
-onion_next_hop_in_cpath(crypt_path_t *cpath)
-{
-  crypt_path_t *hop = cpath;
-  do {
-    if (hop->state != CPATH_STATE_OPEN)
-      return hop;
-    hop = hop->next;
-  } while (hop != cpath);
-  return NULL;
-}
-
 /** Choose a suitable next hop for the circuit <b>circ</b>.
  * Append the hop info to circ->cpath.
  *
@@ -2727,33 +2442,11 @@ onion_extend_cpath(origin_circuit_t *circ)
             extend_info_describe(info),
             cur_len+1, build_state_get_exit_nickname(state));
 
-  onion_append_hop(&circ->cpath, info);
+  cpath_append_hop(&circ->cpath, info);
   extend_info_free(info);
   return 0;
 }
 
-/** Create a new hop, annotate it with information about its
- * corresponding router <b>choice</b>, and append it to the
- * end of the cpath <b>head_ptr</b>. */
-STATIC int
-onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
-{
-  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
-
-  /* link hop into the cpath, at the end. */
-  onion_append_to_cpath(head_ptr, hop);
-
-  hop->magic = CRYPT_PATH_MAGIC;
-  hop->state = CPATH_STATE_CLOSED;
-
-  hop->extend_info = extend_info_dup(choice);
-
-  hop->package_window = circuit_initial_package_window();
-  hop->deliver_window = CIRCWINDOW_START;
-
-  return 0;
-}
-
 /** Allocate a new extend_info object based on the various arguments. */
 extend_info_t *
 extend_info_new(const char *nickname,
@@ -2895,8 +2588,8 @@ extend_info_dup(extend_info_t *info)
  * If there is no chosen exit, or if we don't know the node_t for
  * the chosen exit, return NULL.
  */
-const node_t *
-build_state_get_exit_node(cpath_build_state_t *state)
+MOCK_IMPL(const node_t *,
+build_state_get_exit_node,(cpath_build_state_t *state))
 {
   if (!state || !state->chosen_exit)
     return NULL;
@@ -2958,7 +2651,7 @@ extend_info_supports_ntor(const extend_info_t* ei)
 {
   tor_assert(ei);
   /* Valid ntor keys have at least one non-zero byte */
-  return !tor_mem_is_zero(
+  return !fast_mem_is_zero(
                           (const char*)ei->curve25519_onion_key.public_key,
                           CURVE25519_PUBKEY_LEN);
 }
diff --git a/src/core/or/circuitbuild.h b/src/core/or/circuitbuild.h
index 969f5b5cc3..e62bb41de9 100644
--- a/src/core/or/circuitbuild.h
+++ b/src/core/or/circuitbuild.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,29 +29,19 @@ struct circuit_guard_state_t *origin_circuit_get_guard_state(
 int circuit_handle_first_hop(origin_circuit_t *circ);
 void circuit_n_chan_done(channel_t *chan, int status,
                          int close_origin_circuits);
-int inform_testing_reachability(void);
 int circuit_timeout_want_to_count_circ(const origin_circuit_t *circ);
 int circuit_send_next_onion_skin(origin_circuit_t *circ);
 void circuit_note_clock_jumped(int64_t seconds_elapsed, bool was_idle);
-int circuit_extend(cell_t *cell, circuit_t *circ);
-int circuit_init_cpath_crypto(crypt_path_t *cpath,
-                              const char *key_data, size_t key_data_len,
-                              int reverse, int is_hs_v3);
 struct created_cell_t;
 int circuit_finish_handshake(origin_circuit_t *circ,
                              const struct created_cell_t *created_cell);
 int circuit_truncated(origin_circuit_t *circ, int reason);
-int onionskin_answer(or_circuit_t *circ,
-                     const struct created_cell_t *created_cell,
-                     const char *keys, size_t keys_len,
-                     const uint8_t *rend_circ_nonce);
 MOCK_DECL(int, circuit_all_predicted_ports_handled, (time_t now,
                                                      int *need_uptime,
                                                      int *need_capacity));
 
 int circuit_append_new_exit(origin_circuit_t *circ, extend_info_t *info);
 int circuit_extend_to_new_exit(origin_circuit_t *circ, extend_info_t *info);
-void onion_append_to_cpath(crypt_path_t **head_ptr, crypt_path_t *new_hop);
 extend_info_t *extend_info_new(const char *nickname,
                                const char *rsa_id_digest,
                                const struct ed25519_public_key_t *ed_id,
@@ -70,7 +60,8 @@ int circuit_can_use_tap(const origin_circuit_t *circ);
 int circuit_has_usable_onion_key(const origin_circuit_t *circ);
 int extend_info_has_preferred_onion_key(const extend_info_t* ei);
 const uint8_t *build_state_get_exit_rsa_id(cpath_build_state_t *state);
-const node_t *build_state_get_exit_node(cpath_build_state_t *state);
+MOCK_DECL(const node_t *,
+          build_state_get_exit_node,(cpath_build_state_t *state));
 const char *build_state_get_exit_nickname(cpath_build_state_t *state);
 
 struct circuit_guard_state_t;
@@ -80,11 +71,26 @@ const node_t *choose_good_entry_server(uint8_t purpose,
                            struct circuit_guard_state_t **guard_state_out);
 void circuit_upgrade_circuits_from_guard_wait(void);
 
+struct ed25519_public_key_t;
+
+MOCK_DECL(channel_t *,
+channel_connect_for_circuit,(const tor_addr_t *addr,
+                             uint16_t port,
+                             const char *id_digest,
+                             const struct ed25519_public_key_t *ed_id));
+
+struct create_cell_t;
+MOCK_DECL(int,
+circuit_deliver_create_cell,(circuit_t *circ,
+                             const struct create_cell_t *create_cell,
+                             int relayed));
+
 #ifdef CIRCUITBUILD_PRIVATE
 STATIC circid_t get_unique_circ_id_by_chan(channel_t *chan);
 STATIC int new_route_len(uint8_t purpose, extend_info_t *exit_ei,
-                         smartlist_t *nodes);
-MOCK_DECL(STATIC int, count_acceptable_nodes, (smartlist_t *nodes));
+                         const smartlist_t *nodes);
+MOCK_DECL(STATIC int, count_acceptable_nodes, (const smartlist_t *nodes,
+                                               int direct));
 
 STATIC int onion_extend_cpath(origin_circuit_t *circ);
 
@@ -92,11 +98,6 @@ STATIC int
 onion_pick_cpath_exit(origin_circuit_t *circ, extend_info_t *exit_ei,
                       int is_hs_v3_rp_circuit);
 
-#if defined(TOR_UNIT_TESTS)
-unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
 #endif /* defined(CIRCUITBUILD_PRIVATE) */
 
 #endif /* !defined(TOR_CIRCUITBUILD_H) */
diff --git a/src/core/or/circuitlist.c b/src/core/or/circuitlist.c
index ccf3041bb4..384835667d 100644
--- a/src/core/or/circuitlist.c
+++ b/src/core/or/circuitlist.c
@@ -1,7 +1,7 @@
 /* Copyright 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -51,6 +51,7 @@
  * logic, which was originally circuit-focused.
  **/
 #define CIRCUITLIST_PRIVATE
+#define OCIRC_EVENT_PRIVATE
 #include "lib/cc/torint.h"  /* TOR_PRIuSZ */
 
 #include "core/or/or.h"
@@ -61,11 +62,13 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/circuitstats.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/crypt_path.h"
 #include "core/mainloop/connection.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/crypt_ops/crypto_dh.h"
@@ -94,7 +97,9 @@
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
+
+#include "core/or/ocirc_event.h"
 
 #include "ht.h"
 
@@ -127,7 +132,6 @@ static smartlist_t *circuits_pending_other_guards = NULL;
  * circuit_mark_for_close and which are waiting for circuit_about_to_free. */
 static smartlist_t *circuits_pending_close = NULL;
 
-static void circuit_free_cpath_node(crypt_path_t *victim);
 static void cpath_ref_decref(crypt_path_reference_t *cpath_ref);
 static void circuit_about_to_free_atexit(circuit_t *circ);
 static void circuit_about_to_free(circuit_t *circ);
@@ -142,6 +146,9 @@ static int any_opened_circs_cached_val = 0;
 
 /********* END VARIABLES ************/
 
+/* Implement circuit handle helpers. */
+HANDLE_IMPL(circuit, circuit_t,)
+
 or_circuit_t *
 TO_OR_CIRCUIT(circuit_t *x)
 {
@@ -208,10 +215,10 @@ chan_circid_entry_hash_(chan_circid_circuit_map_t *a)
 static HT_HEAD(chan_circid_map, chan_circid_circuit_map_t)
      chan_circid_map = HT_INITIALIZER();
 HT_PROTOTYPE(chan_circid_map, chan_circid_circuit_map_t, node,
-             chan_circid_entry_hash_, chan_circid_entries_eq_)
+             chan_circid_entry_hash_, chan_circid_entries_eq_);
 HT_GENERATE2(chan_circid_map, chan_circid_circuit_map_t, node,
              chan_circid_entry_hash_, chan_circid_entries_eq_, 0.6,
-             tor_reallocarray_, tor_free_)
+             tor_reallocarray_, tor_free_);
 
 /** The most recently returned entry from circuit_get_by_circid_chan;
  * used to improve performance when many cells arrive in a row from the
@@ -481,6 +488,54 @@ circuit_set_n_circid_chan(circuit_t *circ, circid_t id,
   }
 }
 
+/**
+ * Helper function to publish a message about events on an origin circuit
+ *
+ * Publishes a message to subscribers of origin circuit events, and
+ * sends the control event.
+ **/
+int
+circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                     int reason_code)
+{
+  ocirc_cevent_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  tor_assert(circ);
+
+  msg->gid = circ->global_identifier;
+  msg->evtype = tp;
+  msg->reason = reason_code;
+  msg->onehop = circ->build_state->onehop_tunnel;
+
+  ocirc_cevent_publish(msg);
+  return control_event_circuit_status(circ, tp, reason_code);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * circuit_set_state() calls this to notify subscribers about a change
+ * of the state of an origin circuit.  @a circ must be an origin
+ * circuit.
+ **/
+static void
+circuit_state_publish(const circuit_t *circ)
+{
+  ocirc_state_msg_t *msg = tor_malloc(sizeof(*msg));
+  const origin_circuit_t *ocirc;
+
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+  ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
+  /* Only inbound OR circuits can be in this state, not origin circuits. */
+  tor_assert(circ->state != CIRCUIT_STATE_ONIONSKIN_PENDING);
+
+  msg->gid = ocirc->global_identifier;
+  msg->state = circ->state;
+  msg->onehop = ocirc->build_state->onehop_tunnel;
+
+  ocirc_state_publish(msg);
+}
+
 /** Change the state of <b>circ</b> to <b>state</b>, adding it to or removing
  * it from lists as appropriate. */
 void
@@ -510,6 +565,8 @@ circuit_set_state(circuit_t *circ, uint8_t state)
   if (state == CIRCUIT_STATE_GUARD_WAIT || state == CIRCUIT_STATE_OPEN)
     tor_assert(!circ->n_chan_create_cell);
   circ->state = state;
+  if (CIRCUIT_IS_ORIGIN(circ))
+    circuit_state_publish(circ);
 }
 
 /** Append to <b>out</b> all circuits in state CHAN_WAIT waiting for
@@ -767,6 +824,8 @@ circuit_purpose_to_controller_string(uint8_t purpose)
       return "PATH_BIAS_TESTING";
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "HS_VANGUARDS";
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "CIRCUIT_PADDING";
 
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
@@ -796,6 +855,7 @@ circuit_purpose_to_controller_hs_state_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_CONTROLLER:
     case CIRCUIT_PURPOSE_PATH_BIAS_TESTING:
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
       return NULL;
 
     case CIRCUIT_PURPOSE_INTRO_POINT:
@@ -896,6 +956,9 @@ circuit_purpose_to_string(uint8_t purpose)
     case CIRCUIT_PURPOSE_HS_VANGUARDS:
       return "Hidden service: Pre-built vanguard circuit";
 
+    case CIRCUIT_PURPOSE_C_CIRCUIT_PADDING:
+      return "Circuit kept open for padding";
+
     default:
       tor_snprintf(buf, sizeof(buf), "UNKNOWN_%d", (int)purpose);
       return buf;
@@ -931,6 +994,7 @@ init_circuit_base(circuit_t *circ)
 
   circ->package_window = circuit_initial_package_window();
   circ->deliver_window = CIRCWINDOW_START;
+  circuit_reset_sendme_randomness(circ);
   cell_queue_init(&circ->n_chan_cells);
 
   smartlist_add(circuit_get_global_list(), circ);
@@ -1072,7 +1136,7 @@ circuit_free_(circuit_t *circ)
    * circuit is closed. This is to avoid any code path that free registered
    * circuits without closing them before. This needs to be done before the
    * hs identifier is freed. */
-  hs_circ_cleanup(circ);
+  hs_circ_cleanup_on_free(circ);
 
   if (CIRCUIT_IS_ORIGIN(circ)) {
     origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
@@ -1092,7 +1156,7 @@ circuit_free_(circuit_t *circ)
 
     if (ocirc->build_state) {
         extend_info_free(ocirc->build_state->chosen_exit);
-        circuit_free_cpath_node(ocirc->build_state->pending_final_cpath);
+        cpath_free(ocirc->build_state->pending_final_cpath);
         cpath_ref_decref(ocirc->build_state->service_pending_final_cpath_ref);
     }
     tor_free(ocirc->build_state);
@@ -1171,11 +1235,23 @@ circuit_free_(circuit_t *circ)
    * "active" checks will be violated. */
   cell_queue_clear(&circ->n_chan_cells);
 
+  /* Cleanup possible SENDME state. */
+  if (circ->sendme_last_digests) {
+    SMARTLIST_FOREACH(circ->sendme_last_digests, uint8_t *, d, tor_free(d));
+    smartlist_free(circ->sendme_last_digests);
+  }
+
   log_info(LD_CIRC, "Circuit %u (id: %" PRIu32 ") has been freed.",
            n_circ_id,
            CIRCUIT_IS_ORIGIN(circ) ?
               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
 
+  /* Free any circuit padding structures */
+  circpad_circuit_free_all_machineinfos(circ);
+
+  /* Clear all dangling handle references. */
+  circuit_handles_clear(circ);
+
   if (should_free) {
     memwipe(mem, 0xAA, memlen); /* poison memory */
     tor_free(mem);
@@ -1207,10 +1283,10 @@ circuit_clear_cpath(origin_circuit_t *circ)
   while (cpath->next && cpath->next != head) {
     victim = cpath;
     cpath = victim->next;
-    circuit_free_cpath_node(victim);
+    cpath_free(victim);
   }
 
-  circuit_free_cpath_node(cpath);
+  cpath_free(cpath);
 
   circ->cpath = NULL;
 }
@@ -1267,29 +1343,13 @@ circuit_free_all(void)
   HT_CLEAR(chan_circid_map, &chan_circid_map);
 }
 
-/** Deallocate space associated with the cpath node <b>victim</b>. */
-static void
-circuit_free_cpath_node(crypt_path_t *victim)
-{
-  if (!victim)
-    return;
-
-  relay_crypto_clear(&victim->crypto);
-  onion_handshake_state_release(&victim->handshake_state);
-  crypto_dh_free(victim->rend_dh_handshake_state);
-  extend_info_free(victim->extend_info);
-
-  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
-  tor_free(victim);
-}
-
 /** Release a crypt_path_reference_t*, which may be NULL. */
 static void
 cpath_ref_decref(crypt_path_reference_t *cpath_ref)
 {
   if (cpath_ref != NULL) {
     if (--(cpath_ref->refcount) == 0) {
-      circuit_free_cpath_node(cpath_ref->cpath);
+      cpath_free(cpath_ref->cpath);
       tor_free(cpath_ref);
     }
   }
@@ -2140,6 +2200,11 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
   tor_assert(line);
   tor_assert(file);
 
+  /* Check whether the circuitpadding subsystem wants to block this close */
+  if (circpad_marked_circuit_for_padding(circ, reason)) {
+    return;
+  }
+
   if (circ->marked_for_close) {
     log_warn(LD_BUG,
         "Duplicate call to circuit_mark_for_close at %s:%d"
@@ -2195,7 +2260,7 @@ circuit_mark_for_close_, (circuit_t *circ, int reason, int line,
   }
 
   /* Notify the HS subsystem that this circuit is closing. */
-  hs_circ_cleanup(circ);
+  hs_circ_cleanup_on_close(circ);
 
   if (circuits_pending_close == NULL)
     circuits_pending_close = smartlist_new();
@@ -2270,50 +2335,13 @@ circuit_about_to_free(circuit_t *circ)
     smartlist_remove(circuits_pending_other_guards, circ);
   }
   if (CIRCUIT_IS_ORIGIN(circ)) {
-    control_event_circuit_status(TO_ORIGIN_CIRCUIT(circ),
+    circuit_event_status(TO_ORIGIN_CIRCUIT(circ),
      (circ->state == CIRCUIT_STATE_OPEN ||
       circ->state == CIRCUIT_STATE_GUARD_WAIT) ?
                                  CIRC_EVENT_CLOSED:CIRC_EVENT_FAILED,
      orig_reason);
   }
 
-  if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    int timed_out = (reason == END_CIRC_REASON_TIMEOUT);
-    tor_assert(circ->state == CIRCUIT_STATE_OPEN);
-    tor_assert(ocirc->build_state->chosen_exit);
-    if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
-        ocirc->rend_data) {
-      /* treat this like getting a nack from it */
-      log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
-          safe_str_client(rend_data_get_address(ocirc->rend_data)),
-          safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
-          timed_out ? "Recording timeout." : "Removing from descriptor.");
-      rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
-                                             ocirc->rend_data,
-                                             timed_out ?
-                                             INTRO_POINT_FAILURE_TIMEOUT :
-                                             INTRO_POINT_FAILURE_GENERIC);
-    }
-  } else if (circ->purpose == CIRCUIT_PURPOSE_C_INTRODUCING &&
-             reason != END_CIRC_REASON_TIMEOUT) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    if (ocirc->build_state->chosen_exit && ocirc->rend_data) {
-      if (orig_reason != END_CIRC_REASON_IP_NOW_REDUNDANT &&
-          ocirc->rend_data) {
-        log_info(LD_REND, "Failed intro circ %s to %s "
-            "(building circuit to intro point). "
-            "Marking intro point as possibly unreachable.",
-            safe_str_client(rend_data_get_address(ocirc->rend_data)),
-            safe_str_client(build_state_get_exit_nickname(
-                                              ocirc->build_state)));
-        rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
-                                              ocirc->rend_data,
-                                              INTRO_POINT_FAILURE_UNREACHABLE);
-      }
-    }
-  }
-
   if (circ->n_chan) {
     circuit_clear_cell_queue(circ, circ->n_chan);
     /* Only send destroy if the channel isn't closing anyway */
@@ -2374,13 +2402,9 @@ marked_circuit_free_cells(circuit_t *circ)
     return;
   }
   cell_queue_clear(&circ->n_chan_cells);
-  if (circ->n_mux)
-    circuitmux_clear_num_cells(circ->n_mux, circ);
   if (! CIRCUIT_IS_ORIGIN(circ)) {
     or_circuit_t *orcirc = TO_OR_CIRCUIT(circ);
     cell_queue_clear(&orcirc->p_chan_cells);
-    if (orcirc->p_mux)
-      circuitmux_clear_num_cells(orcirc->p_mux, circ);
   }
 }
 
@@ -2724,59 +2748,6 @@ circuits_handle_oom(size_t current_allocation)
              n_dirconns_killed);
 }
 
-/** Verify that cpath layer <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-void
-assert_cpath_layer_ok(const crypt_path_t *cp)
-{
-//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
-//  tor_assert(cp->port);
-  tor_assert(cp);
-  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
-  switch (cp->state)
-    {
-    case CPATH_STATE_OPEN:
-      relay_crypto_assert_ok(&cp->crypto);
-      FALLTHROUGH;
-    case CPATH_STATE_CLOSED:
-      /*XXXX Assert that there's no handshake_state either. */
-      tor_assert(!cp->rend_dh_handshake_state);
-      break;
-    case CPATH_STATE_AWAITING_KEYS:
-      /* tor_assert(cp->dh_handshake_state); */
-      break;
-    default:
-      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
-      tor_assert(0);
-    }
-  tor_assert(cp->package_window >= 0);
-  tor_assert(cp->deliver_window >= 0);
-}
-
-/** Verify that cpath <b>cp</b> has all of its invariants
- * correct. Trigger an assert if anything is invalid.
- */
-static void
-assert_cpath_ok(const crypt_path_t *cp)
-{
-  const crypt_path_t *start = cp;
-
-  do {
-    assert_cpath_layer_ok(cp);
-    /* layers must be in sequence of: "open* awaiting? closed*" */
-    if (cp != start) {
-      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      } else if (cp->state == CPATH_STATE_OPEN) {
-        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
-      }
-    }
-    cp = cp->next;
-    tor_assert(cp);
-  } while (cp != start);
-}
-
 /** Verify that circuit <b>c</b> has all of its invariants
  * correct. Trigger an assert if anything is invalid.
  */
@@ -2838,7 +2809,7 @@ assert_circuit_ok,(const circuit_t *c))
                !smartlist_contains(circuits_pending_chans, c));
   }
   if (origin_circ && origin_circ->cpath) {
-    assert_cpath_ok(origin_circ->cpath);
+    cpath_assert_ok(origin_circ->cpath);
   }
   if (c->purpose == CIRCUIT_PURPOSE_REND_ESTABLISHED) {
     tor_assert(or_circ);
diff --git a/src/core/or/circuitlist.h b/src/core/or/circuitlist.h
index b87c6a3667..fd7e22e4c0 100644
--- a/src/core/or/circuitlist.h
+++ b/src/core/or/circuitlist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,8 +12,10 @@
 #ifndef TOR_CIRCUITLIST_H
 #define TOR_CIRCUITLIST_H
 
+#include "lib/container/handles.h"
 #include "lib/testsupport/testsupport.h"
 #include "feature/hs/hs_ident.h"
+#include "core/or/ocirc_event.h"
 
 /** Circuit state: I'm the origin, still haven't done all my handshakes. */
 #define CIRCUIT_STATE_BUILDING 0
@@ -91,31 +93,33 @@
 #define CIRCUIT_PURPOSE_C_HS_MAX_ 13
 /** This circuit is used for build time measurement only */
 #define CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT 14
-#define CIRCUIT_PURPOSE_C_MAX_ 14
+/** This circuit is being held open by circuit padding */
+#define CIRCUIT_PURPOSE_C_CIRCUIT_PADDING 15
+#define CIRCUIT_PURPOSE_C_MAX_ 15
 
-#define CIRCUIT_PURPOSE_S_HS_MIN_ 15
+#define CIRCUIT_PURPOSE_S_HS_MIN_ 16
 /** Hidden-service-side circuit purpose: at the service, waiting for
  * introductions. */
-#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 15
+#define CIRCUIT_PURPOSE_S_ESTABLISH_INTRO 16
 /** Hidden-service-side circuit purpose: at the service, successfully
  * established intro. */
-#define CIRCUIT_PURPOSE_S_INTRO 16
+#define CIRCUIT_PURPOSE_S_INTRO 17
 /** Hidden-service-side circuit purpose: at the service, connecting to rend
  * point. */
-#define CIRCUIT_PURPOSE_S_CONNECT_REND 17
+#define CIRCUIT_PURPOSE_S_CONNECT_REND 18
 /** Hidden-service-side circuit purpose: at the service, rendezvous
  * established. */
-#define CIRCUIT_PURPOSE_S_REND_JOINED 18
+#define CIRCUIT_PURPOSE_S_REND_JOINED 19
 /** This circuit is used for uploading hsdirs */
-#define CIRCUIT_PURPOSE_S_HSDIR_POST 19
-#define CIRCUIT_PURPOSE_S_HS_MAX_ 19
+#define CIRCUIT_PURPOSE_S_HSDIR_POST 20
+#define CIRCUIT_PURPOSE_S_HS_MAX_ 20
 
 /** A testing circuit; not meant to be used for actual traffic. */
-#define CIRCUIT_PURPOSE_TESTING 20
+#define CIRCUIT_PURPOSE_TESTING 21
 /** A controller made this circuit and Tor should not use it. */
-#define CIRCUIT_PURPOSE_CONTROLLER 21
+#define CIRCUIT_PURPOSE_CONTROLLER 22
 /** This circuit is used for path bias probing only */
-#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 22
+#define CIRCUIT_PURPOSE_PATH_BIAS_TESTING 23
 
 /** This circuit is used for vanguards/restricted paths.
  *
@@ -123,9 +127,9 @@
  *  on-demand. When an HS operation needs to take place (e.g. connect to an
  *  intro point), these circuits are then cannibalized and repurposed to the
  *  actual needed HS purpose. */
-#define CIRCUIT_PURPOSE_HS_VANGUARDS 23
+#define CIRCUIT_PURPOSE_HS_VANGUARDS 24
 
-#define CIRCUIT_PURPOSE_MAX_ 23
+#define CIRCUIT_PURPOSE_MAX_ 24
 /** A catch-all for unrecognized purposes. Currently we don't expect
  * to make or see any circuits with this purpose. */
 #define CIRCUIT_PURPOSE_UNKNOWN 255
@@ -184,6 +188,8 @@ void channel_mark_circid_unusable(channel_t *chan, circid_t id);
 void channel_mark_circid_usable(channel_t *chan, circid_t id);
 time_t circuit_id_when_marked_unusable_on_channel(circid_t circ_id,
                                                   channel_t *chan);
+int circuit_event_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                         int reason_code);
 void circuit_set_state(circuit_t *circ, uint8_t state);
 void circuit_close_all_marked(void);
 int32_t circuit_initial_package_window(void);
@@ -213,7 +219,7 @@ void circuit_mark_all_dirty_circs_as_unusable(void);
 void circuit_synchronize_written_or_bandwidth(const circuit_t *c,
                                               circuit_channel_direction_t dir);
 MOCK_DECL(void, circuit_mark_for_close_, (circuit_t *circ, int reason,
-                                          int line, const char *file));
+                                          int line, const char *cfile));
 int circuit_get_cpath_len(origin_circuit_t *circ);
 int circuit_get_cpath_opened_len(const origin_circuit_t *);
 void circuit_clear_cpath(origin_circuit_t *circ);
@@ -225,7 +231,6 @@ int circuit_count_pending_on_channel(channel_t *chan);
 #define circuit_mark_for_close(c, reason)                               \
   circuit_mark_for_close_((c), (reason), __LINE__, SHORT_FILE__)
 
-void assert_cpath_layer_ok(const crypt_path_t *cp);
 MOCK_DECL(void, assert_circuit_ok,(const circuit_t *c));
 void circuit_free_all(void);
 void circuits_handle_oom(size_t current_allocation);
@@ -238,6 +243,11 @@ MOCK_DECL(void, channel_note_destroy_not_pending,
 
 smartlist_t *circuit_find_circuits_to_upgrade_from_guard_wait(void);
 
+/* Declare the handle helpers */
+HANDLE_DECL(circuit, circuit_t, )
+#define circuit_handle_free(h)    \
+    FREE_AND_NULL(circuit_handle_t, circuit_handle_free_, (h))
+
 #ifdef CIRCUITLIST_PRIVATE
 STATIC void circuit_free_(circuit_t *circ);
 #define circuit_free(circ) FREE_AND_NULL(circuit_t, circuit_free_, (circ))
diff --git a/src/core/or/circuitmux.c b/src/core/or/circuitmux.c
index e7309553c4..be54ae6ec6 100644
--- a/src/core/or/circuitmux.c
+++ b/src/core/or/circuitmux.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -69,25 +69,21 @@
  *     made to attach all existing circuits to the new policy.
  **/
 
+#define CIRCUITMUX_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitmux.h"
 #include "core/or/relay.h"
 
-#include "core/or/cell_queue_st.h"
-#include "core/or/destroy_cell_queue_st.h"
 #include "core/or/or_circuit_st.h"
 
-/*
- * Private typedefs for circuitmux.c
- */
+#include "lib/crypt_ops/crypto_util.h"
 
 /*
- * Map of muxinfos for circuitmux_t to use; struct is defined below (name
- * of struct must match HT_HEAD line).
+ * Private typedefs for circuitmux.c
  */
-typedef struct chanid_circid_muxinfo_map chanid_circid_muxinfo_map_t;
 
 /*
  * Hash table entry (yeah, calling it chanid_circid_muxinfo_s seems to
@@ -100,57 +96,14 @@ typedef struct chanid_circid_muxinfo_t chanid_circid_muxinfo_t;
  * a count of queued cells.
  */
 
-typedef struct circuit_muxinfo_s circuit_muxinfo_t;
-
-/*
- * Structures for circuitmux.c
- */
-
-struct circuitmux_s {
-  /* Keep count of attached, active circuits */
-  unsigned int n_circuits, n_active_circuits;
-
-  /* Total number of queued cells on all circuits */
-  unsigned int n_cells;
-
-  /*
-   * Map from (channel ID, circuit ID) pairs to circuit_muxinfo_t
-   */
-  chanid_circid_muxinfo_map_t *chanid_circid_map;
-
-  /** List of queued destroy cells */
-  destroy_cell_queue_t destroy_cell_queue;
-  /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
-   * returned the destroy queue. Used to force alternation between
-   * destroy/non-destroy cells.
-   *
-   * XXXX There is no reason to think that alternating is a particularly good
-   * approach -- it's just designed to prevent destroys from starving other
-   * cells completely.
-   */
-  unsigned int last_cell_was_destroy : 1;
-  /** Destroy counter: increment this when a destroy gets queued, decrement
-   * when we unqueue it, so we can test to make sure they don't starve.
-   */
-  int64_t destroy_ctr;
-
-  /*
-   * Circuitmux policy; if this is non-NULL, it can override the built-
-   * in round-robin active circuits behavior.  This is how EWMA works in
-   * the new circuitmux_t world.
-   */
-  const circuitmux_policy_t *policy;
-
-  /* Policy-specific data */
-  circuitmux_policy_data_t *policy_data;
-};
+typedef struct circuit_muxinfo_t circuit_muxinfo_t;
 
 /*
  * This struct holds whatever we want to store per attached circuit on a
  * circuitmux_t; right now, just the count of queued cells and the direction.
  */
 
-struct circuit_muxinfo_s {
+struct circuit_muxinfo_t {
   /* Count of cells on this circuit at last update */
   unsigned int cell_count;
   /* Direction of flow */
@@ -222,15 +175,12 @@ chanid_circid_entry_hash(chanid_circid_muxinfo_t *a)
   return (unsigned) siphash24g(data, sizeof(data));
 }
 
-/* Declare the struct chanid_circid_muxinfo_map type */
-HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t);
-
 /* Emit a bunch of hash table stuff */
 HT_PROTOTYPE(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node,
-             chanid_circid_entry_hash, chanid_circid_entries_eq)
+             chanid_circid_entry_hash, chanid_circid_entries_eq);
 HT_GENERATE2(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t, node,
              chanid_circid_entry_hash, chanid_circid_entries_eq, 0.6,
-             tor_reallocarray_, tor_free_)
+             tor_reallocarray_, tor_free_);
 
 /*
  * Circuitmux alloc/free functions
@@ -295,9 +245,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /* Clear n_mux */
-            circ->n_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else if (circ->magic == OR_CIRCUIT_MAGIC) {
@@ -310,12 +257,6 @@ circuitmux_detach_all_circuits(circuitmux_t *cmux, smartlist_t *detached_out)
               circuitmux_make_circuit_inactive(cmux, circ);
             }
 
-            /*
-             * It has a sensible p_chan and direction == CELL_DIRECTION_IN,
-             * so clear p_mux.
-             */
-            TO_OR_CIRCUIT(circ)->p_mux = NULL;
-
             if (detached_out)
               smartlist_add(detached_out, circ);
           } else {
@@ -837,18 +778,14 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
      */
     log_info(LD_CIRC,
              "Circuit %u on channel %"PRIu64 " was already attached to "
-             "cmux %p (trying to attach to %p)",
+             "(trying to attach to %p)",
              (unsigned)circ_id, (channel_id),
-             ((direction == CELL_DIRECTION_OUT) ?
-                circ->n_mux : TO_OR_CIRCUIT(circ)->p_mux),
              cmux);
 
     /*
      * The mux pointer on this circuit and the direction in result should
      * match; otherwise assert.
      */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == cmux);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == cmux);
     tor_assert(hashent->muxinfo.direction == direction);
 
     /*
@@ -873,13 +810,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
              "Attaching circuit %u on channel %"PRIu64 " to cmux %p",
               (unsigned)circ_id, (channel_id), cmux);
 
-    /*
-     * Assert that the circuit doesn't already have a mux for this
-     * direction.
-     */
-    if (direction == CELL_DIRECTION_OUT) tor_assert(circ->n_mux == NULL);
-    else tor_assert(TO_OR_CIRCUIT(circ)->p_mux == NULL);
-
     /* Insert it in the map */
     hashent = tor_malloc_zero(sizeof(*hashent));
     hashent->chan_id = channel_id;
@@ -903,10 +833,6 @@ circuitmux_attach_circuit,(circuitmux_t *cmux, circuit_t *circ,
     HT_INSERT(chanid_circid_muxinfo_map, cmux->chanid_circid_map,
               hashent);
 
-    /* Set the circuit's mux for this direction */
-    if (direction == CELL_DIRECTION_OUT) circ->n_mux = cmux;
-    else TO_OR_CIRCUIT(circ)->p_mux = cmux;
-
     /* Update counters */
     ++(cmux->n_circuits);
     if (cell_count > 0) {
@@ -994,14 +920,14 @@ circuitmux_detach_circuit,(circuitmux_t *cmux, circuit_t *circ))
 
     /* Consistency check: the direction must match the direction searched */
     tor_assert(last_searched_direction == hashent->muxinfo.direction);
-    /* Clear the circuit's mux for this direction */
-    if (last_searched_direction == CELL_DIRECTION_OUT) circ->n_mux = NULL;
-    else TO_OR_CIRCUIT(circ)->p_mux = NULL;
 
     /* Now remove it from the map */
     HT_REMOVE(chanid_circid_muxinfo_map, cmux->chanid_circid_map, hashent);
 
-    /* Free the hash entry */
+    /* Wipe and free the hash entry */
+    // This isn't sensitive, but we want to be sure to know if we're accessing
+    // this accidentally.
+    memwipe(hashent, 0xef, sizeof(*hashent));
     tor_free(hashent);
   }
 }
diff --git a/src/core/or/circuitmux.h b/src/core/or/circuitmux.h
index 67cd9bcdd8..191ca12e30 100644
--- a/src/core/or/circuitmux.h
+++ b/src/core/or/circuitmux.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,11 +12,11 @@
 #include "core/or/or.h"
 #include "lib/testsupport/testsupport.h"
 
-typedef struct circuitmux_policy_s circuitmux_policy_t;
-typedef struct circuitmux_policy_data_s circuitmux_policy_data_t;
-typedef struct circuitmux_policy_circ_data_s circuitmux_policy_circ_data_t;
+typedef struct circuitmux_policy_t circuitmux_policy_t;
+typedef struct circuitmux_policy_data_t circuitmux_policy_data_t;
+typedef struct circuitmux_policy_circ_data_t circuitmux_policy_circ_data_t;
 
-struct circuitmux_policy_s {
+struct circuitmux_policy_t {
   /* Allocate cmux-wide policy-specific data */
   circuitmux_policy_data_t * (*alloc_cmux_data)(circuitmux_t *cmux);
   /* Free cmux-wide policy-specific data */
@@ -67,7 +67,7 @@ struct circuitmux_policy_s {
  * wide data; it just has the magic number in the base struct.
  */
 
-struct circuitmux_policy_data_s {
+struct circuitmux_policy_data_t {
   uint32_t magic;
 };
 
@@ -76,7 +76,7 @@ struct circuitmux_policy_data_s {
  * specific data; it just has the magic number in the base struct.
  */
 
-struct circuitmux_policy_circ_data_s {
+struct circuitmux_policy_circ_data_t {
   uint32_t magic;
 };
 
@@ -158,5 +158,61 @@ void circuitmux_mark_destroyed_circids_usable(circuitmux_t *cmux,
 MOCK_DECL(int, circuitmux_compare_muxes,
           (circuitmux_t *cmux_1, circuitmux_t *cmux_2));
 
+#ifdef CIRCUITMUX_PRIVATE
+
+#include "core/or/destroy_cell_queue_st.h"
+
+/*
+ * Map of muxinfos for circuitmux_t to use; struct is defined below (name
+ * of struct must match HT_HEAD line).
+ */
+typedef HT_HEAD(chanid_circid_muxinfo_map, chanid_circid_muxinfo_t)
+  chanid_circid_muxinfo_map_t;
+
+/*
+ * Structures for circuitmux.c
+ */
+
+struct circuitmux_t {
+  /* Keep count of attached, active circuits */
+  unsigned int n_circuits, n_active_circuits;
+
+  /* Total number of queued cells on all circuits */
+  unsigned int n_cells;
+
+  /*
+   * Map from (channel ID, circuit ID) pairs to circuit_muxinfo_t
+   */
+  chanid_circid_muxinfo_map_t *chanid_circid_map;
+
+  /** List of queued destroy cells */
+  destroy_cell_queue_t destroy_cell_queue;
+  /** Boolean: True iff the last cell to circuitmux_get_first_active_circuit
+   * returned the destroy queue. Used to force alternation between
+   * destroy/non-destroy cells.
+   *
+   * XXXX There is no reason to think that alternating is a particularly good
+   * approach -- it's just designed to prevent destroys from starving other
+   * cells completely.
+   */
+  unsigned int last_cell_was_destroy : 1;
+  /** Destroy counter: increment this when a destroy gets queued, decrement
+   * when we unqueue it, so we can test to make sure they don't starve.
+   */
+  int64_t destroy_ctr;
+
+  /*
+   * Circuitmux policy; if this is non-NULL, it can override the built-
+   * in round-robin active circuits behavior.  This is how EWMA works in
+   * the new circuitmux_t world.
+   */
+  const circuitmux_policy_t *policy;
+
+  /* Policy-specific data */
+  circuitmux_policy_data_t *policy_data;
+};
+
+#endif /* defined(CIRCUITMUX_PRIVATE) */
+
 #endif /* !defined(TOR_CIRCUITMUX_H) */
 
diff --git a/src/core/or/circuitmux_ewma.c b/src/core/or/circuitmux_ewma.c
index 3f83c3fd5a..0dcd22e8a7 100644
--- a/src/core/or/circuitmux_ewma.c
+++ b/src/core/or/circuitmux_ewma.c
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,6 +38,7 @@
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
 #include "app/config/or_options_st.h"
 
@@ -58,115 +59,6 @@
 /** The natural logarithm of 0.5. */
 #define LOG_ONEHALF -0.69314718055994529
 
-/*** EWMA structures ***/
-
-typedef struct cell_ewma_s cell_ewma_t;
-typedef struct ewma_policy_data_s ewma_policy_data_t;
-typedef struct ewma_policy_circ_data_s ewma_policy_circ_data_t;
-
-/**
- * The cell_ewma_t structure keeps track of how many cells a circuit has
- * transferred recently.  It keeps an EWMA (exponentially weighted moving
- * average) of the number of cells flushed from the circuit queue onto a
- * connection in channel_flush_from_first_active_circuit().
- */
-
-struct cell_ewma_s {
-  /** The last 'tick' at which we recalibrated cell_count.
-   *
-   * A cell sent at exactly the start of this tick has weight 1.0. Cells sent
-   * since the start of this tick have weight greater than 1.0; ones sent
-   * earlier have less weight. */
-  unsigned int last_adjusted_tick;
-  /** The EWMA of the cell count. */
-  double cell_count;
-  /** True iff this is the cell count for a circuit's previous
-   * channel. */
-  unsigned int is_for_p_chan : 1;
-  /** The position of the circuit within the OR connection's priority
-   * queue. */
-  int heap_index;
-};
-
-struct ewma_policy_data_s {
-  circuitmux_policy_data_t base_;
-
-  /**
-   * Priority queue of cell_ewma_t for circuits with queued cells waiting
-   * for room to free up on the channel that owns this circuitmux.  Kept
-   * in heap order according to EWMA.  This was formerly in channel_t, and
-   * in or_connection_t before that.
-   */
-  smartlist_t *active_circuit_pqueue;
-
-  /**
-   * The tick on which the cell_ewma_ts in active_circuit_pqueue last had
-   * their ewma values rescaled.  This was formerly in channel_t, and in
-   * or_connection_t before that.
-   */
-  unsigned int active_circuit_pqueue_last_recalibrated;
-};
-
-struct ewma_policy_circ_data_s {
-  circuitmux_policy_circ_data_t base_;
-
-  /**
-   * The EWMA count for the number of cells flushed from this circuit
-   * onto this circuitmux.  Used to determine which circuit to flush
-   * from next.  This was formerly in circuit_t and or_circuit_t.
-   */
-  cell_ewma_t cell_ewma;
-
-  /**
-   * Pointer back to the circuit_t this is for; since we're separating
-   * out circuit selection policy like this, we can't attach cell_ewma_t
-   * to the circuit_t any more, so we can't use SUBTYPE_P directly to a
-   * circuit_t like before; instead get it here.
-   */
-  circuit_t *circ;
-};
-
-#define EWMA_POL_DATA_MAGIC 0x2fd8b16aU
-#define EWMA_POL_CIRC_DATA_MAGIC 0x761e7747U
-
-/*** Downcasts for the above types ***/
-
-static ewma_policy_data_t *
-TO_EWMA_POL_DATA(circuitmux_policy_data_t *);
-
-static ewma_policy_circ_data_t *
-TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *);
-
-/**
- * Downcast a circuitmux_policy_data_t to an ewma_policy_data_t and assert
- * if the cast is impossible.
- */
-
-static inline ewma_policy_data_t *
-TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
-{
-  if (!pol) return NULL;
-  else {
-    tor_assert(pol->magic == EWMA_POL_DATA_MAGIC);
-    return DOWNCAST(ewma_policy_data_t, pol);
-  }
-}
-
-/**
- * Downcast a circuitmux_policy_circ_data_t to an ewma_policy_circ_data_t
- * and assert if the cast is impossible.
- */
-
-static inline ewma_policy_circ_data_t *
-TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
-{
-  if (!pol) return NULL;
-  else {
-    tor_assert(pol->magic == EWMA_POL_CIRC_DATA_MAGIC);
-    return DOWNCAST(ewma_policy_circ_data_t, pol);
-  }
-}
-
 /*** Static declarations for circuitmux_ewma.c ***/
 
 static void add_cell_ewma(ewma_policy_data_t *pol, cell_ewma_t *ewma);
@@ -295,6 +187,7 @@ ewma_free_cmux_data(circuitmux_t *cmux,
   pol = TO_EWMA_POL_DATA(pol_data);
 
   smartlist_free(pol->active_circuit_pqueue);
+  memwipe(pol, 0xda, sizeof(ewma_policy_data_t));
   tor_free(pol);
 }
 
@@ -361,7 +254,7 @@ ewma_free_circ_data(circuitmux_t *cmux,
   if (!pol_circ_data) return;
 
   cdata = TO_EWMA_POL_CIRC_DATA(pol_circ_data);
-
+  memwipe(cdata, 0xdc, sizeof(ewma_policy_circ_data_t));
   tor_free(cdata);
 }
 
@@ -530,7 +423,7 @@ ewma_cmp_cmux(circuitmux_t *cmux_1, circuitmux_policy_data_t *pol_data_1,
       /* Pick whichever one has the better best circuit */
       return compare_cell_ewma_counts(ce1, ce2);
     } else {
-      if (ce1 != NULL ) {
+      if (ce1 != NULL) {
         /* We only have a circuit on cmux_1, so prefer it */
         return -1;
       } else if (ce2 != NULL) {
@@ -716,7 +609,7 @@ cmux_ewma_set_options(const or_options_t *options,
   /* convert halflife into halflife-per-tick. */
   halflife /= EWMA_TICK_LEN;
   /* compute per-tick scale factor. */
-  ewma_scale_factor = exp( LOG_ONEHALF / halflife );
+  ewma_scale_factor = exp(LOG_ONEHALF / halflife);
   log_info(LD_OR,
            "Enabled cell_ewma algorithm because of value in %s; "
            "scale factor is %f per %d seconds",
diff --git a/src/core/or/circuitmux_ewma.h b/src/core/or/circuitmux_ewma.h
index b45ce1f916..e41cf9e0f0 100644
--- a/src/core/or/circuitmux_ewma.h
+++ b/src/core/or/circuitmux_ewma.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,9 +22,117 @@ void cmux_ewma_set_options(const or_options_t *options,
 void circuitmux_ewma_free_all(void);
 
 #ifdef CIRCUITMUX_EWMA_PRIVATE
+
+/*** EWMA structures ***/
+
+typedef struct cell_ewma_t cell_ewma_t;
+typedef struct ewma_policy_data_t ewma_policy_data_t;
+typedef struct ewma_policy_circ_data_t ewma_policy_circ_data_t;
+
+/**
+ * The cell_ewma_t structure keeps track of how many cells a circuit has
+ * transferred recently.  It keeps an EWMA (exponentially weighted moving
+ * average) of the number of cells flushed from the circuit queue onto a
+ * connection in channel_flush_from_first_active_circuit().
+ */
+
+struct cell_ewma_t {
+  /** The last 'tick' at which we recalibrated cell_count.
+   *
+   * A cell sent at exactly the start of this tick has weight 1.0. Cells sent
+   * since the start of this tick have weight greater than 1.0; ones sent
+   * earlier have less weight. */
+  unsigned int last_adjusted_tick;
+  /** The EWMA of the cell count. */
+  double cell_count;
+  /** True iff this is the cell count for a circuit's previous
+   * channel. */
+  unsigned int is_for_p_chan : 1;
+  /** The position of the circuit within the OR connection's priority
+   * queue. */
+  int heap_index;
+};
+
+struct ewma_policy_data_t {
+  circuitmux_policy_data_t base_;
+
+  /**
+   * Priority queue of cell_ewma_t for circuits with queued cells waiting
+   * for room to free up on the channel that owns this circuitmux.  Kept
+   * in heap order according to EWMA.  This was formerly in channel_t, and
+   * in or_connection_t before that.
+   */
+  smartlist_t *active_circuit_pqueue;
+
+  /**
+   * The tick on which the cell_ewma_ts in active_circuit_pqueue last had
+   * their ewma values rescaled.  This was formerly in channel_t, and in
+   * or_connection_t before that.
+   */
+  unsigned int active_circuit_pqueue_last_recalibrated;
+};
+
+struct ewma_policy_circ_data_t {
+  circuitmux_policy_circ_data_t base_;
+
+  /**
+   * The EWMA count for the number of cells flushed from this circuit
+   * onto this circuitmux.  Used to determine which circuit to flush
+   * from next.  This was formerly in circuit_t and or_circuit_t.
+   */
+  cell_ewma_t cell_ewma;
+
+  /**
+   * Pointer back to the circuit_t this is for; since we're separating
+   * out circuit selection policy like this, we can't attach cell_ewma_t
+   * to the circuit_t any more, so we can't use SUBTYPE_P directly to a
+   * circuit_t like before; instead get it here.
+   */
+  circuit_t *circ;
+};
+
+#define EWMA_POL_DATA_MAGIC 0x2fd8b16aU
+#define EWMA_POL_CIRC_DATA_MAGIC 0x761e7747U
+
+/*** Downcasts for the above types ***/
+
+/**
+ * Downcast a circuitmux_policy_data_t to an ewma_policy_data_t and assert
+ * if the cast is impossible.
+ */
+
+static inline ewma_policy_data_t *
+TO_EWMA_POL_DATA(circuitmux_policy_data_t *pol)
+{
+  if (!pol) return NULL;
+  else {
+    tor_assertf(pol->magic == EWMA_POL_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_DATA_MAGIC);
+    return DOWNCAST(ewma_policy_data_t, pol);
+  }
+}
+
+/**
+ * Downcast a circuitmux_policy_circ_data_t to an ewma_policy_circ_data_t
+ * and assert if the cast is impossible.
+ */
+
+static inline ewma_policy_circ_data_t *
+TO_EWMA_POL_CIRC_DATA(circuitmux_policy_circ_data_t *pol)
+{
+  if (!pol) return NULL;
+  else {
+    tor_assertf(pol->magic == EWMA_POL_CIRC_DATA_MAGIC,
+                "Mismatch: %"PRIu32" != %"PRIu32,
+                pol->magic, EWMA_POL_CIRC_DATA_MAGIC);
+    return DOWNCAST(ewma_policy_circ_data_t, pol);
+  }
+}
+
 STATIC unsigned cell_ewma_get_current_tick_and_fraction(double *remainder_out);
 STATIC void cell_ewma_initialize_ticks(void);
-#endif
 
-#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
+#endif /* defined(CIRCUITMUX_EWMA_PRIVATE) */
 
+#endif /* !defined(TOR_CIRCUITMUX_EWMA_H) */
diff --git a/src/core/or/circuitpadding.c b/src/core/or/circuitpadding.c
new file mode 100644
index 0000000000..43f4a31624
--- /dev/null
+++ b/src/core/or/circuitpadding.c
@@ -0,0 +1,3101 @@
+/* Copyright (c) 2017 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.c
+ * \brief Circuit-level padding implementation
+ *
+ * \details
+ *
+ * This file implements Tor proposal 254 "Padding Negotiation" which is heavily
+ * inspired by the paper "Toward an Efficient Website Fingerprinting Defense"
+ * by M. Juarez, M. Imani, M. Perry, C. Diaz, M. Wright.
+ *
+ * In particular the code in this file describes mechanisms for clients to
+ * negotiate various types of circuit-level padding from relays.
+ *
+ * Each padding type is described by a state machine (circpad_machine_spec_t),
+ * which is also referred as a "padding machine" in this file.  Currently,
+ * these state machines are hardcoded in the source code (e.g. see
+ * circpad_machines_init()), but in the future we will be able to
+ * serialize them in the torrc or the consensus.
+ *
+ * As specified by prop#254, clients can negotiate padding with relays by using
+ * PADDING_NEGOTIATE cells. After successful padding negotiation, padding
+ * machines are assigned to the circuit in their mutable form as a
+ * circpad_machine_runtime_t.
+ *
+ * Each state of a padding state machine can be either:
+ * - A histogram that specifies inter-arrival padding delays.
+ * - Or a parametrized probability distribution that specifies inter-arrival
+ *   delays (see circpad_distribution_type_t).
+ *
+ * Padding machines start from the START state and finish with the END
+ * state. They can transition between states using the events in
+ * circpad_event_t.
+ *
+ * When a padding machine reaches the END state, it gets wiped from the circuit
+ * so that other padding machines can take over if needed (see
+ * circpad_machine_spec_transitioned_to_end()).
+ *
+ ****************************
+ * General notes:
+ *
+ * All used machines should be heap allocated and placed into
+ * origin_padding_machines/relay_padding_machines so that they get correctly
+ * cleaned up by the circpad_free_all() function.
+ **/
+
+#define CIRCUITPADDING_PRIVATE
+
+#include <math.h>
+#include "lib/math/fp.h"
+#include "lib/math/prob_distr.h"
+#include "core/or/or.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/relay.h"
+#include "feature/stats/rephist.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "core/or/channel.h"
+
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/crypto/relay_crypto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "app/config/config.h"
+
+static inline circpad_circuit_state_t circpad_circuit_state(
+                                        origin_circuit_t *circ);
+static void circpad_setup_machine_on_circ(circuit_t *on_circ,
+                                        const circpad_machine_spec_t *machine);
+static double circpad_distribution_sample(circpad_distribution_t dist);
+
+static inline void circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi);
+
+/** Cached consensus params */
+static uint8_t circpad_padding_disabled;
+static uint8_t circpad_padding_reduced;
+static uint8_t circpad_global_max_padding_percent;
+static uint16_t circpad_global_allowed_cells;
+static uint16_t circpad_max_circ_queued_cells;
+
+/** Global cell counts, for rate limiting */
+static uint64_t circpad_global_padding_sent;
+static uint64_t circpad_global_nonpadding_sent;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 1 (ie: are for client side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *origin_padding_machines = NULL;
+
+/** This is the list of circpad_machine_spec_t's parsed from consensus and
+ *  torrc that have origin_side == 0 (ie: are for relay side).
+ *
+ *  The machines in this smartlist are considered immutable and they are used
+ *  as-is by circuits so they should not change or get deallocated in Tor's
+ *  runtime and as long as circuits are alive. */
+STATIC smartlist_t *relay_padding_machines = NULL;
+
+#ifndef COCCI
+/** Loop over the current padding state machines using <b>loop_var</b> as the
+ *  loop variable. */
+#define FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                         \
+  STMT_BEGIN                                                             \
+  for (int loop_var = 0; loop_var < CIRCPAD_MAX_MACHINES; loop_var++) {
+#define FOR_EACH_CIRCUIT_MACHINE_END } STMT_END ;
+
+/** Loop over the current active padding state machines using <b>loop_var</b>
+ *  as the loop variable. If a machine is not active, skip it. */
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(loop_var, circ)            \
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(loop_var)                               \
+  if (!(circ)->padding_info[loop_var])                           \
+    continue;
+#define FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END } STMT_END ;
+#endif /* !defined(COCCI) */
+
+/**
+ * Free the machineinfo at an index
+ */
+static void
+circpad_circuit_machineinfo_free_idx(circuit_t *circ, int idx)
+{
+  if (circ->padding_info[idx]) {
+    log_fn(LOG_INFO,LD_CIRC, "Freeing padding info idx %d on circuit %u (%d)",
+           idx, CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+           circ->purpose);
+
+    tor_free(circ->padding_info[idx]->histogram);
+    timer_free(circ->padding_info[idx]->padding_timer);
+    tor_free(circ->padding_info[idx]);
+  }
+}
+
+/**
+ * Return true if circpad has decided to hold the circuit open for additional
+ * padding. This function is used to take and retain ownership of certain
+ * types of circuits that have padding machines on them, that have been passed
+ * to circuit_mark_for_close().
+ *
+ * circuit_mark_for_close() calls this function to ask circpad if any padding
+ * machines want to keep the circuit open longer to pad.
+ *
+ * Any non-measurement circuit that was closed for a normal, non-error reason
+ * code may be held open for up to CIRCPAD_DELAY_INFINITE microseconds between
+ * network-driven cell events.
+ *
+ * After CIRCPAD_DELAY_INFINITE microseconds of silence on a circuit, this
+ * function will no longer hold it open (it will return 0 regardless of
+ * what the machines ask for, and thus circuit_expire_old_circuits_clientside()
+ * will close the circuit after roughly 1.25hr of idle time, maximum,
+ * regardless of the padding machine state.
+ */
+int
+circpad_marked_circuit_for_padding(circuit_t *circ, int reason)
+{
+  /* If the circuit purpose is measurement or path bias, don't
+   * hold it open */
+  if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING ||
+      circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT) {
+    return 0;
+  }
+
+  /* If the circuit is closed for any reason other than these three valid,
+   * client-side close reasons, do not try to keep it open. It is probably
+   * damaged or unusable. Note this is OK with vanguards because
+   * controller-closed circuits have REASON=REQUESTED, so vanguards-closed
+   * circuits will not be held open (we want them to close ASAP). */
+  if (!(reason == END_CIRC_REASON_NONE ||
+        reason == END_CIRC_REASON_FINISHED ||
+        reason == END_CIRC_REASON_IP_NOW_REDUNDANT)) {
+    return 0;
+  }
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    circpad_machine_runtime_t *mi = circ->padding_info[i];
+    if (!mi) {
+      continue; // No padding runtime info; check next machine
+    }
+
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+
+    /* If we're in END state (NULL here), then check next machine */
+    if (!state) {
+      continue; // check next machine
+    }
+
+    /* If the machine does not want to control the circuit close itself, then
+     * check the next machine */
+    if (!circ->padding_machine[i]->manage_circ_lifetime) {
+      continue; // check next machine
+    }
+
+    /* If the machine has reached the END state, we can close. Check next
+     * machine. */
+    if (mi->current_state == CIRCPAD_STATE_END) {
+      continue; // check next machine
+    }
+
+    log_info(LD_CIRC, "Circuit %d is not marked for close because of a "
+             "pending padding machine in index %d.",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0, i);
+
+    /* If the machine has had no network events at all within the
+     * last circpad_delay_t timespan, it's in some deadlock state.
+     * Tell circuit_mark_for_close() that we don't own it anymore.
+     * This will allow circuit_expire_old_circuits_clientside() to
+     * close it.
+     */
+    if (circ->padding_info[i]->last_cell_time_sec +
+        (time_t)CIRCPAD_DELAY_MAX_SECS < approx_time()) {
+      log_notice(LD_BUG, "Circuit %d was not marked for close because of a "
+               "pending padding machine in index %d for over an hour. "
+               "Circuit is a %s",
+               CIRCUIT_IS_ORIGIN(circ) ?
+               TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0,
+               i, circuit_purpose_to_string(circ->purpose));
+
+      return 0; // abort timer reached; mark the circuit for close now
+    }
+
+    /* If we weren't marked dirty yet, let's pretend we're dirty now.
+     * ("Dirty" means that a circuit has been used for application traffic
+     * by Tor.. Dirty circuits have different expiry times, and are not
+     * considered in counts of built circuits, etc. By claiming that we're
+     * dirty, the rest of Tor will make decisions as if we were actually
+     * used by application data.
+     *
+     * This is most important for circuit_expire_old_circuits_clientside(),
+     * where we want that function to expire us after the padding machine
+     * has shut down, but using the MaxCircuitDirtiness timer instead of
+     * the idle circuit timer (again, we want this because we're not
+     * supposed to look idle to Guard nodes that can see our lifespan). */
+    if (!circ->timestamp_dirty)
+      circ->timestamp_dirty = approx_time();
+
+    /* Take ownership of the circuit */
+    circuit_change_purpose(circ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+    return 1;
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+
+  return 0; // No machine wanted to keep the circuit open; mark for close
+}
+
+/**
+ * Free all the machineinfos in <b>circ</b> that match <b>machine_num</b>.
+ *
+ * Returns true if any machineinfos with that number were freed.
+ * False otherwise. */
+static int
+free_circ_machineinfos_with_machine_num(circuit_t *circ, int machine_num)
+{
+  int found = 0;
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    if (circ->padding_machine[i] &&
+        circ->padding_machine[i]->machine_num == machine_num) {
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      circ->padding_machine[i] = NULL;
+      found = 1;
+    }
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return found;
+}
+
+/**
+ * Free all padding machines and mutable info associated with circuit
+ */
+void
+circpad_circuit_free_all_machineinfos(circuit_t *circ)
+{
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    circpad_circuit_machineinfo_free_idx(circ, i);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Allocate a new mutable machineinfo structure.
+ */
+STATIC circpad_machine_runtime_t *
+circpad_circuit_machineinfo_new(circuit_t *on_circ, int machine_index)
+{
+  circpad_machine_runtime_t *mi =
+    tor_malloc_zero(sizeof(circpad_machine_runtime_t));
+  mi->machine_index = machine_index;
+  mi->on_circ = on_circ;
+  mi->last_cell_time_sec = approx_time();
+
+  return mi;
+}
+
+/**
+ * Return the circpad_state_t for the current state based on the
+ * mutable info.
+ *
+ * This function returns NULL when the machine is in the end state or in an
+ * invalid state.
+ */
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    return NULL;
+  } else if (BUG(mi->current_state >= machine->num_states)) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Invalid circuit padding state %d",
+           mi->current_state);
+
+    return NULL;
+  }
+
+  return &machine->states[mi->current_state];
+}
+
+/**
+ * Get the lower bound of a histogram bin.
+ *
+ * You can obtain the upper bound using histogram_get_bin_upper_bound().
+ *
+ * This function can also be called with 'bin' set to a value equal or greater
+ * than histogram_len in which case the infinity bin is chosen and
+ * CIRCPAD_DELAY_INFINITE is returned.
+ */
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* The infinity bin has an upper bound of infinity, so make sure we return
+   * that if they ask for it. */
+  if (bin > CIRCPAD_INFINITY_BIN(state)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  return state->histogram_edges[bin] + rtt_add_usec;
+}
+
+/**
+ * Like circpad_histogram_bin_to_usec() but return the upper bound of bin.
+ * (The upper bound is included in the bin.)
+ */
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin)
+{
+  return circpad_histogram_bin_to_usec(mi, bin+1) - 1;
+}
+
+/** Return the midpoint of the histogram bin <b>bin_index</b>. */
+static circpad_delay_t
+circpad_get_histogram_bin_midpoint(const circpad_machine_runtime_t *mi,
+                           int bin_index)
+{
+  circpad_delay_t left_bound = circpad_histogram_bin_to_usec(mi, bin_index);
+  circpad_delay_t right_bound = histogram_get_bin_upper_bound(mi, bin_index);
+
+  return left_bound + (right_bound - left_bound)/2;
+}
+
+/**
+ * Return the bin that contains the usec argument.
+ * "Contains" is defined as us in [lower, upper).
+ *
+ * This function will never return the infinity bin (histogram_len-1), in order
+ * to simplify the rest of the code, so if a usec is provided that falls above
+ * the highest non-infinity bin, that bin index will be returned.
+ */
+STATIC circpad_hist_index_t
+circpad_histogram_usec_to_bin(const circpad_machine_runtime_t *mi,
+                              circpad_delay_t usec)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  circpad_delay_t rtt_add_usec = 0;
+  circpad_hist_index_t bin;
+
+  /* Our state should have been checked to be non-null by the caller
+   * (circpad_machine_remove_token()) */
+  if (BUG(state == NULL)) {
+    return 0;
+  }
+
+  /* If we are using an RTT estimate, consider it as well. */
+  if (state->use_rtt_estimate) {
+    rtt_add_usec = mi->rtt_estimate_usec;
+  }
+
+  /* Walk through the bins and check the upper bound of each bin, if 'usec' is
+   * less-or-equal to that, return that bin. If rtt_estimate is enabled then
+   * add that to the upper bound of each bin.
+   *
+   * We don't want to return the infinity bin here, so don't go there. */
+  for (bin = 0 ; bin < CIRCPAD_INFINITY_BIN(state) ; bin++) {
+    if (usec <= histogram_get_bin_upper_bound(mi, bin) + rtt_add_usec) {
+      return bin;
+    }
+  }
+
+  /* We don't want to return the infinity bin here, so if we still didn't find
+   * the right bin, return the highest non-infinity bin */
+  return CIRCPAD_INFINITY_BIN(state)-1;
+}
+
+/**
+ * Return true if the machine supports token removal.
+ *
+ * Token removal is equivalent to having a mutable histogram in the
+ * circpad_machine_runtime_t mutable info. So while we're at it,
+ * let's assert that everything is consistent between the mutable
+ * runtime and the readonly machine spec.
+ */
+static inline int
+circpad_is_token_removal_supported(circpad_machine_runtime_t *mi)
+{
+  /* No runtime histogram == no token removal */
+  if (mi->histogram == NULL) {
+    /* Machines that don't want token removal are trying to avoid
+     * potentially expensive mallocs, extra memory accesses, and/or
+     * potentially expensive monotime calls. Let's minimize checks
+     * and keep this path fast. */
+    tor_assert_nonfatal(mi->histogram_len == 0);
+    return 0;
+  } else {
+    /* Machines that do want token removal are less sensitive to performance.
+     * Let's spend some time to check that our state is consistent and sane */
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return 1;
+    }
+    tor_assert_nonfatal(state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE);
+    tor_assert_nonfatal(state->histogram_len == mi->histogram_len);
+    tor_assert_nonfatal(mi->histogram_len != 0);
+    return 1;
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * This function frees any token bins allocated from a previous state
+ *
+ * Called after a state transition, or if the bins are empty.
+ */
+STATIC void
+circpad_machine_setup_tokens(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+
+  /* If this state doesn't exist, or doesn't have token removal,
+   * free any previous state's runtime histogram, and bail.
+   *
+   * If we don't have a token removal strategy, we also don't need a runtime
+   * histogram and we rely on the immutable one in machine_spec_t. */
+  if (!state || state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE) {
+    if (mi->histogram) {
+      tor_free(mi->histogram);
+      mi->histogram = NULL;
+      mi->histogram_len = 0;
+    }
+    return;
+  }
+
+  /* Try to avoid re-mallocing if we don't really need to */
+  if (!mi->histogram || (mi->histogram
+          && mi->histogram_len != state->histogram_len)) {
+    tor_free(mi->histogram); // null ok
+    mi->histogram = tor_malloc_zero(sizeof(circpad_hist_token_t)
+                                    *state->histogram_len);
+  }
+  mi->histogram_len = state->histogram_len;
+
+  memcpy(mi->histogram, state->histogram,
+         sizeof(circpad_hist_token_t)*state->histogram_len);
+}
+
+/**
+ * Choose a length for this state (in cells), if specified.
+ */
+static void
+circpad_choose_state_length(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  double length;
+
+  if (!state || state->length_dist.type == CIRCPAD_DIST_NONE) {
+    mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+    return;
+  }
+
+  length = circpad_distribution_sample(state->length_dist);
+  length = MAX(0, length);
+  length += state->start_length;
+
+  if (state->max_length) {
+    length = MIN(length, state->max_length);
+  }
+
+  mi->state_length = clamp_double_to_int64(length);
+
+  log_info(LD_CIRC, "State length sampled to %"PRIu64" for circuit %u",
+      mi->state_length, CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+}
+
+/**
+ * Sample a value from our iat_dist, and clamp it safely
+ * to circpad_delay_t.
+ *
+ * Before returning, add <b>delay_shift</b> (can be zero) to the sampled value.
+ */
+static circpad_delay_t
+circpad_distribution_sample_iat_delay(const circpad_state_t *state,
+                                      circpad_delay_t delay_shift)
+{
+  double val = circpad_distribution_sample(state->iat_dist);
+  /* These comparisons are safe, because the output is in the range
+   * [0, 2**32), and double has a precision of 53 bits. */
+  /* We want a positive sample value */
+  val = MAX(0, val);
+  /* Respect the maximum sample setting */
+  val = MIN(val, state->dist_max_sample_usec);
+
+  /* Now apply the shift:
+   * This addition is exact: val is at most 2**32-1, delay_shift is at most
+   * 2**32-1, and doubles have a precision of 53 bits. */
+  val += delay_shift;
+
+  /* Clamp the distribution at infinite delay val */
+  return (circpad_delay_t)MIN(tor_llround(val), CIRCPAD_DELAY_INFINITE);
+}
+
+/**
+ * Sample an expected time-until-next-packet delay from the histogram or
+ * probability distribution.
+ *
+ * A bin of the histogram is chosen with probability proportional to the number
+ * of tokens in each bin, and then a time value is chosen uniformly from that
+ * bin's [start,end) time range.
+ */
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = circpad_machine_current_state(mi);
+  const circpad_hist_token_t *histogram = NULL;
+  circpad_hist_index_t curr_bin = 0;
+  circpad_delay_t bin_start, bin_end;
+  /* These three must all be larger than circpad_hist_token_t, because
+   * we sum several circpad_hist_token_t values across the histogram */
+  uint64_t curr_weight = 0;
+  uint64_t histogram_total_tokens = 0;
+  uint64_t bin_choice;
+
+  tor_assert(state);
+
+  if (state->iat_dist.type != CIRCPAD_DIST_NONE) {
+    /* Sample from a fixed IAT distribution and return */
+    circpad_delay_t iat_delay_shift = state->use_rtt_estimate ?
+      mi->rtt_estimate_usec + state->dist_added_shift_usec :
+      state->dist_added_shift_usec;
+    return circpad_distribution_sample_iat_delay(state, iat_delay_shift);
+  } else if (circpad_is_token_removal_supported(mi)) {
+    histogram = mi->histogram;
+    for (circpad_hist_index_t b = 0; b < state->histogram_len; b++)
+      histogram_total_tokens += histogram[b];
+  } else {
+    /* We have a histogram, but it's immutable */
+    histogram = state->histogram;
+    histogram_total_tokens = state->histogram_total_tokens;
+  }
+
+  /* If we are out of tokens, don't schedule padding. */
+  if (!histogram_total_tokens) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  bin_choice = crypto_fast_rng_get_uint64(get_thread_fast_rng(),
+                                          histogram_total_tokens);
+
+  /* Skip all the initial zero bins */
+  while (!histogram[curr_bin]) {
+    curr_bin++;
+  }
+  curr_weight = histogram[curr_bin];
+
+  // TODO: This is not constant-time. Pretty sure we don't
+  // really need it to be, though.
+  while (curr_weight < bin_choice) {
+    curr_bin++;
+    /* It should be impossible to run past the end of the histogram */
+    if (BUG(curr_bin >= state->histogram_len)) {
+      return CIRCPAD_DELAY_INFINITE;
+    }
+    curr_weight += histogram[curr_bin];
+  }
+
+  /* Do some basic checking of the current bin we are in */
+  if (BUG(curr_bin >= state->histogram_len) ||
+      BUG(histogram[curr_bin] == 0)) {
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  // Store this index to remove the token upon callback.
+  if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE) {
+    mi->chosen_bin = curr_bin;
+  }
+
+  if (curr_bin >= CIRCPAD_INFINITY_BIN(state)) {
+    if (state->token_removal != CIRCPAD_TOKEN_REMOVAL_NONE &&
+        mi->histogram[curr_bin] > 0) {
+      mi->histogram[curr_bin]--;
+    }
+
+    // Infinity: Don't send a padding packet. Wait for a real packet
+    // and then see if our bins are empty or what else we should do.
+    return CIRCPAD_DELAY_INFINITE;
+  }
+
+  tor_assert(curr_bin < CIRCPAD_INFINITY_BIN(state));
+
+  bin_start = circpad_histogram_bin_to_usec(mi, curr_bin);
+  /* We don't need to reduct 1 from the upper bound because the random range
+   * function below samples from [bin_start, bin_end) */
+  bin_end = circpad_histogram_bin_to_usec(mi, curr_bin+1);
+
+  /* Bin edges are monotonically increasing so this is a bug. Handle it. */
+  if (BUG(bin_start >= bin_end)) {
+    return bin_start;
+  }
+
+  return (circpad_delay_t)crypto_fast_rng_uint64_range(get_thread_fast_rng(),
+                                                       bin_start, bin_end);
+}
+
+/**
+ * Sample a value from the specified probability distribution.
+ *
+ * Uses functions from src/lib/math/prob_distr.c .
+ */
+static double
+circpad_distribution_sample(circpad_distribution_t dist)
+{
+  log_fn(LOG_DEBUG,LD_CIRC, "Sampling delay with distribution %d",
+         dist.type);
+
+  switch (dist.type) {
+    case CIRCPAD_DIST_NONE:
+      {
+        /* We should not get in here like this */
+        tor_assert_nonfatal_unreached();
+        return 0;
+      }
+    case CIRCPAD_DIST_UNIFORM:
+      {
+        // param2 is upper bound, param1 is lower
+        const struct uniform_t my_uniform = {
+          .base = UNIFORM(my_uniform),
+          .a = dist.param1,
+          .b = dist.param2,
+        };
+        return dist_sample(&my_uniform.base);
+      }
+    case CIRCPAD_DIST_LOGISTIC:
+      {
+      /* param1 is Mu, param2 is sigma. */
+        const struct logistic_t my_logistic = {
+          .base = LOGISTIC(my_logistic),
+          .mu = dist.param1,
+          .sigma = dist.param2,
+        };
+        return dist_sample(&my_logistic.base);
+      }
+    case CIRCPAD_DIST_LOG_LOGISTIC:
+      {
+        /* param1 is Alpha, param2 is 1.0/Beta */
+        const struct log_logistic_t my_log_logistic = {
+          .base = LOG_LOGISTIC(my_log_logistic),
+          .alpha = dist.param1,
+          .beta = dist.param2,
+        };
+        return dist_sample(&my_log_logistic.base);
+      }
+    case CIRCPAD_DIST_GEOMETRIC:
+      {
+        /* param1 is 'p' (success probability) */
+        const struct geometric_t my_geometric = {
+          .base = GEOMETRIC(my_geometric),
+          .p = dist.param1,
+        };
+        return dist_sample(&my_geometric.base);
+      }
+    case CIRCPAD_DIST_WEIBULL:
+      {
+        /* param1 is k, param2 is Lambda */
+        const struct weibull_t my_weibull = {
+          .base = WEIBULL(my_weibull),
+          .k = dist.param1,
+          .lambda = dist.param2,
+        };
+        return dist_sample(&my_weibull.base);
+      }
+    case CIRCPAD_DIST_PARETO:
+      {
+        /* param1 is sigma, param2 is xi, no more params for mu so we use 0 */
+        const struct genpareto_t my_genpareto = {
+          .base = GENPARETO(my_genpareto),
+          .mu = 0,
+          .sigma = dist.param1,
+          .xi = dist.param2,
+        };
+        return dist_sample(&my_genpareto.base);
+      }
+  }
+
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+/**
+ * Find the index of the first bin whose upper bound is
+ * greater than the target, and that has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_higher_index(const circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  /* Don't remove from the infinity bin */
+  for (; bin < CIRCPAD_INFINITY_BIN(mi); bin++) {
+    if (mi->histogram[bin] &&
+        histogram_get_bin_upper_bound(mi, bin) >= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return mi->histogram_len;
+}
+
+/**
+ * Find the index of the first bin whose lower bound is lower or equal to
+ * <b>target_bin_usec</b>, and that still has tokens remaining.
+ *
+ * Used for histograms with token removal.
+ */
+static circpad_hist_index_t
+circpad_machine_first_lower_index(const circpad_machine_runtime_t *mi,
+                                  circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+                                                           target_bin_usec);
+
+  for (; bin >= 0; bin--) {
+    if (mi->histogram[bin] &&
+        circpad_histogram_bin_to_usec(mi, bin) <= target_bin_usec) {
+      return bin;
+    }
+  }
+
+  return -1;
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * greater than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                    circpad_delay_t target_bin_usec)
+{
+  /* We need to remove the token from the first bin
+   * whose upper bound is greater than the target, and that
+   * has tokens remaining. */
+  circpad_hist_index_t bin = circpad_machine_first_higher_index(mi,
+                                                     target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/**
+ * Remove a token from the first non-empty bin whose upper bound is
+ * lower than the target.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                   circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_machine_first_lower_index(mi,
+          target_bin_usec);
+
+  if (bin >= 0 && bin < CIRCPAD_INFINITY_BIN(mi)) {
+    if (!BUG(mi->histogram[bin] == 0)) {
+      mi->histogram[bin]--;
+    }
+  }
+}
+
+/* Helper macro: Ensure that the bin has tokens available, and BUG out of the
+ * function if it's not the case. */
+#define ENSURE_BIN_CAPACITY(bin_index) \
+  if (BUG(mi->histogram[bin_index] == 0)) {                   \
+    return;                                                   \
+  }
+
+/**
+ * Remove a token from the closest non-empty bin to the target.
+ *
+ * If use_usec is true, measure "closest" in terms of the next closest bin
+ * midpoint.
+ *
+ * If it is false, use bin index distance only.
+ *
+ * Used for histograms with token removal.
+ */
+STATIC void
+circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                     circpad_delay_t target_bin_usec,
+                                     bool use_usec)
+{
+  circpad_hist_index_t lower, higher, current;
+  circpad_hist_index_t bin_to_remove = -1;
+
+  lower = circpad_machine_first_lower_index(mi, target_bin_usec);
+  higher = circpad_machine_first_higher_index(mi, target_bin_usec);
+  current = circpad_histogram_usec_to_bin(mi, target_bin_usec);
+
+  /* Sanity check the results */
+  if (BUG(lower > current) || BUG(higher < current)) {
+    return;
+  }
+
+  /* Take care of edge cases first */
+  if (higher == mi->histogram_len && lower == -1) {
+    /* All bins are empty */
+    return;
+  } else if (higher == mi->histogram_len) {
+    /* All higher bins are empty */
+    ENSURE_BIN_CAPACITY(lower);
+    mi->histogram[lower]--;
+    return;
+  } else if (lower == -1) {
+    /* All lower bins are empty */
+    ENSURE_BIN_CAPACITY(higher);
+    mi->histogram[higher]--;
+    return;
+  }
+
+  /* Now handle the intermediate cases */
+  if (use_usec) {
+    /* Find the closest bin midpoint to the target */
+    circpad_delay_t lower_usec = circpad_get_histogram_bin_midpoint(mi, lower);
+    circpad_delay_t higher_usec =
+      circpad_get_histogram_bin_midpoint(mi, higher);
+
+    if (target_bin_usec < lower_usec) {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    } else if (target_bin_usec > higher_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else if (target_bin_usec-lower_usec > higher_usec-target_bin_usec) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      bin_to_remove = higher;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      bin_to_remove = lower;
+    }
+    mi->histogram[bin_to_remove]--;
+    log_debug(LD_CIRC, "Removing token from bin %d", bin_to_remove);
+    return;
+  } else {
+    if (current - lower > higher - current) {
+      // Higher bin is closer
+      ENSURE_BIN_CAPACITY(higher);
+      mi->histogram[higher]--;
+      return;
+    } else {
+      // Lower bin is closer
+      ENSURE_BIN_CAPACITY(lower);
+      mi->histogram[lower]--;
+      return;
+    }
+  }
+}
+
+#undef ENSURE_BIN_CAPACITY
+
+/**
+ * Remove a token from the exact bin corresponding to the target.
+ *
+ * If it is empty, do nothing.
+ *
+ * Used for histograms with token removal.
+ */
+static void
+circpad_machine_remove_exact(circpad_machine_runtime_t *mi,
+                             circpad_delay_t target_bin_usec)
+{
+  circpad_hist_index_t bin = circpad_histogram_usec_to_bin(mi,
+          target_bin_usec);
+
+  if (mi->histogram[bin] > 0)
+    mi->histogram[bin]--;
+}
+
+/**
+ * Check our state's cell limit count and tokens.
+ *
+ * Returns 1 if either limits are hit and we decide to change states,
+ * otherwise returns 0.
+ */
+static circpad_decision_t
+check_machine_token_supply(circpad_machine_runtime_t *mi)
+{
+  uint32_t histogram_total_tokens = 0;
+
+  /* Check if bins empty. This requires summing up the current mutable
+   * machineinfo histogram token total and checking if it is zero.
+   * Machineinfo does not keep a running token count. We're assuming the
+   * extra space is not worth this short loop iteration.
+   *
+   * We also do not count infinity bin in histogram totals.
+   */
+  if (circpad_is_token_removal_supported(mi)) {
+    for (circpad_hist_index_t b = 0; b < CIRCPAD_INFINITY_BIN(mi); b++)
+      histogram_total_tokens += mi->histogram[b];
+
+    /* If we change state, we're done */
+    if (histogram_total_tokens == 0) {
+      if (circpad_internal_event_bins_empty(mi) == CIRCPAD_STATE_CHANGED)
+        return CIRCPAD_STATE_CHANGED;
+    }
+  }
+
+  if (mi->state_length == 0) {
+    return circpad_internal_event_state_length_up(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Count that a padding packet was sent.
+ *
+ * This updates our state length count, our machine rate limit counts,
+ * and if token removal is used, decrements the histogram.
+ */
+static inline void
+circpad_machine_count_padding_sent(circpad_machine_runtime_t *mi)
+{
+  /* If we have a valid state length bound, consider it */
+  if (mi->state_length != CIRCPAD_STATE_LENGTH_INFINITE &&
+      !BUG(mi->state_length <= 0)) {
+    mi->state_length--;
+  }
+
+  /*
+   * Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct.
+   */
+  mi->padding_sent++;
+  if (mi->padding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  circpad_global_padding_sent++;
+
+  /* If we have a mutable histogram, reduce the token count from
+   * the chosen padding bin (this assumes we always send padding
+   * when we intended to). */
+  if (circpad_is_token_removal_supported(mi)) {
+    /* Check array bounds and token count before removing */
+    if (!BUG(mi->chosen_bin >= mi->histogram_len) &&
+        !BUG(mi->histogram[mi->chosen_bin] == 0)) {
+      mi->histogram[mi->chosen_bin]--;
+    }
+  }
+}
+
+/**
+ * Count a nonpadding packet as being sent.
+ *
+ * This function updates our overhead accounting variables, as well
+ * as decrements the state limit packet counter, if the latter was
+ * flagged as applying to non-padding as well.
+ */
+static inline void
+circpad_machine_count_nonpadding_sent(circpad_machine_runtime_t *mi)
+{
+  /* Update non-padding counts for rate limiting: We scale at UINT16_MAX
+   * because we only use this for a percentile limit of 2 sig figs, and
+   * space is scare in the machineinfo struct. */
+  mi->nonpadding_sent++;
+  if (mi->nonpadding_sent == UINT16_MAX) {
+    mi->padding_sent /= 2;
+    mi->nonpadding_sent /= 2;
+  }
+
+  /* Update any state packet length limits that apply */
+  circpad_machine_update_state_length_for_nonpadding(mi);
+
+  /* Remove a token from the histogram, if applicable */
+  circpad_machine_remove_token(mi);
+}
+
+/**
+ * Decrement the state length counter for a non-padding packet.
+ *
+ * Only updates the state length if we're using that feature, we
+ * have a state, and the machine wants to count non-padding packets
+ * towards the state length.
+ */
+static inline void
+circpad_machine_update_state_length_for_nonpadding(
+        circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+
+  if (mi->state_length == CIRCPAD_STATE_LENGTH_INFINITE)
+    return;
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+
+  /* If we're enforcing a state length on non-padding packets,
+   * decrement it */
+  if (state->length_includes_nonpadding &&
+      mi->state_length > 0) {
+    mi->state_length--;
+  }
+}
+
+/**
+ * When a non-padding packet arrives, remove a token from the bin
+ * corresponding to the delta since last sent packet. If that bin
+ * is empty, choose a token based on the specified removal strategy
+ * in the state machine.
+ */
+STATIC void
+circpad_machine_remove_token(circpad_machine_runtime_t *mi)
+{
+  const circpad_state_t *state = NULL;
+  circpad_time_t current_time;
+  circpad_delay_t target_bin_usec;
+
+  /* Dont remove any tokens if there was no padding scheduled */
+  if (!mi->padding_scheduled_at_usec) {
+    return;
+  }
+
+  state = circpad_machine_current_state(mi);
+
+  /* If we are not in a padding state (like start or end), we're done */
+  if (!state)
+    return;
+  /* Don't remove any tokens if we're not doing token removal */
+  if (state->token_removal == CIRCPAD_TOKEN_REMOVAL_NONE)
+    return;
+
+  current_time = monotime_absolute_usec();
+
+  /* If we have scheduled padding some time in the future, we want to see what
+     bin we are in at the current time */
+  target_bin_usec = (circpad_delay_t)
+                  MIN((current_time - mi->padding_scheduled_at_usec),
+                      CIRCPAD_DELAY_INFINITE-1);
+
+  /* We are treating this non-padding cell as a padding cell, so we cancel
+     padding timer, if present. */
+  mi->padding_scheduled_at_usec = 0;
+  if (mi->is_padding_timer_scheduled) {
+    mi->is_padding_timer_scheduled = 0;
+    timer_disable(mi->padding_timer);
+  }
+
+  /* Perform the specified token removal strategy */
+  switch (state->token_removal) {
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 1);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_CLOSEST:
+      circpad_machine_remove_closest_token(mi, target_bin_usec, 0);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_LOWER:
+      circpad_machine_remove_lower_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_HIGHER:
+      circpad_machine_remove_higher_token(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_EXACT:
+      circpad_machine_remove_exact(mi, target_bin_usec);
+      break;
+    case CIRCPAD_TOKEN_REMOVAL_NONE:
+    default:
+      tor_assert_nonfatal_unreached();
+      log_warn(LD_BUG, "Circpad: Unknown token removal strategy %d",
+               state->token_removal);
+      break;
+  }
+}
+
+/**
+ * Send a relay command with a relay cell payload on a circuit to
+ * the particular hopnum.
+ *
+ * Hopnum starts at 1 (1=guard, 2=middle, 3=exit, etc).
+ *
+ * Payload may be null.
+ *
+ * Returns negative on error, 0 on success.
+ */
+MOCK_IMPL(STATIC signed_error_t,
+circpad_send_command_to_hop,(origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len))
+{
+  crypt_path_t *target_hop = circuit_get_cpath_hop(circ, hopnum);
+  signed_error_t ret;
+
+  /* Check that the cpath has the target hop */
+  if (!target_hop) {
+    log_fn(LOG_WARN, LD_BUG, "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier, circuit_get_cpath_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Check that the target hop is opened */
+  if (target_hop->state != CPATH_STATE_OPEN) {
+    log_fn(LOG_WARN,LD_CIRC,
+           "Padding circuit %u has %d hops, not %d",
+           circ->global_identifier,
+           circuit_get_cpath_opened_len(circ), hopnum);
+    return -1;
+  }
+
+  /* Send the drop command to the second hop */
+  ret = relay_send_command_from_edge(0, TO_CIRCUIT(circ), relay_command,
+                                     (const char*)payload, payload_len,
+                                     target_hop);
+  return ret;
+}
+
+/**
+ * Callback helper to send a padding cell.
+ *
+ * This helper is called after our histogram-sampled delay period passes
+ * without another packet being sent first. If a packet is sent before this
+ * callback happens, it is canceled. So when we're called here, send padding
+ * right away.
+ *
+ * If sending this padding cell forced us to transition states return
+ * CIRCPAD_STATE_CHANGED. Otherwise return CIRCPAD_STATE_UNCHANGED.
+ */
+circpad_decision_t
+circpad_send_padding_cell_for_callback(circpad_machine_runtime_t *mi)
+{
+  circuit_t *circ = mi->on_circ;
+  int machine_idx = mi->machine_index;
+  mi->padding_scheduled_at_usec = 0;
+  circpad_statenum_t state = mi->current_state;
+
+  /* Make sure circuit didn't close on us */
+  if (mi->on_circ->marked_for_close) {
+    log_fn(LOG_INFO,LD_CIRC,
+           "Padding callback on circuit marked for close (%u). Ignoring.",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+         TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_CHANGED;
+  }
+
+  circpad_machine_count_padding_sent(mi);
+
+  if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+    circpad_send_command_to_hop(TO_ORIGIN_CIRCUIT(mi->on_circ),
+                                CIRCPAD_GET_MACHINE(mi)->target_hopnum,
+                                RELAY_COMMAND_DROP, NULL, 0);
+    log_info(LD_CIRC, "Callback: Sending padding to origin circuit %u"
+             " (%d) [length: %"PRIu64"]",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier,
+             mi->on_circ->purpose, mi->state_length);
+  } else {
+    // If we're a non-origin circ, we can just send from here as if we're the
+    // edge.
+    if (TO_OR_CIRCUIT(circ)->p_chan_cells.n <= circpad_max_circ_queued_cells) {
+      log_info(LD_CIRC, "Callback: Sending padding to circuit (%d)"
+               " [length: %"PRIu64"]", mi->on_circ->purpose, mi->state_length);
+      relay_send_command_from_edge(0, mi->on_circ, RELAY_COMMAND_DROP, NULL,
+                                   0, NULL);
+      rep_hist_padding_count_write(PADDING_TYPE_DROP);
+    } else {
+      static ratelim_t cell_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&cell_lim,LOG_NOTICE,LD_CIRC,
+                     "Too many cells (%d) in circ queue to send padding.",
+                      TO_OR_CIRCUIT(circ)->p_chan_cells.n);
+    }
+  }
+
+  /* This is a padding cell sent from the client or from the middle node,
+   * (because it's invoked from circuitpadding.c) */
+  circpad_cell_event_padding_sent(circ);
+
+  /* The circpad_cell_event_padding_sent() could cause us to transition.
+   * Check that we still have a padding machineinfo, and then check our token
+   * supply. */
+  if (circ->padding_info[machine_idx] != NULL) {
+    if (state != circ->padding_info[machine_idx]->current_state)
+      return CIRCPAD_STATE_CHANGED;
+    else
+      return check_machine_token_supply(circ->padding_info[machine_idx]);
+  } else {
+    return CIRCPAD_STATE_CHANGED;
+  }
+}
+
+/**
+ * Tor-timer compatible callback that tells us to send a padding cell.
+ *
+ * Timers are associated with circpad_machine_runtime_t's. When the machineinfo
+ * is freed on a circuit, the timers are cancelled. Since the lifetime
+ * of machineinfo is always longer than the timers, handles are not
+ * needed.
+ */
+static void
+circpad_send_padding_callback(tor_timer_t *timer, void *args,
+                              const struct monotime_t *time)
+{
+  circpad_machine_runtime_t *mi = ((circpad_machine_runtime_t*)args);
+  (void)timer; (void)time;
+
+  if (mi && mi->on_circ) {
+    assert_circuit_ok(mi->on_circ);
+    circpad_send_padding_cell_for_callback(mi);
+  } else {
+    // This shouldn't happen (represents a timer leak)
+    log_fn(LOG_WARN,LD_CIRC,
+            "Circuit closed while waiting for padding timer.");
+    tor_fragile_assert();
+  }
+
+  // TODO-MP-AP: Unify this counter with channelpadding for rephist stats
+  //total_timers_pending--;
+}
+
+/**
+ * Cache our consensus parameters upon consensus update.
+ */
+void
+circpad_new_consensus_params(const networkstatus_t *ns)
+{
+  circpad_padding_disabled =
+      networkstatus_get_param(ns, "circpad_padding_disabled",
+         0, 0, 1);
+
+  circpad_padding_reduced =
+      networkstatus_get_param(ns, "circpad_padding_reduced",
+         0, 0, 1);
+
+  circpad_global_allowed_cells =
+      networkstatus_get_param(ns, "circpad_global_allowed_cells",
+         0, 0, UINT16_MAX-1);
+
+  circpad_global_max_padding_percent =
+      networkstatus_get_param(ns, "circpad_global_max_padding_pct",
+         0, 0, 100);
+
+  circpad_max_circ_queued_cells =
+      networkstatus_get_param(ns, "circpad_max_circ_queued_cells",
+         CIRCWINDOW_START_MAX, 0, 50*CIRCWINDOW_START_MAX);
+}
+
+/**
+ * Return true if padding is allowed by torrc and consensus.
+ */
+static bool
+circpad_is_padding_allowed(void)
+{
+  /* If padding has been disabled in the consensus, don't send any more
+   * padding. Technically the machine should be shut down when the next
+   * machine condition check happens, but machine checks only happen on
+   * certain circuit events, and if padding is disabled due to some
+   * network overload or DoS condition, we really want to stop ASAP. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding) {
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * Check this machine against its padding limits, as well as global
+ * consensus limits.
+ *
+ * We have two limits: a percent and a cell count. The cell count
+ * limit must be reached before the percent is enforced (this is to
+ * optionally allow very light padding of things like circuit setup
+ * while there is no other traffic on the circuit).
+ *
+ * TODO: Don't apply limits to machines form torrc.
+ *
+ * Returns 1 if limits are set and we've hit them. Otherwise returns 0.
+ */
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+
+  /* If machine_padding_pct is non-zero, and we've sent more
+   * than the allowed count of padding cells, then check our
+   * percent limits for this machine. */
+  if (machine->max_padding_percent &&
+      mi->padding_sent >= machine->allowed_padding_count) {
+    uint32_t total_cells = mi->padding_sent + mi->nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*(uint32_t)mi->padding_sent) / total_cells >
+        machine->max_padding_percent) {
+      return 1; // limit is reached. Stop.
+    }
+  }
+
+  /* If circpad_max_global_padding_pct is non-zero, and we've
+   * sent more than the global padding cell limit, then check our
+   * global tor process percentage limit on padding. */
+  if (circpad_global_max_padding_percent &&
+      circpad_global_padding_sent >= circpad_global_allowed_cells) {
+    uint64_t total_cells = circpad_global_padding_sent +
+              circpad_global_nonpadding_sent;
+
+    /* Check the percent */
+    if ((100*circpad_global_padding_sent) / total_cells >
+        circpad_global_max_padding_percent) {
+      return 1; // global limit reached. Stop.
+    }
+  }
+
+  return 0; // All good!
+}
+
+/**
+ * Schedule the next padding time according to the machineinfo on a
+ * circuit.
+ *
+ * The histograms represent inter-packet-delay. Whenever you get an packet
+ * event you should be scheduling your next timer (after cancelling any old
+ * ones and updating tokens accordingly).
+ *
+ * Returns 1 if we decide to transition states (due to infinity bin),
+ * 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *mi))
+{
+  circpad_delay_t in_usec = 0;
+  struct timeval timeout;
+  tor_assert(mi);
+
+  /* Don't schedule padding if it is disabled */
+  if (!circpad_is_padding_allowed()) {
+    static ratelim_t padding_lim = RATELIM_INIT(600);
+    log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+         "Padding has been disabled, but machine still on circuit %"PRIu64
+         ", %d",
+         mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+         mi->on_circ->n_circ_id);
+
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Don't schedule padding if we are currently in dormant mode. */
+  if (!is_participating_on_network()) {
+    log_info(LD_CIRC, "Not scheduling padding because we are dormant.");
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  // Don't pad in end (but  also don't cancel any previously
+  // scheduled padding either).
+  if (mi->current_state == CIRCPAD_STATE_END) {
+    log_fn(LOG_INFO, LD_CIRC, "Padding end state on circuit %u",
+         CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check our padding limits */
+  if (circpad_machine_reached_padding_limit(mi)) {
+   if (CIRCUIT_IS_ORIGIN(mi->on_circ)) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Padding machine has reached padding limit on circuit %u",
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier);
+    } else {
+      static ratelim_t padding_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&padding_lim,LOG_INFO,LD_CIRC,
+           "Padding machine has reached padding limit on circuit %"PRIu64
+           ", %d",
+           mi->on_circ->n_chan ? mi->on_circ->n_chan->global_identifier : 0,
+           mi->on_circ->n_circ_id);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (mi->is_padding_timer_scheduled) {
+    /* Cancel current timer (if any) */
+    timer_disable(mi->padding_timer);
+    mi->is_padding_timer_scheduled = 0;
+  }
+
+  /* in_usec = in microseconds */
+  in_usec = circpad_machine_sample_delay(mi);
+  /* If we're using token removal, we need to know when the padding
+   * was scheduled at, so we can remove the appropriate token if
+   * a non-padding cell is sent before the padding timer expires.
+   *
+   * However, since monotime is unpredictably expensive, let's avoid
+   * using it for machines that don't need token removal. */
+  if (circpad_is_token_removal_supported(mi)) {
+    mi->padding_scheduled_at_usec = monotime_absolute_usec();
+  } else {
+    mi->padding_scheduled_at_usec = 1;
+  }
+  log_fn(LOG_INFO,LD_CIRC,"\tPadding in %u usec on circuit %u", in_usec,
+       CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0);
+
+  // Don't schedule if we have infinite delay.
+  if (in_usec == CIRCPAD_DELAY_INFINITE) {
+    return circpad_internal_event_infinity(mi);
+  }
+
+  if (mi->state_length == 0) {
+    /* If we're at length 0, that means we hit 0 after sending
+     * a cell earlier, and emitted an event for it, but
+     * for whatever reason we did not decide to change states then.
+     * So maybe the machine is waiting for bins empty, or for an
+     * infinity event later? That would be a strange machine,
+     * but there's no reason to make it impossible. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  if (in_usec <= 0) {
+    return circpad_send_padding_cell_for_callback(mi);
+  }
+
+  timeout.tv_sec = in_usec/TOR_USEC_PER_SEC;
+  timeout.tv_usec = (in_usec%TOR_USEC_PER_SEC);
+
+  log_fn(LOG_INFO, LD_CIRC, "\tPadding circuit %u in %u sec, %u usec",
+     CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+           TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+          (unsigned)timeout.tv_sec, (unsigned)timeout.tv_usec);
+
+  if (mi->padding_timer) {
+    timer_set_cb(mi->padding_timer, circpad_send_padding_callback, mi);
+  } else {
+    mi->padding_timer =
+        timer_new(circpad_send_padding_callback, mi);
+  }
+  timer_schedule(mi->padding_timer, &timeout);
+  mi->is_padding_timer_scheduled = 1;
+
+  // TODO-MP-AP: Unify with channelpadding counter
+  //rep_hist_padding_count_timers(++total_timers_pending);
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * If the machine transitioned to the END state, we need
+ * to check to see if it wants us to shut it down immediately.
+ * If it does, then we need to send the appropiate negotiation commands
+ * depending on which side it is.
+ *
+ * After this function is called, mi may point to freed memory. Do
+ * not access it.
+ */
+static void
+circpad_machine_spec_transitioned_to_end(circpad_machine_runtime_t *mi)
+{
+  const circpad_machine_spec_t *machine = CIRCPAD_GET_MACHINE(mi);
+  circuit_t *on_circ = mi->on_circ;
+
+  log_fn(LOG_INFO,LD_CIRC, "Padding machine in end state on circuit %u (%d)",
+         CIRCUIT_IS_ORIGIN(on_circ) ?
+         TO_ORIGIN_CIRCUIT(on_circ)->global_identifier : 0,
+         on_circ->purpose);
+
+  /*
+   * We allow machines to shut down and delete themselves as opposed
+   * to just going back to START or waiting forever in END so that
+   * we can handle the case where this machine started while it was
+   * the only machine that matched conditions, but *since* then more
+   * "higher ranking" machines now match the conditions, and would
+   * be given a chance to take precedence over this one in
+   * circpad_add_matching_machines().
+   *
+   * Returning to START or waiting forever in END would not give those
+   * other machines a chance to be launched, where as shutting down
+   * here does.
+   */
+  if (machine->should_negotiate_end) {
+    if (machine->is_origin_side) {
+      /* We free the machine info here so that we can be replaced
+       * by a different machine. But we must leave the padding_machine
+       * in place to wait for the negotiated response */
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(on_circ),
+                                machine->machine_num,
+                                machine->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    } else {
+      circpad_circuit_machineinfo_free_idx(on_circ,
+                                           machine->machine_index);
+      circpad_padding_negotiated(on_circ,
+                                machine->machine_num,
+                                CIRCPAD_COMMAND_STOP,
+                                CIRCPAD_RESPONSE_OK);
+      on_circ->padding_machine[machine->machine_index] = NULL;
+    }
+  }
+}
+
+/**
+ * Generic state transition function for padding state machines.
+ *
+ * Given an event and our mutable machine info, decide if/how to
+ * transition to a different state, and perform actions accordingly.
+ *
+ * Returns 1 if we transition states, 0 otherwise.
+ */
+MOCK_IMPL(circpad_decision_t,
+circpad_machine_spec_transition,(circpad_machine_runtime_t *mi,
+                            circpad_event_t event))
+{
+  const circpad_state_t *state =
+      circpad_machine_current_state(mi);
+
+  /* If state is null we are in the end state. */
+  if (!state) {
+    /* If we in end state we don't pad no matter what. */
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+
+  /* Check if this event is ignored or causes a cancel */
+  if (state->next_state[event] == CIRCPAD_STATE_IGNORE) {
+    return CIRCPAD_STATE_UNCHANGED;
+  } else if (state->next_state[event] == CIRCPAD_STATE_CANCEL) {
+    /* Check cancel events and cancel any pending padding */
+    mi->padding_scheduled_at_usec = 0;
+    if (mi->is_padding_timer_scheduled) {
+      mi->is_padding_timer_scheduled = 0;
+      /* Cancel current timer (if any) */
+      timer_disable(mi->padding_timer);
+    }
+    return CIRCPAD_STATE_UNCHANGED;
+  } else {
+    circpad_statenum_t s = state->next_state[event];
+    /* See if we need to transition to any other states based on this event.
+     * Whenever a transition happens, even to our own state, we schedule
+     * padding.
+     *
+     * So if a state only wants to schedule padding for an event, it specifies
+     * a transition to itself. All non-specified events are ignored.
+     */
+    log_fn(LOG_INFO, LD_CIRC,
+           "Circuit %u circpad machine %d transitioning from %u to %u",
+             CIRCUIT_IS_ORIGIN(mi->on_circ) ?
+             TO_ORIGIN_CIRCUIT(mi->on_circ)->global_identifier : 0,
+           mi->machine_index, mi->current_state, s);
+
+    /* If this is not the same state, switch and init tokens,
+     * otherwise just reschedule padding. */
+    if (mi->current_state != s) {
+      mi->current_state = s;
+      circpad_machine_setup_tokens(mi);
+      circpad_choose_state_length(mi);
+
+      /* If we transition to the end state, check to see
+       * if this machine wants to be shut down at end */
+      if (s == CIRCPAD_STATE_END) {
+        circpad_machine_spec_transitioned_to_end(mi);
+        /* We transitioned but we don't pad in end. Also, mi
+         * may be freed. Returning STATE_CHANGED prevents us
+         * from accessing it in any callers of this function. */
+        return CIRCPAD_STATE_CHANGED;
+      }
+
+      /* We transitioned to a new state, schedule padding */
+      circpad_machine_schedule_padding(mi);
+      return CIRCPAD_STATE_CHANGED;
+    }
+
+    /* We transitioned back to the same state. Schedule padding,
+     * and inform if that causes a state transition. */
+    return circpad_machine_schedule_padding(mi);
+  }
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/**
+ * Estimate the circuit RTT from the current middle hop out to the
+ * end of the circuit.
+ *
+ * We estimate RTT by calculating the time between "receive" and
+ * "send" at a middle hop. This is because we "receive" a cell
+ * from the origin, and then relay it towards the exit before a
+ * response comes back. It is that response time from the exit side
+ * that we want to measure, so that we can make use of it for synthetic
+ * response delays.
+ */
+static void
+circpad_estimate_circ_rtt_on_received(circuit_t *circ,
+                                      circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ) || mi->stop_rtt_update)
+    return;
+
+  /* If we already have a last received packet time, that means we
+   * did not get a response before this packet. The RTT estimate
+   * only makes sense if we do not have multiple packets on the
+   * wire, so stop estimating if this is the second packet
+   * back to back. However, for the first set of back-to-back
+   * packets, we can wait until the very first response comes back
+   * to us, to measure that RTT (for the response to optimistic
+   * data, for example). Hence stop_rtt_update is only checked
+   * in this received side function, and not in send side below.
+   */
+  if (mi->last_received_time_usec) {
+    /* We also allow multiple back-to-back packets if the circuit is not
+     * opened, to handle var cells.
+     * XXX: Will this work with out var cell plans? Maybe not,
+     * since we're opened at the middle hop as soon as we process
+     * one var extend2 :/ */
+    if (circ->state == CIRCUIT_STATE_OPEN) {
+      log_fn(LOG_INFO, LD_CIRC,
+           "Stopping padding RTT estimation on circuit (%"PRIu64
+           ", %d) after two back to back packets. Current RTT: %d",
+           circ->n_chan ?  circ->n_chan->global_identifier : 0,
+           circ->n_circ_id, mi->rtt_estimate_usec);
+      mi->stop_rtt_update = 1;
+
+      if (!mi->rtt_estimate_usec) {
+        static ratelim_t rtt_lim = RATELIM_INIT(600);
+        log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+          "Circuit got two cells back to back before estimating RTT.");
+      }
+    }
+  } else {
+    const circpad_state_t *state = circpad_machine_current_state(mi);
+    if (BUG(!state)) {
+      return;
+    }
+
+    /* Since monotime is unpredictably expensive, only update this field
+     * if rtt estimates are needed. Otherwise, stop the rtt update. */
+    if (state->use_rtt_estimate) {
+      mi->last_received_time_usec = monotime_absolute_usec();
+    } else {
+      /* Let's fast-path future decisions not to update rtt if the
+       * feature is not in use. */
+      mi->stop_rtt_update = 1;
+    }
+  }
+}
+
+/**
+ * Handles the "send" side of RTT calculation at middle nodes.
+ *
+ * This function calculates the RTT from the middle to the end
+ * of the circuit by subtracting the last received cell timestamp
+ * from the current time. It allows back-to-back cells until
+ * the circuit is opened, to allow for var cell handshakes.
+ * XXX: Check our var cell plans to make sure this will work.
+ */
+static void
+circpad_estimate_circ_rtt_on_send(circuit_t *circ,
+                                  circpad_machine_runtime_t *mi)
+{
+  /* Origin circuits don't estimate RTT. They could do it easily enough,
+   * but they have no reason to use it in any delay calculations. */
+  if (CIRCUIT_IS_ORIGIN(circ))
+    return;
+
+  /* If last_received_time_usec is non-zero, we are waiting for a response
+   * from the exit side. Calculate the time delta and use it as RTT. */
+  if (mi->last_received_time_usec) {
+    circpad_time_t rtt_time = monotime_absolute_usec() -
+        mi->last_received_time_usec;
+
+    /* Reset the last RTT packet time, so we can tell if two cells
+     * arrive back to back */
+    mi->last_received_time_usec = 0;
+
+    /* Use INT32_MAX to ensure the addition doesn't overflow */
+    if (rtt_time >= INT32_MAX) {
+      log_fn(LOG_WARN,LD_CIRC,
+             "Circuit padding RTT estimate overflowed: %"PRIu64
+             " vs %"PRIu64, monotime_absolute_usec(),
+               mi->last_received_time_usec);
+      return;
+    }
+
+    /* If the old RTT estimate is lower than this one, use this one, because
+     * the circuit is getting longer. If this estimate is somehow
+     * faster than the previous, then maybe that was network jitter, or a
+     * bad monotonic clock source (so our ratchet returned a zero delta).
+     * In that case, average them. */
+    if (mi->rtt_estimate_usec < (circpad_delay_t)rtt_time) {
+      mi->rtt_estimate_usec = (circpad_delay_t)rtt_time;
+    } else {
+      mi->rtt_estimate_usec += (circpad_delay_t)rtt_time;
+      mi->rtt_estimate_usec /= 2;
+    }
+  } else if (circ->state == CIRCUIT_STATE_OPEN) {
+    /* If last_received_time_usec is zero, then we have gotten two cells back
+     * to back. Stop estimating RTT in this case. Note that we only
+     * stop RTT update if the circuit is opened, to allow for RTT estimates
+     * of var cells during circ setup. */
+    if (!mi->rtt_estimate_usec && !mi->stop_rtt_update) {
+      static ratelim_t rtt_lim = RATELIM_INIT(600);
+      log_fn_ratelim(&rtt_lim,LOG_NOTICE,LD_BUG,
+        "Circuit sent two cells back to back before estimating RTT.");
+    }
+    mi->stop_rtt_update = 1;
+  }
+}
+
+/**
+ * A "non-padding" cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_sent(circuit_t *on_circ)
+{
+  /* Update global cell count */
+  circpad_global_nonpadding_sent++;
+
+  /* If there are no machines then this loop should not iterate */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_send(on_circ, on_circ->padding_info[i]);
+
+    /* Then, do accounting */
+    circpad_machine_count_nonpadding_sent(on_circ->padding_info[i]);
+
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                                 CIRCPAD_EVENT_NONPADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/** Check if this cell or circuit are related to circuit padding and handle
+ *  them if so.  Return 0 if the cell was handled in this subsystem and does
+ *  not need any other consideration, otherwise return 1.
+ */
+int
+circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                            crypt_path_t *layer_hint,
+                            const relay_header_t *rh)
+{
+  /* First handle the padding commands, since we want to ignore any other
+   * commands if this circuit is padding-specific. */
+  switch (rh->command) {
+    case RELAY_COMMAND_DROP:
+      /* Already examined in circpad_deliver_recognized_relay_cell_events */
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATE:
+      circpad_handle_padding_negotiate(circ, cell);
+      return 0;
+    case RELAY_COMMAND_PADDING_NEGOTIATED:
+      if (circpad_handle_padding_negotiated(circ, cell, layer_hint) == 0)
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
+      return 0;
+  }
+
+  /* If this is a padding circuit we don't need to parse any other commands
+   * than the padding ones. Just drop them to the floor.
+   *
+   * Note: we deliberately do not call circuit_read_valid_data() here. The
+   * vanguards addon (specifically the 'bandguards' component's dropped cell
+   * detection) will thus close this circuit, as it would for any other
+   * unexpected cell. However, default tor will *not* close the circuit.
+   *
+   * This is intentional. We are not yet certain that is it optimal to keep
+   * padding circuits open in cases like these, rather than closing them.
+   * We suspect that continuing to pad is optimal against a passive classifier,
+   * but as soon as the adversary is active (even as a client adversary) this
+   * might change.
+   *
+   * So as a way forward, we log the cell command and circuit number, to
+   * help us enumerate the most common instances of this in testing with
+   * vanguards, to see which are common enough to verify and handle
+   * properly.
+   * - Mike
+   */
+  if (circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Ignored cell (%d) that arrived in padding circuit "
+                      " %u.", rh->command, CIRCUIT_IS_ORIGIN(circ) ?
+                           TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+    return 0;
+  }
+
+  return 1;
+}
+
+/**
+ * A "non-padding" cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_nonpadding_received(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* First, update any timestamps */
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_estimate_circ_rtt_on_received(on_circ, on_circ->padding_info[i]);
+
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                               CIRCPAD_EVENT_NONPADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been sent from this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we sent a cell into the network.
+ * For middle relay circuits, this means we sent a cell towards the
+ * origin.
+ */
+void
+circpad_cell_event_padding_sent(circuit_t *on_circ)
+{
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    /* Check to see if we've run out of tokens for this state already,
+     * and if not, check for other state transitions */
+    if (check_machine_token_supply(on_circ->padding_info[i])
+        == CIRCPAD_STATE_UNCHANGED) {
+      /* If removing a token did not cause a transition, check if
+       * non-padding sent event should */
+
+      on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+      circpad_machine_spec_transition(on_circ->padding_info[i],
+                             CIRCPAD_EVENT_PADDING_SENT);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * A padding cell has been received by this endpoint. React
+ * according to any padding state machines on the circuit.
+ *
+ * For origin circuits, this means we read a cell from the network.
+ * For middle relay circuits, this means we received a cell from the
+ * origin.
+ */
+void
+circpad_cell_event_padding_received(circuit_t *on_circ)
+{
+  /* identical to padding sent */
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, on_circ) {
+    on_circ->padding_info[i]->last_cell_time_sec = approx_time();
+    circpad_machine_spec_transition(on_circ->padding_info[i],
+                              CIRCPAD_EVENT_PADDING_RECV);
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * An "infinite" delay has ben chosen from one of our histograms.
+ *
+ * "Infinite" delays mean don't send padding -- but they can also
+ * mean transition to another state depending on the state machine
+ * definitions. Check the rules and react accordingly.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_INFINITY);
+}
+
+/**
+ * All of the bins of our current state's histogram's are empty.
+ *
+ * Check to see if this means transition to another state, and if
+ * not, refill the tokens.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *mi)
+{
+  if (circpad_machine_spec_transition(mi, CIRCPAD_EVENT_BINS_EMPTY)
+      == CIRCPAD_STATE_CHANGED) {
+    return CIRCPAD_STATE_CHANGED;
+  } else {
+    /* If we dont transition, then we refill the tokens */
+    circpad_machine_setup_tokens(mi);
+    return CIRCPAD_STATE_UNCHANGED;
+  }
+}
+
+/**
+ * This state has used up its cell count. Emit the event and
+ * see if we transition.
+ *
+ * Return 1 if we decide to transition, 0 otherwise.
+ */
+circpad_decision_t
+circpad_internal_event_state_length_up(circpad_machine_runtime_t *mi)
+{
+  return circpad_machine_spec_transition(mi, CIRCPAD_EVENT_LENGTH_COUNT);
+}
+
+/**
+ * Returns true if the circuit matches the conditions.
+ */
+static inline bool
+circpad_machine_conditions_met(origin_circuit_t *circ,
+                               const circpad_machine_spec_t *machine)
+{
+  /* If padding is disabled, no machines should match/apply. This has
+   * the effect of shutting down all machines, and not adding any more. */
+  if (circpad_padding_disabled || !get_options()->CircuitPadding)
+    return 0;
+
+  /* If the consensus or our torrc has selected reduced connection padding,
+   * then only allow this machine if it is flagged as acceptable under
+   * reduced padding conditions */
+  if (circpad_padding_reduced || get_options()->ReducedCircuitPadding) {
+    if (!machine->conditions.reduced_padding_ok)
+      return 0;
+  }
+
+  if (!(circpad_circ_purpose_to_mask(TO_CIRCUIT(circ)->purpose)
+      & machine->conditions.purpose_mask))
+    return 0;
+
+  if (machine->conditions.requires_vanguards) {
+    const or_options_t *options = get_options();
+
+    /* Pinned middles are effectively vanguards */
+    if (!(options->HSLayer2Nodes || options->HSLayer3Nodes))
+      return 0;
+  }
+
+  /* We check for any bits set in the circuit state mask so that machines
+   * can say any of the following through their state bitmask:
+   * "I want to apply to circuits with either streams or no streams"; OR
+   * "I only want to apply to circuits with streams"; OR
+   * "I only want to apply to circuits without streams". */
+  if (!(circpad_circuit_state(circ) & machine->conditions.state_mask))
+    return 0;
+
+  if (circuit_get_cpath_opened_len(circ) < machine->conditions.min_hops)
+    return 0;
+
+  return 1;
+}
+
+/**
+ * Returns a minimized representation of the circuit state.
+ *
+ * The padding code only cares if the circuit is building,
+ * opened, used for streams, and/or still has relay early cells.
+ * This returns a bitmask of all state properities that apply to
+ * this circuit.
+ */
+static inline
+circpad_circuit_state_t
+circpad_circuit_state(origin_circuit_t *circ)
+{
+  circpad_circuit_state_t retmask = 0;
+
+  if (circ->p_streams)
+    retmask |= CIRCPAD_CIRC_STREAMS;
+  else
+    retmask |= CIRCPAD_CIRC_NO_STREAMS;
+
+  /* We use has_opened to prevent cannibialized circs from flapping. */
+  if (circ->has_opened)
+    retmask |= CIRCPAD_CIRC_OPENED;
+  else
+    retmask |= CIRCPAD_CIRC_BUILDING;
+
+  if (circ->remaining_relay_early_cells > 0)
+    retmask |= CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  else
+    retmask |= CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+
+  return retmask;
+}
+
+/**
+ * Convert a normal circuit purpose into a bitmask that we can
+ * use for determining matching circuits.
+ */
+circpad_purpose_mask_t
+circpad_circ_purpose_to_mask(uint8_t circ_purpose)
+{
+  /* Treat OR circ purposes as ignored. They should not be passed here*/
+  if (BUG(circ_purpose <= CIRCUIT_PURPOSE_OR_MAX_)) {
+    return 0;
+  }
+
+  /* Treat new client circuit purposes as "OMG ITS EVERYTHING".
+   * This also should not happen */
+  if (BUG(circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1 > 32)) {
+    return CIRCPAD_PURPOSE_ALL;
+  }
+
+  /* Convert the purpose to a bit position */
+  return 1 << (circ_purpose - CIRCUIT_PURPOSE_OR_MAX_ - 1);
+}
+
+/**
+ * Shut down any machines whose conditions no longer match
+ * the current circuit.
+ */
+static void
+circpad_shutdown_old_machines(origin_circuit_t *on_circ)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+  FOR_EACH_ACTIVE_CIRCUIT_MACHINE_BEGIN(i, circ) {
+    if (!circpad_machine_conditions_met(on_circ,
+                                        circ->padding_machine[i])) {
+      // Clear machineinfo (frees timers)
+      circpad_circuit_machineinfo_free_idx(circ, i);
+      // Send padding negotiate stop
+      circpad_negotiate_padding(on_circ,
+                                circ->padding_machine[i]->machine_num,
+                                circ->padding_machine[i]->target_hopnum,
+                                CIRCPAD_COMMAND_STOP);
+    }
+  } FOR_EACH_ACTIVE_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Negotiate new machines that would apply to this circuit, given the machines
+ * inside <b>machines_sl</b>.
+ *
+ * This function checks to see if we have any free machine indexes,
+ * and for each free machine index, it initializes the most recently
+ * added origin-side padding machine that matches the target machine
+ * index and circuit conditions, and negotiates it with the appropriate
+ * middle relay.
+ */
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl)
+{
+  circuit_t *circ = TO_CIRCUIT(on_circ);
+
+#ifdef TOR_UNIT_TESTS
+  /* Tests don't have to init our padding machines */
+  if (!machines_sl)
+    return;
+#endif
+
+  /* If padding negotiation failed before, do not try again */
+  if (on_circ->padding_negotiation_failed)
+    return;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* If there is a padding machine info, this index is occupied.
+     * No need to check conditions for this index. */
+    if (circ->padding_info[i])
+      continue;
+
+    /* We have a free machine index. Check the origin padding
+     * machines in reverse order, so that more recently added
+     * machines take priority over older ones. */
+    SMARTLIST_FOREACH_REVERSE_BEGIN(machines_sl,
+                                    circpad_machine_spec_t *,
+                                    machine) {
+      /* Machine definitions have a specific target machine index.
+       * This is so event ordering is deterministic with respect
+       * to which machine gets events first when there are two
+       * machines installed on a circuit. Make sure we only
+       * add this machine if its target machine index is free. */
+      if (machine->machine_index == i &&
+          circpad_machine_conditions_met(on_circ, machine)) {
+
+        // We can only replace this machine if the target hopnum
+        // is the same, otherwise we'll get invalid data
+        if (circ->padding_machine[i]) {
+          if (circ->padding_machine[i]->target_hopnum !=
+              machine->target_hopnum)
+            continue;
+          /* Replace it. (Don't free - is global). */
+          circ->padding_machine[i] = NULL;
+        }
+
+        /* Set up the machine immediately so that the slot is occupied.
+         * We will tear it down on error return, or if there is an error
+         * response from the relay. */
+        circpad_setup_machine_on_circ(circ, machine);
+        if (circpad_negotiate_padding(on_circ, machine->machine_num,
+                                  machine->target_hopnum,
+                                  CIRCPAD_COMMAND_START) < 0) {
+          log_info(LD_CIRC,
+                   "Padding not negotiated. Cleaning machine from circuit %u",
+             CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+          circpad_circuit_machineinfo_free_idx(circ, i);
+          circ->padding_machine[i] = NULL;
+          on_circ->padding_negotiation_failed = 1;
+        } else {
+          /* Success. Don't try any more machines on this index */
+          break;
+        }
+      }
+    } SMARTLIST_FOREACH_END(machine);
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+}
+
+/**
+ * Event that tells us we added a hop to an origin circuit.
+ *
+ * This event is used to decide if we should create a padding machine
+ * on a circuit.
+ */
+void
+circpad_machine_event_circ_added_hop(origin_circuit_t *on_circ)
+{
+  /* Since our padding conditions do not specify a max_hops,
+   * all we can do is add machines here */
+  circpad_add_matching_machines(on_circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is now built.
+ *
+ * Shut down any machines that only applied to un-built circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_built(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Circpad purpose changed event.
+ *
+ * Shut down any machines that don't apply to our circ purpose.
+ * Activate any new ones that do.
+ */
+void
+circpad_machine_event_circ_purpose_changed(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Event that tells us that an origin circuit is out of RELAY_EARLY
+ * cells.
+ *
+ * Shut down any machines that only applied to RELAY_EARLY circuits.
+ * Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_relay_early(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams attached event.
+ *
+ * Called from link_apconn_to_circ() and handle_hs_exit_conn()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Streams detached event.
+ *
+ * Called from circuit_detach_stream()
+ *
+ * Shut down any machines that only applied to machines without
+ * streams. Activate any new ones.
+ */
+void
+circpad_machine_event_circ_has_no_streams(origin_circuit_t *circ)
+{
+  circpad_shutdown_old_machines(circ);
+  circpad_add_matching_machines(circ, origin_padding_machines);
+}
+
+/**
+ * Verify that padding is coming from the expected hop.
+ *
+ * Returns true if from_hop matches the target hop from
+ * one of our padding machines.
+ *
+ * Returns false if we're not an origin circuit, or if from_hop
+ * does not match one of the padding machines.
+ */
+bool
+circpad_padding_is_from_expected_hop(circuit_t *circ,
+                                     crypt_path_t *from_hop)
+{
+  crypt_path_t *target_hop = NULL;
+  if (!CIRCUIT_IS_ORIGIN(circ))
+    return 0;
+
+  FOR_EACH_CIRCUIT_MACHINE_BEGIN(i) {
+    /* We have to check padding_machine and not padding_info/active
+     * machines here because padding may arrive after we shut down a
+     * machine. The info is gone, but the padding_machine waits
+     * for the padding_negotiated response to come back. */
+    if (!circ->padding_machine[i])
+      continue;
+
+    target_hop = circuit_get_cpath_hop(TO_ORIGIN_CIRCUIT(circ),
+                    circ->padding_machine[i]->target_hopnum);
+
+    if (target_hop == from_hop)
+      return 1;
+  } FOR_EACH_CIRCUIT_MACHINE_END;
+
+  return 0;
+}
+
+/**
+ * Deliver circpad events for an "unrecognized cell".
+ *
+ * Unrecognized cells are sent to relays and are forwarded
+ * onto the next hop of their circuits. Unrecognized cells
+ * are by definition not padding. We need to tell relay-side
+ * state machines that a non-padding cell was sent or received,
+ * depending on the direction, so they can update their histograms
+ * and decide to pad or not.
+ */
+void
+circpad_deliver_unrecognized_cell_events(circuit_t *circ,
+                                         cell_direction_t dir)
+{
+  // We should never see unrecognized cells at origin.
+  // Our caller emits a warn when this happens.
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    return;
+  }
+
+  if (dir == CELL_DIRECTION_OUT) {
+    /* When direction is out (away from origin), then we received non-padding
+       cell coming from the origin to us. */
+    circpad_cell_event_nonpadding_received(circ);
+  } else if (dir == CELL_DIRECTION_IN) {
+    /* It's in and not origin, so the cell is going away from us.
+     * So we are relaying a non-padding cell towards the origin. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for "recognized" relay cells.
+ *
+ * Recognized cells are destined for this hop, either client or middle.
+ * Check if this is a padding cell or not, and send the appropiate
+ * received event.
+ */
+void
+circpad_deliver_recognized_relay_cell_events(circuit_t *circ,
+                                             uint8_t relay_command,
+                                             crypt_path_t *layer_hint)
+{
+  if (relay_command == RELAY_COMMAND_DROP) {
+    rep_hist_padding_count_read(PADDING_TYPE_DROP);
+
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      if (circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), 0);
+      } else {
+        /* This is unexpected padding. Ignore it for now. */
+        return;
+      }
+    }
+
+    /* The cell should be recognized by now, which means that we are on the
+       destination, which means that we received a padding cell. We might be
+       the client or the Middle node, still, because leaky-pipe. */
+    circpad_cell_event_padding_received(circ);
+    log_fn(LOG_INFO, LD_CIRC, "Got padding cell on %s circuit %u.",
+           CIRCUIT_IS_ORIGIN(circ) ? "origin" : "non-origin",
+           CIRCUIT_IS_ORIGIN(circ) ?
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier : 0);
+  } else {
+    /* We received a non-padding cell on the edge */
+    circpad_cell_event_nonpadding_received(circ);
+  }
+}
+
+/**
+ * Deliver circpad events for relay cells sent from us.
+ *
+ * If this is a padding cell, update our padding stats
+ * and deliver the event. Otherwise just deliver the event.
+ */
+void
+circpad_deliver_sent_relay_cell_events(circuit_t *circ,
+                                       uint8_t relay_command)
+{
+  /* RELAY_COMMAND_DROP is the multi-hop (aka circuit-level) padding cell in
+   * tor. (CELL_PADDING is a channel-level padding cell, which is not relayed
+   * or processed here).
+   *
+   * We do generate events for PADDING_NEGOTIATE and PADDING_NEGOTIATED cells.
+   */
+  if (relay_command == RELAY_COMMAND_DROP) {
+    /* Optimization: The event for RELAY_COMMAND_DROP is sent directly
+     * from circpad_send_padding_cell_for_callback(). This is to avoid
+     * putting a cell_t and a relay_header_t on the stack repeatedly
+     * if we decide to send a long train of padding cells back-to-back
+     * with 0 delay. So we do nothing here. */
+    return;
+  } else {
+    /* This is a non-padding cell sent from the client or from
+     * this node. */
+    circpad_cell_event_nonpadding_sent(circ);
+  }
+}
+
+/**
+ * Initialize the states array for a circpad machine.
+ */
+void
+circpad_machine_states_init(circpad_machine_spec_t *machine,
+                            circpad_statenum_t num_states)
+{
+  if (BUG(num_states > CIRCPAD_MAX_MACHINE_STATES)) {
+    num_states = CIRCPAD_MAX_MACHINE_STATES;
+  }
+
+  machine->num_states = num_states;
+  machine->states = tor_malloc_zero(sizeof(circpad_state_t)*num_states);
+
+  /* Initialize the default next state for all events to
+   * "ignore" -- if events aren't specified, they are ignored. */
+  for (circpad_statenum_t s = 0; s < num_states; s++) {
+    for (int e = 0; e < CIRCPAD_NUM_EVENTS; e++) {
+      machine->states[s].next_state[e] = CIRCPAD_STATE_IGNORE;
+    }
+  }
+}
+
+static void
+circpad_setup_machine_on_circ(circuit_t *on_circ,
+                              const circpad_machine_spec_t *machine)
+{
+  if (CIRCUIT_IS_ORIGIN(on_circ) && !machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up non-origin machine on origin circuit!");
+    return;
+  }
+
+  if (!CIRCUIT_IS_ORIGIN(on_circ) && machine->is_origin_side) {
+    log_fn(LOG_WARN, LD_BUG,
+           "Can't set up origin machine on non-origin circuit!");
+    return;
+  }
+
+  IF_BUG_ONCE(on_circ->padding_machine[machine->machine_index] != NULL) {
+    return;
+  }
+  IF_BUG_ONCE(on_circ->padding_info[machine->machine_index] != NULL) {
+    return;
+  }
+
+  /* Log message */
+  if (CIRCUIT_IS_ORIGIN(on_circ)) {
+    log_info(LD_CIRC, "Registering machine %s to origin circ %u (%d)",
+             machine->name,
+             TO_ORIGIN_CIRCUIT(on_circ)->global_identifier, on_circ->purpose);
+  } else {
+    log_info(LD_CIRC, "Registering machine %s to non-origin circ (%d)",
+             machine->name, on_circ->purpose);
+  }
+
+  on_circ->padding_info[machine->machine_index] =
+      circpad_circuit_machineinfo_new(on_circ, machine->machine_index);
+  on_circ->padding_machine[machine->machine_index] = machine;
+}
+
+/** Validate a single state of a padding machine */
+static bool
+padding_machine_state_is_valid(const circpad_state_t *state)
+{
+  int b;
+  uint32_t tokens_count = 0;
+  circpad_delay_t prev_bin_edge = 0;
+
+  /* We only validate histograms */
+  if (!state->histogram_len) {
+    return true;
+  }
+
+  /* We need at least two bins in a histogram */
+  if (state->histogram_len < 2) {
+    log_warn(LD_CIRC, "You can't have a histogram with less than 2 bins");
+    return false;
+  }
+
+  /* For each machine state, if it's a histogram, make sure all the
+   * histogram edges are well defined (i.e. are strictly monotonic). */
+  for (b = 0 ; b < state->histogram_len ; b++) {
+    /* Check that histogram edges are strictly increasing. Ignore the first
+     * edge since it can be zero. */
+    if (prev_bin_edge >= state->histogram_edges[b] && b > 0) {
+      log_warn(LD_CIRC, "Histogram edges are not increasing [%u/%u]",
+               prev_bin_edge, state->histogram_edges[b]);
+      return false;
+    }
+
+    prev_bin_edge = state->histogram_edges[b];
+
+    /* Also count the number of tokens as we go through the histogram states */
+    tokens_count += state->histogram[b];
+  }
+  /* Verify that the total number of tokens is correct */
+  if (tokens_count != state->histogram_total_tokens) {
+    log_warn(LD_CIRC, "Histogram token count is wrong [%u/%u]",
+             tokens_count, state->histogram_total_tokens);
+    return false;
+  }
+
+  return true;
+}
+
+/** Basic validation of padding machine */
+static bool
+padding_machine_is_valid(const circpad_machine_spec_t *machine)
+{
+  int i;
+
+  /* Validate the histograms of the padding machine */
+  for (i = 0 ; i < machine->num_states ; i++) {
+    if (!padding_machine_state_is_valid(&machine->states[i])) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/* Validate and register <b>machine</b> into <b>machine_list</b>. If
+ * <b>machine_list</b> is NULL, then just validate. */
+void
+circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                 smartlist_t *machine_list)
+{
+  if (!padding_machine_is_valid(machine)) {
+    log_warn(LD_CIRC, "Machine #%u is invalid. Ignoring.",
+             machine->machine_num);
+    return;
+  }
+
+  if (machine_list) {
+    smartlist_add(machine_list, machine);
+  }
+}
+
+#ifdef TOR_UNIT_TESTS
+/* These padding machines are only used for tests pending #28634. */
+static void
+circpad_circ_client_machine_init(void)
+{
+  circpad_machine_spec_t *circ_client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  circ_client_machine->conditions.min_hops = 2;
+  circ_client_machine->conditions.state_mask =
+      CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  circ_client_machine->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circ_client_machine->conditions.reduced_padding_ok = 1;
+
+  circ_client_machine->target_hopnum = 2;
+  circ_client_machine->is_origin_side = 1;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_client_machine, 3);
+
+  circ_client_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* If we are in burst state, and we send a non-padding cell, then we cancel
+     the timer for the next padding cell:
+     We dont want to send fake extends when actual extends are going on */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1]= 1000000;
+
+  /* We have 5 tokens in the histogram, which means that all circuits will look
+   * like they have 7 hops (since we start this machine after the second hop,
+   * and tokens are decremented for any valid hops, and fake extends are
+   * used after that -- 2+5==7). */
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 5;
+
+  circ_client_machine->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 5;
+
+  circ_client_machine->machine_num = smartlist_len(origin_padding_machines);
+  circpad_register_padding_machine(circ_client_machine,
+                                   origin_padding_machines);
+}
+
+static void
+circpad_circ_responder_machine_init(void)
+{
+  circpad_machine_spec_t *circ_responder_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Shut down the machine after we've sent enough packets */
+  circ_responder_machine->should_negotiate_end = 1;
+
+  /* The relay-side doesn't care what hopnum it is, but for consistency,
+   * let's match the client */
+  circ_responder_machine->target_hopnum = 2;
+  circ_responder_machine->is_origin_side = 0;
+
+  /* Start, gap, burst */
+  circpad_machine_states_init(circ_responder_machine, 3);
+
+  /* This is the settings of the state machine. In the future we are gonna
+     serialize this into the consensus or the torrc */
+
+  /* We transition to the burst state on padding receive and on non-padding
+   * recieve */
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_responder_machine->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we _stay_ in the burst state when a non-padding
+   * is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  /* Inside the burst state we transition to the gap state when we receive a
+   * padding cell */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* These describe the padding charasteristics when in burst state */
+
+  /* use_rtt_estimate tries to estimate how long padding cells take to go from
+     C->M, and uses that as what as the base of the histogram */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+  /* The histogram is 2 bins: an empty one, and infinity */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_len = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram_edges[1] =
+    1000000;
+  /* During burst state we wait forever for padding to arrive.
+
+     We are waiting for a padding cell from the client to come in, so that we
+     respond, and we immitate how extend looks like */
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  // Only infinity bin:
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_BURST].
+      histogram_total_tokens = 1;
+
+  /* From the gap state, we _stay_ in the gap state, when we receive padding
+   * or non padding */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_GAP;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_GAP;
+
+  /* And from the gap state, we go to the end, when the bins are empty or a
+   * non-padding cell is sent */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_END;
+
+  // FIXME: Tune this histogram
+
+  /* The gap state is the delay you wait after you receive a padding cell
+     before you send a padding response */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].use_rtt_estimate = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_len = 6;
+  /* Specify histogram bins */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[0]= 500;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[1]= 1000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[2]= 2000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[3]= 4000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[4]= 8000;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_edges[5]= 16000;
+  /* Specify histogram tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[0] = 0;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[1] = 1;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[2] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[3] = 2;
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram[4] = 1;
+  /* Total number of tokens */
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].histogram_total_tokens = 6;
+
+  circ_responder_machine->states[CIRCPAD_STATE_GAP].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC;
+
+  circ_responder_machine->machine_num = smartlist_len(relay_padding_machines);
+  circpad_register_padding_machine(circ_responder_machine,
+                                   relay_padding_machines);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize all of our padding machines.
+ *
+ * This is called at startup. It sets up some global machines, and then
+ * loads some from torrc, and from the tor consensus.
+ */
+void
+circpad_machines_init(void)
+{
+  tor_assert_nonfatal(origin_padding_machines == NULL);
+  tor_assert_nonfatal(relay_padding_machines == NULL);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  /* Register machines for hiding client-side intro circuits */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+
+  /* Register machines for hiding client-side rendezvous circuits */
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  // TODO: Parse machines from consensus and torrc
+#ifdef TOR_UNIT_TESTS
+  circpad_circ_client_machine_init();
+  circpad_circ_responder_machine_init();
+#endif
+}
+
+/**
+ * Free our padding machines
+ */
+void
+circpad_machines_free(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH(origin_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(origin_padding_machines);
+  }
+
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH(relay_padding_machines,
+                      circpad_machine_spec_t *,
+                      m, tor_free(m->states); tor_free(m));
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/**
+ * Check the Protover info to see if a node supports padding.
+ */
+static bool
+circpad_node_supports_padding(const node_t *node)
+{
+  if (node->rs) {
+    log_fn(LOG_INFO, LD_CIRC, "Checking padding: %s",
+           node->rs->pv.supports_hs_setup_padding ?
+              "supported" : "unsupported");
+    return node->rs->pv.supports_hs_setup_padding;
+  }
+
+  log_fn(LOG_INFO, LD_CIRC, "Empty routerstatus in padding check");
+  return 0;
+}
+
+/**
+ * Get a node_t for the nth hop in our circuit, starting from 1.
+ *
+ * Returns node_t from the consensus for that hop, if it is opened.
+ * Otherwise returns NULL.
+ */
+MOCK_IMPL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop))
+{
+  crypt_path_t *iter = circuit_get_cpath_hop(circ, hop);
+
+  if (!iter || iter->state != CPATH_STATE_OPEN)
+    return NULL;
+
+  return node_get_by_id(iter->extend_info->identity_digest);
+}
+
+/**
+ * Return true if a particular circuit supports padding
+ * at the desired hop.
+ */
+static bool
+circpad_circuit_supports_padding(origin_circuit_t *circ,
+                                 int target_hopnum)
+{
+  const node_t *hop;
+
+  if (!(hop = circuit_get_nth_node(circ, target_hopnum))) {
+    return 0;
+  }
+
+  return circpad_node_supports_padding(hop);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_negotiate_padding(origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command)
+{
+  circpad_negotiate_t type;
+  cell_t cell;
+  ssize_t len;
+
+  /* Check that the target hop lists support for padding in
+   * its ProtoVer fields */
+  if (!circpad_circuit_supports_padding(circ, target_hopnum)) {
+    return -1;
+  }
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiate_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiate_set_command(&type, command);
+  circpad_negotiate_set_version(&type, 0);
+  circpad_negotiate_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiate_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return -1;
+
+  log_fn(LOG_INFO,LD_CIRC,
+         "Negotiating padding on circuit %u (%d), command %d",
+         circ->global_identifier, TO_CIRCUIT(circ)->purpose, command);
+
+  return circpad_send_command_to_hop(circ, target_hopnum,
+                                     RELAY_COMMAND_PADDING_NEGOTIATE,
+                                     cell.payload, len);
+}
+
+/**
+ * Try to negotiate padding.
+ *
+ * Returns 1 if successful (or already set up), 0 otherwise.
+ */
+bool
+circpad_padding_negotiated(circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response)
+{
+  circpad_negotiated_t type;
+  cell_t cell;
+  ssize_t len;
+
+  memset(&cell, 0, sizeof(cell_t));
+  memset(&type, 0, sizeof(circpad_negotiated_t));
+  // This gets reset to RELAY_EARLY appropriately by
+  // relay_send_command_from_edge_. At least, it looks that way.
+  // QQQ-MP-AP: Verify that.
+  cell.command = CELL_RELAY;
+
+  circpad_negotiated_set_command(&type, command);
+  circpad_negotiated_set_response(&type, response);
+  circpad_negotiated_set_version(&type, 0);
+  circpad_negotiated_set_machine_type(&type, machine);
+
+  if ((len = circpad_negotiated_encode(cell.payload, CELL_PAYLOAD_SIZE,
+        &type)) < 0)
+    return 0;
+
+  /* Use relay_send because we're from the middle to the origin. We don't
+   * need to specify a target hop or layer_hint. */
+  return relay_send_command_from_edge(0, circ,
+                                      RELAY_COMMAND_PADDING_NEGOTIATED,
+                                      (void*)cell.payload,
+                                      (size_t)len, NULL) == 0;
+}
+
+/**
+ * Parse and react to a padding_negotiate cell.
+ *
+ * This is called at the middle node upon receipt of the client's choice of
+ * state machine, so that it can use the requested state machine index, if
+ * it is available.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiate(circuit_t *circ, cell_t *cell)
+{
+  int retval = 0;
+  circpad_negotiate_t *negotiate;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiate cell unsupported at origin (circuit %u)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiate_parse(&negotiate, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATE cell; dropping.");
+    return -1;
+  }
+
+  if (negotiate->command == CIRCPAD_COMMAND_STOP) {
+    /* Free the machine corresponding to this machine type */
+    if (free_circ_machineinfos_with_machine_num(circ,
+                negotiate->machine_type)) {
+      log_info(LD_CIRC, "Received STOP command for machine %u",
+               negotiate->machine_type);
+      goto done;
+    }
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received circuit padding stop command for unknown machine.");
+    goto err;
+  } else if (negotiate->command == CIRCPAD_COMMAND_START) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            const circpad_machine_spec_t *, m) {
+      if (m->machine_num == negotiate->machine_type) {
+        circpad_setup_machine_on_circ(circ, m);
+        circpad_cell_event_nonpadding_received(circ);
+        goto done;
+      }
+    } SMARTLIST_FOREACH_END(m);
+  }
+
+  err:
+    retval = -1;
+
+  done:
+    circpad_padding_negotiated(circ, negotiate->machine_type,
+                   negotiate->command,
+                   (retval == 0) ? CIRCPAD_RESPONSE_OK : CIRCPAD_RESPONSE_ERR);
+    circpad_negotiate_free(negotiate);
+
+    return retval;
+}
+
+/**
+ * Parse and react to a padding_negotiated cell.
+ *
+ * This is called at the origin upon receipt of the middle's response
+ * to our choice of state machine.
+ *
+ * Returns -1 on error, 0 on success.
+ */
+signed_error_t
+circpad_handle_padding_negotiated(circuit_t *circ, cell_t *cell,
+                                  crypt_path_t *layer_hint)
+{
+  circpad_negotiated_t *negotiated;
+
+  if (!CIRCUIT_IS_ORIGIN(circ)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell unsupported at non-origin.");
+    return -1;
+  }
+
+  /* Verify this came from the expected hop */
+  if (!circpad_padding_is_from_expected_hop(circ, layer_hint)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Padding negotiated cell from wrong hop on circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (circpad_negotiated_parse(&negotiated, cell->payload+RELAY_HEADER_SIZE,
+                               CELL_PAYLOAD_SIZE-RELAY_HEADER_SIZE) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+          "Received malformed PADDING_NEGOTIATED cell on circuit %u; "
+          "dropping.", TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    return -1;
+  }
+
+  if (negotiated->command == CIRCPAD_COMMAND_STOP) {
+    log_info(LD_CIRC,
+             "Received STOP command on PADDING_NEGOTIATED for circuit %u",
+             TO_ORIGIN_CIRCUIT(circ)->global_identifier);
+    /* There may not be a padding_info here if we shut down the
+     * machine in circpad_shutdown_old_machines(). Or, if
+     * circpad_add_matching_matchines() added a new machine,
+     * there may be a padding_machine for a different machine num
+     * than this response. */
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+  } else if (negotiated->command == CIRCPAD_COMMAND_START &&
+             negotiated->response == CIRCPAD_RESPONSE_ERR) {
+    // This can happen due to consensus drift.. free the machines
+    // and be sad
+    free_circ_machineinfos_with_machine_num(circ, negotiated->machine_type);
+    TO_ORIGIN_CIRCUIT(circ)->padding_negotiation_failed = 1;
+    log_fn(LOG_PROTOCOL_WARN, LD_CIRC,
+           "Middle node did not accept our padding request on circuit %u (%d)",
+           TO_ORIGIN_CIRCUIT(circ)->global_identifier,
+           circ->purpose);
+  }
+
+  circpad_negotiated_free(negotiated);
+  return 0;
+}
+
+/** Free memory allocated by this machine spec. */
+STATIC void
+machine_spec_free_(circpad_machine_spec_t *m)
+{
+  if (!m) return;
+
+  tor_free(m->states);
+  tor_free(m);
+}
+
+/** Free all memory allocated by the circuitpadding subsystem. */
+void
+circpad_free_all(void)
+{
+  if (origin_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(origin_padding_machines);
+  }
+  if (relay_padding_machines) {
+    SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                            circpad_machine_spec_t *, m) {
+      machine_spec_free(m);
+    } SMARTLIST_FOREACH_END(m);
+    smartlist_free(relay_padding_machines);
+  }
+}
+
+/* Serialization */
+// TODO: Should we use keyword=value here? Are there helpers for that?
+#if 0
+static void
+circpad_state_serialize(const circpad_state_t *state,
+                        smartlist_t *chunks)
+{
+  smartlist_add_asprintf(chunks, " %u", state->histogram[0]);
+  for (int i = 1; i < state->histogram_len; i++) {
+    smartlist_add_asprintf(chunks, ",%u",
+                           state->histogram[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " 0x%x",
+                         state->transition_cancel_events);
+
+  for (int i = 0; i < CIRCPAD_NUM_STATES; i++) {
+    smartlist_add_asprintf(chunks, ",0x%x",
+                           state->transition_events[i]);
+  }
+
+  smartlist_add_asprintf(chunks, " %u %u",
+                         state->use_rtt_estimate,
+                         state->token_removal);
+}
+
+char *
+circpad_machine_spec_to_string(const circpad_machine_spec_t *machine)
+{
+  smartlist_t *chunks = smartlist_new();
+  char *out;
+  (void)machine;
+
+  circpad_state_serialize(&machine->start, chunks);
+  circpad_state_serialize(&machine->gap, chunks);
+  circpad_state_serialize(&machine->burst, chunks);
+
+  out = smartlist_join_strings(chunks, "", 0, NULL);
+
+  SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
+  smartlist_free(chunks);
+  return out;
+}
+
+// XXX: Writeme
+const circpad_machine_spec_t *
+circpad_string_to_machine(const char *str)
+{
+  (void)str;
+  return NULL;
+}
+
+#endif /* 0 */
diff --git a/src/core/or/circuitpadding.h b/src/core/or/circuitpadding.h
new file mode 100644
index 0000000000..74b69a1c7a
--- /dev/null
+++ b/src/core/or/circuitpadding.h
@@ -0,0 +1,813 @@
+/*
+ * Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding.h
+ * \brief Header file for circuitpadding.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_H
+#define TOR_CIRCUITPADDING_H
+
+#include "trunnel/circpad_negotiation.h"
+#include "lib/evloop/timers.h"
+
+struct circuit_t;
+struct origin_circuit_t;
+struct cell_t;
+
+/**
+ * Signed error return with the specific property that negative
+ * values mean error codes of various semantics, 0 means success,
+ * and positive values are unused.
+ *
+ * XXX: Tor uses this concept a lot but just calls it int. Should we move
+ * this somewhere centralized? Where?
+ */
+typedef int signed_error_t;
+
+/**
+ * These constants specify the types of events that can cause
+ * transitions between state machine states.
+ *
+ * Note that SENT and RECV are relative to this endpoint. For
+ * relays, SENT means packets destined towards the client and
+ * RECV means packets destined towards the relay. On the client,
+ * SENT means packets destined towards the relay, where as RECV
+ * means packets destined towards the client.
+ */
+typedef enum {
+  /* A non-padding cell was received. */
+  CIRCPAD_EVENT_NONPADDING_RECV = 0,
+  /* A non-padding cell was sent. */
+  CIRCPAD_EVENT_NONPADDING_SENT = 1,
+  /* A padding cell (RELAY_COMMAND_DROP) was sent. */
+  CIRCPAD_EVENT_PADDING_SENT = 2,
+  /* A padding cell was received. */
+  CIRCPAD_EVENT_PADDING_RECV = 3,
+  /* We tried to schedule padding but we ended up picking the infinity bin
+   * which means that padding was delayed infinitely */
+  CIRCPAD_EVENT_INFINITY = 4,
+  /* All histogram bins are empty (we are out of tokens) */
+  CIRCPAD_EVENT_BINS_EMPTY = 5,
+  /* This state has used up its cell count */
+  CIRCPAD_EVENT_LENGTH_COUNT = 6
+} circpad_event_t;
+#define CIRCPAD_NUM_EVENTS ((int)CIRCPAD_EVENT_LENGTH_COUNT+1)
+
+/** Boolean type that says if we decided to transition states or not */
+typedef enum {
+  CIRCPAD_STATE_UNCHANGED = 0,
+  CIRCPAD_STATE_CHANGED = 1
+} circpad_decision_t;
+
+/** The type for the things in histogram bins (aka tokens) */
+typedef uint32_t circpad_hist_token_t;
+
+/** The type for histogram indexes (needs to be negative for errors) */
+typedef int8_t circpad_hist_index_t;
+
+/** The type for absolute time, from monotime_absolute_usec() */
+typedef uint64_t circpad_time_t;
+
+/** The type for timer delays, in microseconds */
+typedef uint32_t circpad_delay_t;
+#define CIRCPAD_DELAY_UNITS_PER_SECOND  (1000*1000)
+
+/**
+ * An infinite padding cell delay means don't schedule any padding --
+ * simply wait until a different event triggers a transition.
+ *
+ * This means that the maximum delay we can schedule is UINT32_MAX-1
+ * microseconds, or about 4300 seconds (1.25 hours).
+ * XXX: Is this enough if we want to simulate light, intermittent
+ * activity on an onion service?
+ */
+#define CIRCPAD_DELAY_INFINITE  (UINT32_MAX)
+
+/**
+ * This is the maximum delay that the circuit padding system can have, in
+ * seconds.
+ */
+#define CIRCPAD_DELAY_MAX_SECS   \
+    ((CIRCPAD_DELAY_INFINITE/CIRCPAD_DELAY_UNITS_PER_SECOND)+1)
+
+/**
+ * Macro to clarify when we're checking the infinity bin.
+ *
+ * Works with either circpad_state_t or circpad_machine_runtime_t
+ */
+#define CIRCPAD_INFINITY_BIN(mi)  ((mi)->histogram_len-1)
+
+/**
+ * These constants form a bitfield that specifies when a state machine
+ * should be applied to a circuit.
+ *
+ * If any of these elements is set, then the circuit will be tested against
+ * that specific condition. If an element is unset, then we don't test it.
+ * (E.g., if neither NO_STREAMS or STREAMS are set, then we will not care
+ * whether a circuit has streams attached when we apply a state machine.)
+ *
+ * The helper function circpad_circuit_state() converts circuit state
+ * flags into this more compact representation.
+ */
+typedef enum {
+  /* Only apply machine if the circuit is still building */
+  CIRCPAD_CIRC_BUILDING = 1<<0,
+  /* Only apply machine if the circuit is open */
+  CIRCPAD_CIRC_OPENED = 1<<1,
+  /* Only apply machine if the circuit has no attached streams */
+  CIRCPAD_CIRC_NO_STREAMS = 1<<2,
+  /* Only apply machine if the circuit has attached streams */
+  CIRCPAD_CIRC_STREAMS = 1<<3,
+  /* Only apply machine if the circuit still allows RELAY_EARLY cells */
+  CIRCPAD_CIRC_HAS_RELAY_EARLY = 1<<4,
+  /* Only apply machine if the circuit has depleted its RELAY_EARLY cells
+   * allowance. */
+  CIRCPAD_CIRC_HAS_NO_RELAY_EARLY = 1<<5
+} circpad_circuit_state_t;
+
+/** Bitmask that says "apply this machine to all states" */
+#define CIRCPAD_STATE_ALL   \
+    (CIRCPAD_CIRC_BUILDING|CIRCPAD_CIRC_OPENED| \
+     CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_NO_STREAMS| \
+     CIRCPAD_CIRC_HAS_RELAY_EARLY|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY)
+
+/**
+ * A compact circuit purpose bitfield mask that allows us to compactly
+ * specify which circuit purposes a machine should apply to.
+ *
+ * The helper function circpad_circ_purpose_to_mask() converts circuit
+ * purposes into bit positions in this bitmask.
+ */
+typedef uint32_t circpad_purpose_mask_t;
+
+/** Bitmask that says "apply this machine to all purposes". */
+#define CIRCPAD_PURPOSE_ALL (0xFFFFFFFF)
+
+/**
+ * This type specifies all of the conditions that must be met before
+ * a client decides to initiate padding on a circuit.
+ *
+ * A circuit must satisfy every sub-field in this type in order
+ * to be considered to match the conditions.
+ */
+typedef struct circpad_machine_conditions_t {
+  /** Only apply the machine *if* the circuit has at least this many hops */
+  unsigned min_hops : 3;
+
+  /** Only apply the machine *if* vanguards are enabled */
+  unsigned requires_vanguards : 1;
+
+  /**
+   * This machine is ok to use if reduced padding is set in consensus
+   * or torrc. This machine will still be applied even if reduced padding
+   * is not set; this flag only acts to exclude machines that don't have
+   * it set when reduced padding is requested. Therefore, reduced padding
+   * machines should appear at the lowest priority in the padding machine
+   * lists (aka first in the list), so that non-reduced padding machines
+   * for the same purpose are given a chance to apply when reduced padding
+   * is not requested. */
+  unsigned reduced_padding_ok : 1;
+
+  /** Only apply the machine *if* the circuit's state matches any of
+   *  the bits set in this bitmask. */
+  circpad_circuit_state_t state_mask;
+
+  /** Only apply a machine *if* the circuit's purpose matches one
+   *  of the bits set in this bitmask */
+  circpad_purpose_mask_t purpose_mask;
+
+} circpad_machine_conditions_t;
+
+/**
+ * Token removal strategy options.
+ *
+ * The WTF-PAD histograms are meant to specify a target distribution to shape
+ * traffic towards. This is accomplished by removing tokens from the histogram
+ * when either padding or non-padding cells are sent.
+ *
+ * When we see a non-padding cell at a particular time since the last cell, you
+ * remove a token from the corresponding delay bin. These flags specify
+ * which bin to choose if that bin is already empty.
+ */
+typedef enum {
+  /** Don't remove any tokens */
+  CIRCPAD_TOKEN_REMOVAL_NONE = 0,
+  /**
+   * Remove from the first non-zero higher bin index when current is zero.
+   * This is the recommended strategy from the Adaptive Padding paper. */
+  CIRCPAD_TOKEN_REMOVAL_HIGHER = 1,
+  /** Remove from the first non-zero lower bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_LOWER = 2,
+  /** Remove from the closest non-zero bin index when current is empty. */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST = 3,
+  /** Remove from the closest bin by time value (since bins are
+   *  exponentially spaced). */
+  CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC = 4,
+  /** Only remove from the exact bin corresponding to this delay. If
+   *  the bin is 0, simply do nothing. Don't pick another bin. */
+  CIRCPAD_TOKEN_REMOVAL_EXACT = 5
+} circpad_removal_t;
+
+/**
+ * Distribution types supported by circpad_distribution_sample().
+ *
+ * These can be used instead of histograms for the inter-packet
+ * timing distribution, or to specify a distribution on the number
+ * of cells that can be sent while in a specific state of the state
+ * machine.
+ *
+ * Each distribution takes up to two parameters which are described below. */
+typedef enum {
+  /* No probability distribution is used */
+  CIRCPAD_DIST_NONE = 0,
+  /* Uniform distribution: param1 is lower bound and param2 is upper bound */
+  CIRCPAD_DIST_UNIFORM = 1,
+  /* Logistic distribution: param1 is Mu, param2 is sigma. */
+  CIRCPAD_DIST_LOGISTIC = 2,
+  /* Log-logistic distribution: param1 is Alpha, param2 is 1.0/Beta */
+  CIRCPAD_DIST_LOG_LOGISTIC = 3,
+  /* Geometric distribution: param1 is 'p' (success probability) */
+  CIRCPAD_DIST_GEOMETRIC = 4,
+  /* Weibull distribution: param1 is k, param2 is Lambda */
+  CIRCPAD_DIST_WEIBULL = 5,
+  /* Generalized Pareto distribution: param1 is sigma, param2 is xi */
+  CIRCPAD_DIST_PARETO = 6
+} circpad_distribution_type_t;
+
+/**
+ * Distribution information.
+ *
+ * This type specifies a specific distribution above, as well as
+ * up to two parameters for that distribution. The specific
+ * per-distribution meaning of these parameters is specified
+ * in circpad_distribution_sample().
+ */
+typedef struct circpad_distribution_t {
+  circpad_distribution_type_t type;
+  double param1;
+  double param2;
+} circpad_distribution_t;
+
+/** State number type. Represents current state of state machine. */
+typedef uint16_t circpad_statenum_t;
+#define  CIRCPAD_STATENUM_MAX   (UINT16_MAX)
+
+/** A histogram can be used to sample padding delays given a machine state.
+ * This constant defines the maximum histogram width (i.e. the max number of
+ * bins).
+ *
+ * The current limit is arbitrary and could be raised if there is a need,
+ * however too many bins will be hard to serialize in the future.
+ *
+ * Memory concerns are not so great here since the corresponding histogram and
+ * histogram_edges arrays are global and not per-circuit.
+ *
+ * If we ever upgrade this to a value that can't be represented by 8-bits we
+ * also need to upgrade circpad_hist_index_t.
+ */
+#define CIRCPAD_MAX_HISTOGRAM_LEN (100)
+
+/**
+ * A state of a padding state machine. The information here are immutable and
+ * represent the initial form of the state; it does not get updated as things
+ * happen. The mutable information that gets updated in runtime are carried in
+ * a circpad_machine_runtime_t.
+ *
+ * This struct describes the histograms and/or probability distributions, as
+ * well as parameters of a single state in the adaptive padding machine.
+ * Instances of this struct exist in global circpad machine definitions that
+ * come from torrc or the consensus.
+ */
+typedef struct circpad_state_t {
+  /**
+   * If a histogram is used for this state, this specifies the number of bins
+   * of this histogram. Histograms must have at least 2 bins.
+   *
+   * In particular, the following histogram:
+   *
+   * Tokens
+   *         +
+   *      10 |    +----+
+   *       9 |    |    |           +---------+
+   *       8 |    |    |           |         |
+   *       7 |    |    |     +-----+         |
+   *       6 +----+ Bin+-----+     |         +---------------+
+   *       5 |    | #1 |     |     |         |               |
+   *         | Bin|    | Bin | Bin |  Bin #4 |    Bin #5     |
+   *         | #0 |    | #2  | #3  |         | (infinity bin)|
+   *         |    |    |     |     |         |               |
+   *         |    |    |     |     |         |               |
+   *       0 +----+----+-----+-----+---------+---------------+
+   *         0   100  200   350   500      1000             inf  microseconds
+   *
+   * would be specified the following way:
+   *    histogram_len = 6;
+   *    histogram[] =        {   6,  10,   6,  7,    9,     6 }
+   *    histogram_edges[] =  { 0, 100, 200, 350, 500, 1000 }
+   *
+   * The final bin is called the "infinity bin" and if it's chosen we don't
+   * schedule any padding. The infinity bin is strange because its lower edge
+   * is the max value of possible non-infinite delay allowed by this histogram,
+   * and its upper edge is CIRCPAD_DELAY_INFINITE. You can tell if the infinity
+   * bin is chosen by inspecting its bin index or inspecting its upper edge.
+   *
+   * If a delay probability distribution is used for this state, this is set
+   * to 0. */
+  circpad_hist_index_t histogram_len;
+  /** The histogram itself: an array of uint16s of tokens, whose
+   *  widths are exponentially spaced, in microseconds.
+   *
+   *  This array must have histogram_len elements that are strictly
+   *  monotonically increasing. */
+  circpad_hist_token_t histogram[CIRCPAD_MAX_HISTOGRAM_LEN];
+  /* The histogram bin edges in usec.
+   *
+   * Each element of this array specifies the left edge of the corresponding
+   * bin. The rightmost edge is always infinity and is not specified in this
+   * array.
+   *
+   * This array must have histogram_len elements. */
+  circpad_delay_t histogram_edges[CIRCPAD_MAX_HISTOGRAM_LEN+1];
+  /** Total number of tokens in this histogram. This is a constant and is *not*
+   *  decremented every time we spend a token. It's used for initializing and
+   *  refilling the histogram. */
+  uint32_t histogram_total_tokens;
+
+  /**
+   * Represents a delay probability distribution (aka IAT distribution). It's a
+   * parametrized way of encoding inter-packet delay information in
+   * microseconds. It can be used instead of histograms.
+   *
+   * If it is used, token_removal below must be set to
+   * CIRCPAD_TOKEN_REMOVAL_NONE.
+   *
+   * Start_usec, range_sec, and rtt_estimates are still applied to the
+   * results of sampling from this distribution (range_sec is used as a max).
+   */
+  circpad_distribution_t iat_dist;
+  /*  If a delay probability distribution is used, this is used as the max
+   *  value we can sample from the distribution. However, RTT measurements and
+   *  dist_added_shift gets applied on top of this value to derive the final
+   *  padding delay. */
+  circpad_delay_t dist_max_sample_usec;
+  /*  If a delay probability distribution is used and this is set, we will add
+   *  this value on top of the value sampled from the IAT distribution to
+   *  derive the final padding delay (We also add the RTT measurement if it's
+   *  enabled.). */
+  circpad_delay_t dist_added_shift_usec;
+
+  /**
+   * The length dist is a parameterized way of encoding how long this
+   * state machine runs in terms of sent padding cells or all
+   * sent cells. Values are sampled from this distribution, clamped
+   * to max_len, and then start_len is added to that value.
+   *
+   * It may be specified instead of or in addition to
+   * the infinity bins and bins empty conditions. */
+  circpad_distribution_t length_dist;
+  /** A minimum length value, added to the output of length_dist */
+  uint16_t start_length;
+  /** A cap on the length value that can be sampled from the length_dist */
+  uint64_t max_length;
+
+  /** Should we decrement length when we see a nonpadding packet?
+   * XXX: Are there any machines that actually want to set this to 0? There may
+   * not be. OTOH, it's only a bit.. */
+  unsigned length_includes_nonpadding : 1;
+
+  /**
+   * This is an array that specifies the next state to transition to upon
+   * receipt an event matching the indicated array index.
+   *
+   * This aborts our scheduled packet and switches to the state
+   * corresponding to the index of the array. Tokens are filled upon
+   * this transition.
+   *
+   * States are allowed to transition to themselves, which means re-schedule
+   * a new padding timer. They are also allowed to temporarily "transition"
+   * to the "IGNORE" and "CANCEL" pseudo-states. See defines below
+   * for details on state behavior and meaning.
+   */
+  circpad_statenum_t next_state[CIRCPAD_NUM_EVENTS];
+
+  /**
+   * If true, estimate the RTT from this relay to the exit/website and add that
+   * to start_usec for use as the histogram bin 0 start delay.
+   *
+   * Right now this is only supported for relay-side state machines.
+   */
+  unsigned use_rtt_estimate : 1;
+
+  /** This specifies the token removal strategy to use upon padding and
+   *  non-padding activity. */
+  circpad_removal_t token_removal;
+} circpad_state_t;
+
+/**
+ * The start state for this machine.
+ *
+ * In the original WTF-PAD, this is only used for transition to/from
+ * the burst state. All other fields are not used. But to simplify the
+ * code we've made it a first-class state. This has no performance
+ * consequences, but may make naive serialization of the state machine
+ * large, if we're not careful about how we represent empty fields.
+ */
+#define  CIRCPAD_STATE_START       0
+
+/**
+ * The burst state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD
+ * (https://www.freehaven.net/anonbib/cache/ShWa-Timing06.pdf and
+ * https://www.cs.kau.se/pulls/hot/thebasketcase-wtfpad/), the burst
+ * state serves to detect bursts in traffic. This is done by using longer
+ * delays in its histogram, which represent the expected delays between
+ * bursts of packets in the target stream. If this delay expires without a
+ * real packet being sent, the burst state sends a padding packet and then
+ * immediately transitions to the gap state, which is used to generate
+ * a synthetic padding packet train. In this implementation, this transition
+ * needs to be explicitly specified in the burst state's transition events.
+ *
+ * Because of this flexibility, other padding mechanisms can transition
+ * between these two states arbitrarily, to encode other dynamics of
+ * target traffic.
+ */
+#define  CIRCPAD_STATE_BURST       1
+
+/**
+ * The gap state for this machine.
+ *
+ * In the original Adaptive Padding algorithm and in WTF-PAD, the gap
+ * state serves to simulate an artificial packet train composed of padding
+ * packets. It does this by specifying much lower inter-packet delays than
+ * the burst state, and transitioning back to itself after padding is sent
+ * if these timers expire before real traffic is sent. If real traffic is
+ * sent, it transitions back to the burst state.
+ *
+ * Again, in this implementation, these transitions must be specified
+ * explicitly, and other transitions are also permitted.
+ */
+#define  CIRCPAD_STATE_GAP         2
+
+/**
+ * End is a pseudo-state that causes the machine to go completely
+ * idle, and optionally get torn down (depending on the
+ * value of circpad_machine_spec_t.should_negotiate_end)
+ *
+ * End MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_END         CIRCPAD_STATENUM_MAX
+
+/**
+ * "Ignore" is a pseudo-state that means "do not react to this
+ * event".
+ *
+ * "Ignore" MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_IGNORE         (CIRCPAD_STATENUM_MAX-1)
+
+/**
+ * "Cancel" is a pseudo-state that means "cancel pending timers,
+ * but remain in your current state".
+ *
+ * Cancel MUST NOT occupy a slot in the machine state array.
+ */
+#define  CIRCPAD_STATE_CANCEL         (CIRCPAD_STATENUM_MAX-2)
+
+/**
+ * Since we have 3 pseudo-states, the max state array length is
+ * up to one less than cancel's statenum.
+ */
+#define CIRCPAD_MAX_MACHINE_STATES  (CIRCPAD_STATE_CANCEL-1)
+
+/**
+ * Mutable padding machine info.
+ *
+ * This structure contains mutable information about a padding
+ * machine. The mutable information must be kept separate because
+ * it exists per-circuit, where as the machines themselves are global.
+ * This separation is done to conserve space in the circuit structure.
+ *
+ * This is the per-circuit state that changes regarding the global state
+ * machine. Some parts of it are optional (ie NULL).
+ *
+ * XXX: Play with layout to minimize space on x64 Linux (most common relay).
+ */
+typedef struct circpad_machine_runtime_t {
+  /** The callback pointer for the padding callbacks.
+   *
+   *  These timers stick around the machineinfo until the machineinfo's circuit
+   *  is closed, at which point the timer is cancelled. For this reason it's
+   *  safe to assume that the machineinfo exists if this timer gets
+   *  triggered. */
+  tor_timer_t *padding_timer;
+
+  /** The circuit for this machine */
+  struct circuit_t *on_circ;
+
+  /** A mutable copy of the histogram for the current state.
+   *  NULL if remove_tokens is false for that state */
+  circpad_hist_token_t *histogram;
+  /** Length of the above histogram.
+   * XXX: This field *could* be removed at the expense of added
+   * complexity+overhead for reaching back into the immutable machine
+   * state every time we need to inspect the histogram. It's only a byte,
+   * though, so it seemed worth it.
+   */
+  circpad_hist_index_t histogram_len;
+  /** Remove token from this index upon sending padding */
+  circpad_hist_index_t chosen_bin;
+
+  /** Stop padding/transition if this many cells sent */
+  uint64_t state_length;
+#define CIRCPAD_STATE_LENGTH_INFINITE UINT64_MAX
+
+  /** A scaled count of padding packets sent, used to limit padding overhead.
+   * When this reaches UINT16_MAX, we cut it and nonpadding_sent in half. */
+  uint16_t padding_sent;
+  /** A scaled count of non-padding packets sent, used to limit padding
+   *  overhead. When this reaches UINT16_MAX, we cut it and padding_sent in
+   *  half. */
+  uint16_t nonpadding_sent;
+
+  /**
+   * Timestamp of the most recent cell event (sent, received, padding,
+   * non-padding), in seconds from approx_time().
+   *
+   * Used as an emergency break to stop holding padding circuits open.
+   */
+  time_t last_cell_time_sec;
+
+  /**
+   * EWMA estimate of the RTT of the circuit from this hop
+   * to the exit end, in microseconds. */
+  circpad_delay_t rtt_estimate_usec;
+
+  /**
+   * The last time we got an event relevant to estimating
+   * the RTT. Monotonic time in microseconds since system
+   * start.
+   */
+  circpad_time_t last_received_time_usec;
+
+  /**
+   * The time at which we scheduled a non-padding packet,
+   * or selected an infinite delay.
+   *
+   * Monotonic time in microseconds since system start.
+   * This is 0 if we haven't chosen a padding delay.
+   */
+  circpad_time_t padding_scheduled_at_usec;
+
+  /** What state is this machine in? */
+  circpad_statenum_t current_state;
+
+  /**
+   * True if we have scheduled a timer for padding.
+   *
+   * This is 1 if a timer is pending. It is 0 if
+   * no timer is scheduled. (It can be 0 even when
+   * padding_was_scheduled_at_usec is non-zero).
+   */
+  unsigned is_padding_timer_scheduled : 1;
+
+  /**
+   * If this is true, we have seen full duplex behavior.
+   * Stop updating the RTT.
+   */
+  unsigned stop_rtt_update : 1;
+
+/** Max number of padding machines on each circuit. If changed,
+ * also ensure the machine_index bitwith supports the new size. */
+#define CIRCPAD_MAX_MACHINES    (2)
+  /** Which padding machine index was this for.
+   * (make sure changes to the bitwidth can support the
+   * CIRCPAD_MAX_MACHINES define). */
+  unsigned machine_index : 1;
+
+} circpad_machine_runtime_t;
+
+/** Helper macro to get an actual state machine from a machineinfo */
+#define CIRCPAD_GET_MACHINE(machineinfo) \
+    ((machineinfo)->on_circ->padding_machine[(machineinfo)->machine_index])
+
+/**
+ * This specifies a particular padding machine to use after negotiation.
+ *
+ * The constants for machine_num_t are in trunnel.
+ * We want to be able to define extra numbers in the consensus/torrc, though.
+ */
+typedef uint8_t circpad_machine_num_t;
+
+/** Global state machine structure from the consensus */
+typedef struct circpad_machine_spec_t {
+  /* Just a user-friendly machine name for logs */
+  const char *name;
+
+  /** Global machine number */
+  circpad_machine_num_t machine_num;
+
+  /** Which machine index slot should this machine go into in
+   *  the array on the circuit_t */
+  unsigned machine_index : 1;
+
+  /** Send a padding negotiate to shut down machine at end state? */
+  unsigned should_negotiate_end : 1;
+
+  // These next three fields are origin machine-only...
+  /** Origin side or relay side */
+  unsigned is_origin_side : 1;
+
+  /** Which hop in the circuit should we send padding to/from?
+   *  1-indexed (ie: hop #1 is guard, #2 middle, #3 exit). */
+  unsigned target_hopnum : 3;
+
+  /** If this flag is enabled, don't close circuits that use this machine even
+   *  if another part of Tor wants to close this circuit.
+   *
+   *  If this flag is set, the circuitpadding subsystem will close circuits the
+   *  moment the machine transitions to the END state, and only if the circuit
+   *  has already been asked to be closed by another part of Tor.
+   *
+   *  Circuits that should have been closed but were kept open by a padding
+   *  machine are re-purposed to CIRCUIT_PURPOSE_C_CIRCUIT_PADDING, hence
+   *  machines should take that purpose into account if they are filtering
+   *  circuits by purpose. */
+  unsigned manage_circ_lifetime : 1;
+
+  /** This machine only kills fascists if the following conditions are met. */
+  circpad_machine_conditions_t conditions;
+
+  /** How many padding cells can be sent before we apply overhead limits?
+   * XXX: Note that we can only allow up to 64k of padding cells on an
+   * otherwise quiet circuit. Is this enough? It's 33MB. */
+  uint16_t allowed_padding_count;
+
+  /** Padding percent cap: Stop padding if we exceed this percent overhead.
+   * 0 means no limit. Overhead is defined as percent of total traffic, so
+   * that we can use 0..100 here. This is the same definition as used in
+   * Prop#265. */
+  uint8_t max_padding_percent;
+
+  /** State array: indexed by circpad_statenum_t */
+  circpad_state_t *states;
+
+  /**
+   * Number of states this machine has (ie: length of the states array).
+   * XXX: This field is not needed other than for safety. */
+  circpad_statenum_t num_states;
+} circpad_machine_spec_t;
+
+void circpad_new_consensus_params(const networkstatus_t *ns);
+
+int circpad_marked_circuit_for_padding(circuit_t *circ, int reason);
+
+/**
+ * The following are event call-in points that are of interest to
+ * the state machines. They are called during cell processing. */
+void circpad_deliver_unrecognized_cell_events(struct circuit_t *circ,
+                                              cell_direction_t dir);
+void circpad_deliver_sent_relay_cell_events(struct circuit_t *circ,
+                                            uint8_t relay_command);
+void circpad_deliver_recognized_relay_cell_events(struct circuit_t *circ,
+                                                  uint8_t relay_command,
+                                                  crypt_path_t *layer_hint);
+
+/** Cell events are delivered by the above delivery functions */
+void circpad_cell_event_nonpadding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_nonpadding_received(struct circuit_t *on_circ);
+void circpad_cell_event_padding_sent(struct circuit_t *on_circ);
+void circpad_cell_event_padding_received(struct circuit_t *on_circ);
+
+/** Internal events are events the machines send to themselves */
+circpad_decision_t
+circpad_internal_event_infinity(circpad_machine_runtime_t *mi);
+circpad_decision_t
+circpad_internal_event_bins_empty(circpad_machine_runtime_t *);
+circpad_decision_t circpad_internal_event_state_length_up(
+                                  circpad_machine_runtime_t *);
+
+/** Machine creation events are events that cause us to set up or
+ *  tear down padding state machines. */
+void circpad_machine_event_circ_added_hop(struct origin_circuit_t *on_circ);
+void circpad_machine_event_circ_built(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_purpose_changed(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_streams(struct origin_circuit_t *circ);
+void circpad_machine_event_circ_has_no_streams(struct origin_circuit_t *circ);
+void
+circpad_machine_event_circ_has_no_relay_early(struct origin_circuit_t *circ);
+
+void circpad_machines_init(void);
+void circpad_machines_free(void);
+void circpad_register_padding_machine(circpad_machine_spec_t *machine,
+                                      smartlist_t *machine_list);
+
+void circpad_machine_states_init(circpad_machine_spec_t *machine,
+                                 circpad_statenum_t num_states);
+
+void circpad_circuit_free_all_machineinfos(struct circuit_t *circ);
+
+bool circpad_padding_is_from_expected_hop(struct circuit_t *circ,
+                                         crypt_path_t *from_hop);
+
+/** Serializaton functions for writing to/from torrc and consensus */
+char *circpad_machine_spec_to_string(const circpad_machine_spec_t *machine);
+const circpad_machine_spec_t *circpad_string_to_machine(const char *str);
+
+/* Padding negotiation between client and middle */
+signed_error_t circpad_handle_padding_negotiate(struct circuit_t *circ,
+                                      struct cell_t *cell);
+signed_error_t circpad_handle_padding_negotiated(struct circuit_t *circ,
+                                      struct cell_t *cell,
+                                      crypt_path_t *layer_hint);
+signed_error_t circpad_negotiate_padding(struct origin_circuit_t *circ,
+                          circpad_machine_num_t machine,
+                          uint8_t target_hopnum,
+                          uint8_t command);
+bool circpad_padding_negotiated(struct circuit_t *circ,
+                           circpad_machine_num_t machine,
+                           uint8_t command,
+                           uint8_t response);
+
+circpad_purpose_mask_t circpad_circ_purpose_to_mask(uint8_t circ_purpose);
+
+int circpad_check_received_cell(cell_t *cell, circuit_t *circ,
+                                crypt_path_t *layer_hint,
+                                const relay_header_t *rh);
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_schedule_padding,(circpad_machine_runtime_t *));
+
+MOCK_DECL(circpad_decision_t,
+circpad_machine_spec_transition, (circpad_machine_runtime_t *mi,
+                             circpad_event_t event));
+
+circpad_decision_t circpad_send_padding_cell_for_callback(
+                                 circpad_machine_runtime_t *mi);
+
+void circpad_free_all(void);
+
+#ifdef CIRCUITPADDING_PRIVATE
+STATIC void  machine_spec_free_(circpad_machine_spec_t *m);
+#define machine_spec_free(chan) \
+  FREE_AND_NULL(circpad_machine_spec_t,machine_spec_free_, (m))
+
+STATIC circpad_delay_t
+circpad_machine_sample_delay(circpad_machine_runtime_t *mi);
+
+STATIC bool
+circpad_machine_reached_padding_limit(circpad_machine_runtime_t *mi);
+
+STATIC circpad_delay_t
+circpad_histogram_bin_to_usec(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC const circpad_state_t *
+circpad_machine_current_state(const circpad_machine_runtime_t *mi);
+
+STATIC void circpad_machine_remove_token(circpad_machine_runtime_t *mi);
+
+STATIC circpad_hist_index_t circpad_histogram_usec_to_bin(
+                                       const circpad_machine_runtime_t *mi,
+                                       circpad_delay_t us);
+
+STATIC circpad_machine_runtime_t *circpad_circuit_machineinfo_new(
+                                               struct circuit_t *on_circ,
+                                               int machine_index);
+STATIC void circpad_machine_remove_higher_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_lower_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us);
+STATIC void circpad_machine_remove_closest_token(circpad_machine_runtime_t *mi,
+                                         circpad_delay_t target_bin_us,
+                                         bool use_usec);
+STATIC void circpad_machine_setup_tokens(circpad_machine_runtime_t *mi);
+
+MOCK_DECL(STATIC signed_error_t,
+circpad_send_command_to_hop,(struct origin_circuit_t *circ, uint8_t hopnum,
+                             uint8_t relay_command, const uint8_t *payload,
+                             ssize_t payload_len));
+
+MOCK_DECL(STATIC const node_t *,
+circuit_get_nth_node,(origin_circuit_t *circ, int hop));
+
+STATIC circpad_delay_t
+histogram_get_bin_upper_bound(const circpad_machine_runtime_t *mi,
+                              circpad_hist_index_t bin);
+
+STATIC void
+circpad_add_matching_machines(origin_circuit_t *on_circ,
+                              smartlist_t *machines_sl);
+
+#ifdef TOR_UNIT_TESTS
+extern smartlist_t *origin_padding_machines;
+extern smartlist_t *relay_padding_machines;
+
+#endif
+
+#endif /* defined(CIRCUITPADDING_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_H) */
diff --git a/src/core/or/circuitpadding_machines.c b/src/core/or/circuitpadding_machines.c
new file mode 100644
index 0000000000..98767f9e8f
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.c
@@ -0,0 +1,454 @@
+/* Copyright (c) 2019 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.c
+ * \brief Circuit padding state machines
+ *
+ * Introduce circuit padding machines that will be used by Tor circuits, as
+ * specified by proposal 302 "Hiding onion service clients using padding".
+ *
+ * Right now this file introduces two machines that aim to hide the client-side
+ * of onion service circuits against naive classifiers like the ones from the
+ * "Circuit Fingerprinting Attacks: Passive Deanonymization of Tor Hidden
+ * Services" paper from USENIX. By naive classifiers we mean classifiers that
+ * use basic features like "circuit construction circuits" and "incoming and
+ * outgoing cell counts" and "duration of activity".
+ *
+ * In particular, these machines aim to be lightweight and protect against
+ * these basic classifiers. They don't aim to protect against more advanced
+ * attacks that use deep learning or even correlate various circuit
+ * construction events together. Machines that fool such advanced classifiers
+ * are also possible, but they can't be so lightweight and might require more
+ * WTF-PAD features. So for now we opt for the following two machines:
+ *
+ * Client-side introduction circuit hiding machine:
+ *
+ *    This machine hides client-side introduction circuits by making their
+ *    circuit consruction sequence look like normal general circuits that
+ *    download directory information. Furthermore, the circuits are kept open
+ *    until all the padding has been sent, since intro circuits are usually
+ *    very short lived and this act as a distinguisher. For more info see
+ *    circpad_machine_client_hide_intro_circuits() and the sec.
+ *
+ * Client-side rendezvous circuit hiding machine:
+ *
+ *    This machine hides client-side rendezvous circuits by making their
+ *    circuit construction sequence look like normal general circuits. For more
+ *    details see circpad_machine_client_hide_rend_circuits() and the spec.
+ *
+ * TODO: These are simple machines that carefully manipulate the cells of the
+ *   initial circuit setup procedure to make them look like general
+ *   circuits. In the future, more states can be baked into their state machine
+ *   to do more advanced obfuscation.
+ **/
+
+#define CIRCUITPADDING_MACHINES_PRIVATE
+
+#include "core/or/or.h"
+#include "feature/nodelist/networkstatus.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/circuitlist.h"
+
+#include "core/or/circuitpadding_machines.h"
+#include "core/or/circuitpadding.h"
+
+/** Create a client-side padding machine that aims to hide IP circuits. In
+ *  particular, it keeps intro circuits alive until a bunch of fake traffic has
+ *  been pushed through.
+ */
+void
+circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_ip_circ";
+
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad introduction circuits, and we want to start padding
+   * only after the INTRODUCE1 cell has been sent, so set the purposes
+   * appropriately.
+   *
+   * In particular we want introduction circuits to blend as much as possible
+   * with general circuits. Most general circuits have the following initial
+   * relay cell sequence (outgoing cells marked in [brackets]):
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * Whereas normal introduction circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2
+   *   -> [INTRO1] -> INTRODUCE_ACK
+   *
+   * This means that up to the sixth cell (first line of each sequence above),
+   * both general and intro circuits have identical cell sequences. After that
+   * we want to mimic the second line sequence of
+   *   -> [DATA] -> [DATA] -> DATA -> DATA...(inbound data cells continue)
+   *
+   * We achieve this by starting padding INTRODUCE1 has been sent. With padding
+   * negotiation cells, in the common case of the second line looks like:
+   *   -> [INTRO1] -> [PADDING_NEGOTIATE] -> PADDING_NEGOTIATED -> INTRO_ACK
+   *
+   * Then, the middle node will send between INTRO_MACHINE_MINIMUM_PADDING and
+   * INTRO_MACHINE_MAXIMUM_PADDING cells, to match the "...(inbound data cells
+   * continue)" portion of the trace (aka the rest of an HTTPS response body).
+   */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_INTRODUCE_ACKED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Keep the circuit alive even after the introduction has been finished,
+   * otherwise the short-term lifetime of the circuit will blow our cover */
+  client_machine->manage_circ_lifetime = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* For the origin-side machine, we transition to OBFUSCATE_CIRC_SETUP after
+   * sending PADDING_NEGOTIATE, and we stay there (without sending any padding)
+   * until we receive a STOP from the other side. */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* origin-side machine has no event reactions while in
+   * CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP, so no more state transitions here. */
+
+  /* The client side should never send padding, so it does not need
+   * to specify token removal, or a histogram definition or state lengths.
+   * That is all controlled by the middle node. */
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client intro point hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits. See
+ *  comments on the function above for more details on the workings of the
+ *  machine. */
+void
+circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_ip_circ";
+
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* We want to negotiate END from this side after all our padding is done, so
+   * that the origin-side machine goes into END state, and eventually closes
+   * the circuit. */
+  relay_machine->should_negotiate_end = 1;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = INTRO_MACHINE_MAXIMUM_PADDING;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* For the relay-side machine, we want to transition
+   * START -> OBFUSCATE_CIRC_SETUP upon first non-padding
+   * cell sent (PADDING_NEGOTIATED in this case).  */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* For the relay-side, we want to transition from OBFUSCATE_CIRC_SETUP to END
+   * state when the length finishes. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Now let's define the OBF -> OBF transitions that maintain our padding
+   * flow:
+   *
+   * For the relay-side machine, we want to keep on sending padding bytes even
+   * when nothing else happens on this circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_PADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+  /* For the relay-side machine, we need this transition so that we re-enter
+     the state, after PADDING_NEGOTIATED is sent. Otherwise, the remove token
+     function will disable the timer, and nothing will restart it since there
+     is no other motion on an intro circuit. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* Token removal strategy for OBFUSCATE_CIRC_SETUP state: Don't
+   * remove any tokens.
+   *
+   * We rely on the state length sampling and not token removal, to avoid
+   * the mallocs required to copy the histograms for token removal,
+   * and to avoid monotime calls needed to determine histogram
+   * bins for token removal. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Figure out the length of the OBFUSCATE_CIRC_SETUP state so that it's
+   * randomized. The relay side will send between INTRO_MACHINE_MINIMUM_PADDING
+   * and INTRO_MACHINE_MAXIMUM_PADDING padding cells towards the client. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = INTRO_MACHINE_MINIMUM_PADDING;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = INTRO_MACHINE_MAXIMUM_PADDING;
+
+  /* Configure histogram */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+     histogram_len = 2;
+
+  /* For the relay-side machine we want to batch padding instantly to pretend
+   * its an incoming directory download. So set the histogram edges tight:
+   * (1, 10ms, infinity). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 1000;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 10000;
+
+  /* We put all our tokens in bin 0, which means we want 100% probability
+   * for choosing a inter-packet delay of between 1000 and 10000 microseconds
+   * (1 to 10ms). Since we only have 1 bin, it doesn't matter how many tokens
+   * there are, 1000 out of 1000 is 100% */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1000;
+
+  /* just one bin, so setup the total tokens */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens =
+      relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].histogram[0];
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay intro circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
+
+/************************** Rendezvous-circuit machine ***********************/
+
+/** Create a client-side padding machine that aims to hide rendezvous
+ *  circuits.*/
+void
+circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *client_machine
+      = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  client_machine->name = "client_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  client_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+  client_machine->target_hopnum = 2;
+
+  /* This is a client machine */
+  client_machine->is_origin_side = 1;
+
+  /* We only want to pad rendezvous circuits, and we want to start padding only
+   * after the rendezvous circuit has been established.
+   *
+   * Following a similar argument as for intro circuits, we are aiming for
+   * padded rendezvous circuits to blend in with the initial cell sequence of
+   * general circuits which usually look like this:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [BEGIN] -> CONNECTED
+   *    -> [DATA] -> [DATA] -> DATA -> DATA...(incoming cells continue)
+   *
+   * Whereas normal rendezvous circuits usually look like:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> REND2 -> [BEGIN]
+   *
+   * This means that up to the sixth cell (in the first line), both general and
+   * rend circuits have identical cell sequences.
+   *
+   * After that we want to mimic a [DATA] -> [DATA] -> DATA -> DATA sequence.
+   *
+   * With padding negotiation right after the REND_ESTABLISHED, the sequence
+   * becomes:
+   *
+   * [EXTEND2] -> EXTENDED2 -> [EXTEND2] -> EXTENDED2 -> [EST_REND] -> REND_EST
+   *    -> [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP...
+   *
+   * After which normal application DATA cells continue on the circuit.
+   *
+   * Hence this way we make rendezvous circuits look like general circuits up
+   * till the end of the circuit setup. */
+  client_machine->conditions.purpose_mask =
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_JOINED)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY)|
+    circpad_circ_purpose_to_mask(CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED);
+
+  /* Set padding machine limits to help guard against excessive padding */
+  client_machine->allowed_padding_count = 1;
+  client_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(client_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATE) */
+  client_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_END;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that we send our outgoing [DROP] before the PADDING_NEGOTIATED comes
+   * back from the relay side. */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  client_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  client_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(client_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered client rendezvous circuit hiding padding machine (%u)",
+           client_machine->machine_num);
+}
+
+/** Create a relay-side padding machine that aims to hide IP circuits.
+ *
+ *  This is meant to follow the client-side machine.
+ */
+void
+circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl)
+{
+  circpad_machine_spec_t *relay_machine
+    = tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  relay_machine->name = "relay_rp_circ";
+
+  /* Only pad after the circuit has been built and pad to the middle */
+  relay_machine->conditions.min_hops = 2;
+  relay_machine->conditions.state_mask = CIRCPAD_CIRC_OPENED;
+
+  /* This is a relay-side machine */
+  relay_machine->is_origin_side = 0;
+
+  /* Set padding machine limits to help guard against excessive padding */
+  relay_machine->allowed_padding_count = 1;
+  relay_machine->max_padding_percent = 1;
+
+  /* Two states: START, OBFUSCATE_CIRC_SETUP (and END) */
+  circpad_machine_states_init(relay_machine, 2);
+
+  /* START -> OBFUSCATE_CIRC_SETUP transition upon sending the first
+   * non-padding cell (which is PADDING_NEGOTIATED) */
+  relay_machine->states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_SENT] =
+    CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP;
+
+  /* OBFUSCATE_CIRC_SETUP -> END transition when we send our first
+   * padding packet and/or hit the state length (the state length is 1). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_END;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Don't use a token removal strategy since we don't want to use monotime
+   * functions and we want to avoid mallocing histogram copies. We want
+   * this machine to be light. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    token_removal = CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Instead, to control the volume of padding (we just want to send a single
+   * padding cell) we will use a static state length. We just want one token,
+   * since we want to make the following pattern:
+   * [PADDING_NEGOTIATE] -> [DROP] -> PADDING_NEGOTIATED -> DROP */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.type = CIRCPAD_DIST_UNIFORM;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param1 = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    length_dist.param2 = 2; // rand(1,2) is always 1
+
+  /* Histogram is: (0 msecs, 1 msec, infinity). We want this to be fast so
+   * that the outgoing DROP cell is sent immediately after the
+   * PADDING_NEGOTIATED. */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_len = 2;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[0] = 0;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_edges[1] = 1000;
+
+  /* We want a 100% probability of choosing an inter-packet delay of
+   * between 0 and 1ms. Since we don't use token removal,
+   * the number of tokens does not matter. (And also, state_length
+   * governs how many packets we send). */
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram[0] = 1;
+  relay_machine->states[CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP].
+    histogram_total_tokens = 1;
+
+  /* Register the machine */
+  relay_machine->machine_num = smartlist_len(machines_sl);
+  circpad_register_padding_machine(relay_machine, machines_sl);
+
+  log_info(LD_CIRC,
+           "Registered relay rendezvous circuit hiding padding machine (%u)",
+           relay_machine->machine_num);
+}
diff --git a/src/core/or/circuitpadding_machines.h b/src/core/or/circuitpadding_machines.h
new file mode 100644
index 0000000000..3c9798d42d
--- /dev/null
+++ b/src/core/or/circuitpadding_machines.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file circuitpadding_machines.h
+ * \brief Header file for circuitpadding_machines.c.
+ **/
+
+#ifndef TOR_CIRCUITPADDING_MACHINES_H
+#define TOR_CIRCUITPADDING_MACHINES_H
+
+void circpad_machine_relay_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_intro_circuits(smartlist_t *machines_sl);
+void circpad_machine_relay_hide_rend_circuits(smartlist_t *machines_sl);
+void circpad_machine_client_hide_rend_circuits(smartlist_t *machines_sl);
+
+#ifdef CIRCUITPADDING_MACHINES_PRIVATE
+
+/** State of the padding machines that actually sends padding */
+#define CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP CIRCPAD_STATE_BURST
+
+/** Constants defining the amount of padding that a machine will send to hide
+ *  HS circuits. The actual value is sampled uniformly random between the
+ *  min/max values.
+ */
+
+/** Minimum number of relay-side padding cells to be sent by this machine */
+#define INTRO_MACHINE_MINIMUM_PADDING 7
+/** Maximum number of relay-side padding cells to be sent by this machine.
+ *  The actual value will be sampled between the min and max.*/
+#define INTRO_MACHINE_MAXIMUM_PADDING 10
+
+#endif /* defined(CIRCUITPADDING_MACHINES_PRIVATE) */
+
+#endif /* !defined(TOR_CIRCUITPADDING_MACHINES_H) */
diff --git a/src/core/or/circuitstats.c b/src/core/or/circuitstats.c
index 2cde21fa1f..5875627b93 100644
--- a/src/core/or/circuitstats.c
+++ b/src/core/or/circuitstats.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,8 +29,8 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitstats.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
-#include "feature/control/control.h"
+#include "lib/confmgt/confmgt.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/networkstatus.h"
@@ -44,6 +44,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/relay/relay_periodic.h"
 
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -639,9 +640,9 @@ circuit_build_times_rewind_history(circuit_build_times_t *cbt, int n)
 void
 circuit_build_times_mark_circ_as_measurement_only(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ,
-                               CIRC_EVENT_FAILED,
-                               END_CIRC_REASON_TIMEOUT);
+  circuit_event_status(circ,
+                       CIRC_EVENT_FAILED,
+                       END_CIRC_REASON_TIMEOUT);
   circuit_change_purpose(TO_CIRCUIT(circ),
                          CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
   /* Record this event to check for too many timeouts
@@ -971,7 +972,7 @@ circuit_build_times_update_state(const circuit_build_times_t *cbt,
 /**
  * Shuffle the build times array.
  *
- * Adapted from http://en.wikipedia.org/wiki/Fisher-Yates_shuffle
+ * Adapted from https://en.wikipedia.org/wiki/Fisher-Yates_shuffle
  */
 static void
 circuit_build_times_shuffle_and_store_array(circuit_build_times_t *cbt,
@@ -1182,7 +1183,7 @@ circuit_build_times_parse_state(circuit_build_times_t *cbt,
 
 /**
  * Estimates the Xm and Alpha parameters using
- * http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation
+ * https://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation
  *
  * The notable difference is that we use mode instead of min to estimate Xm.
  * This is because our distribution is frechet-like. We claim this is
@@ -1197,7 +1198,7 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt)
   int n=0,i=0,abandoned_count=0;
   build_time_t max_time=0;
 
-  /* http://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */
+  /* https://en.wikipedia.org/wiki/Pareto_distribution#Parameter_estimation */
   /* We sort of cheat here and make our samples slightly more pareto-like
    * and less frechet-like. */
   cbt->Xm = circuit_build_times_get_xm(cbt);
@@ -1269,9 +1270,9 @@ circuit_build_times_update_alpha(circuit_build_times_t *cbt)
  * We use it to calculate the timeout and also to generate synthetic
  * values of time for circuits that timeout before completion.
  *
- * See http://en.wikipedia.org/wiki/Quantile_function,
- * http://en.wikipedia.org/wiki/Inverse_transform_sampling and
- * http://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_
+ * See https://en.wikipedia.org/wiki/Quantile_function,
+ * https://en.wikipedia.org/wiki/Inverse_transform_sampling and
+ * https://en.wikipedia.org/wiki/Pareto_distribution#Generating_a_
  *     random_sample_from_Pareto_distribution
  * That's right. I'll cite wikipedia all day long.
  *
@@ -1420,6 +1421,7 @@ void
 circuit_build_times_network_is_live(circuit_build_times_t *cbt)
 {
   time_t now = approx_time();
+  // XXXX this should use pubsub
   if (cbt->liveness.nonlive_timeouts > 0) {
     time_t time_since_live = now - cbt->liveness.network_last_live;
     log_notice(LD_CIRC,
diff --git a/src/core/or/circuitstats.h b/src/core/or/circuitstats.h
index 845d7b6722..52c9100f53 100644
--- a/src/core/or/circuitstats.h
+++ b/src/core/or/circuitstats.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -175,7 +175,7 @@ typedef struct {
 } network_liveness_t;
 
 /** Structure for circuit build times history */
-struct circuit_build_times_s {
+struct circuit_build_times_t {
   /** The circular array of recorded build times in milliseconds */
   build_time_t circuit_build_times[CBT_NCIRCUITS_TO_OBSERVE];
   /** Current index in the circuit_build_times circular array */
diff --git a/src/core/or/circuituse.c b/src/core/or/circuituse.c
index efd69fb4a3..a88ccf9dd1 100644
--- a/src/core/or/circuituse.c
+++ b/src/core/or/circuituse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,13 +35,15 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/bridges.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/client/proxymode.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_client.h"
@@ -69,7 +71,7 @@
 #include "core/or/origin_circuit_st.h"
 #include "core/or/socks_request_st.h"
 
-static void circuit_expire_old_circuits_clientside(void);
+STATIC void circuit_expire_old_circuits_clientside(void);
 static void circuit_increment_failure_count(void);
 
 /** Check whether the hidden service destination of the stream at
@@ -177,7 +179,6 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
       purpose == CIRCUIT_PURPOSE_S_HSDIR_POST ||
       purpose == CIRCUIT_PURPOSE_C_HSDIR_GET) {
     tor_addr_t addr;
-    const int family = tor_addr_parse(&addr, conn->socks_request->address);
     if (!exitnode && !build_state->onehop_tunnel) {
       log_debug(LD_CIRC,"Not considering circuit with unknown router.");
       return 0; /* this circuit is screwed and doesn't know it yet,
@@ -198,6 +199,8 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
           return 0; /* this is a circuit to somewhere else */
         if (tor_digest_is_zero(digest)) {
           /* we don't know the digest; have to compare addr:port */
+          const int family = tor_addr_parse(&addr,
+                                            conn->socks_request->address);
           if (family < 0 ||
               !tor_addr_eq(&build_state->chosen_exit->addr, &addr) ||
               build_state->chosen_exit->port != conn->socks_request->port)
@@ -210,12 +213,14 @@ circuit_is_acceptable(const origin_circuit_t *origin_circ,
         return 0;
       }
     }
-    if (origin_circ->prepend_policy && family != -1) {
-      int r = compare_tor_addr_to_addr_policy(&addr,
-                                              conn->socks_request->port,
-                                              origin_circ->prepend_policy);
-      if (r == ADDR_POLICY_REJECTED)
-        return 0;
+    if (origin_circ->prepend_policy) {
+      if (tor_addr_parse(&addr, conn->socks_request->address) != -1) {
+        int r = compare_tor_addr_to_addr_policy(&addr,
+                                                conn->socks_request->port,
+                                                origin_circ->prepend_policy);
+        if (r == ADDR_POLICY_REJECTED)
+          return 0;
+      }
     }
     if (exitnode && !connection_ap_can_use_exit(conn, exitnode)) {
       /* can't exit from this router */
@@ -543,9 +548,10 @@ circuit_expire_building(void)
              MAX(get_circuit_build_close_time_ms()*2 + 1000,
                  options->SocksTimeout * 1000));
 
+  bool fixed_time = circuit_build_times_disabled(get_options());
+
   SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *,victim) {
     struct timeval cutoff;
-    bool fixed_time = circuit_build_times_disabled(get_options());
 
     if (!CIRCUIT_IS_ORIGIN(victim) || /* didn't originate here */
         victim->marked_for_close)     /* don't mess with marked circs */
@@ -727,7 +733,7 @@ circuit_expire_building(void)
           circuit_build_times_enough_to_compute(get_circuit_build_times())) {
 
         log_info(LD_CIRC,
-                 "Deciding to count the timeout for circuit %"PRIu32"\n",
+                 "Deciding to count the timeout for circuit %"PRIu32,
                  TO_ORIGIN_CIRCUIT(victim)->global_identifier);
 
         /* Circuits are allowed to last longer for measurement.
@@ -770,16 +776,11 @@ circuit_expire_building(void)
     if (!(TO_ORIGIN_CIRCUIT(victim)->hs_circ_has_timed_out)) {
       switch (victim->purpose) {
       case CIRCUIT_PURPOSE_C_REND_READY:
-        /* We only want to spare a rend circ if it has been specified in
-         * an INTRODUCE1 cell sent to a hidden service.  A circ's
-         * pending_final_cpath field is non-NULL iff it is a rend circ
-         * and we have tried to send an INTRODUCE1 cell specifying it.
-         * Thus, if the pending_final_cpath field *is* NULL, then we
-         * want to not spare it. */
-        if (TO_ORIGIN_CIRCUIT(victim)->build_state &&
-            TO_ORIGIN_CIRCUIT(victim)->build_state->pending_final_cpath ==
-            NULL)
+        /* We only want to spare a rend circ iff it has been specified in an
+         * INTRODUCE1 cell sent to a hidden service. */
+        if (!hs_circ_is_rend_sent_in_intro1(CONST_TO_ORIGIN_CIRCUIT(victim))) {
           break;
+        }
         FALLTHROUGH;
       case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
       case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED:
@@ -1425,6 +1426,11 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
       if (circ->purpose == CIRCUIT_PURPOSE_S_REND_JOINED) {
         hs_dec_rdv_stream_counter(origin_circ);
       }
+
+      /* If there are no more streams on this circ, tell circpad */
+      if (!origin_circ->p_streams)
+        circpad_machine_event_circ_has_no_streams(origin_circ);
+
       return;
     }
   } else {
@@ -1465,7 +1471,7 @@ circuit_detach_stream(circuit_t *circ, edge_connection_t *conn)
 /** Find each circuit that has been unused for too long, or dirty
  * for too long and has no streams on it: mark it for close.
  */
-static void
+STATIC void
 circuit_expire_old_circuits_clientside(void)
 {
   struct timeval cutoff, now;
@@ -1505,6 +1511,7 @@ circuit_expire_old_circuits_clientside(void)
                 circ->purpose == CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT ||
                 circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
                 circ->purpose == CIRCUIT_PURPOSE_TESTING ||
+                circ->purpose == CIRCUIT_PURPOSE_C_CIRCUIT_PADDING ||
                 (circ->purpose >= CIRCUIT_PURPOSE_C_INTRODUCING &&
                 circ->purpose <= CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED) ||
                 circ->purpose == CIRCUIT_PURPOSE_S_CONNECT_REND) {
@@ -1674,7 +1681,7 @@ circuit_testing_failed(origin_circuit_t *circ, int at_last_hop)
 void
 circuit_has_opened(origin_circuit_t *circ)
 {
-  control_event_circuit_status(circ, CIRC_EVENT_BUILT, 0);
+  circuit_event_status(circ, CIRC_EVENT_BUILT, 0);
 
   /* Remember that this circuit has finished building. Now if we start
    * it building again later (e.g. by extending it), we will know not
@@ -1954,23 +1961,61 @@ have_enough_path_info(int need_exit)
 int
 circuit_purpose_is_hidden_service(uint8_t purpose)
 {
-   if (purpose == CIRCUIT_PURPOSE_HS_VANGUARDS) {
-     return 1;
-   }
-
-   /* Client-side purpose */
-   if (purpose >= CIRCUIT_PURPOSE_C_HS_MIN_ &&
-       purpose <= CIRCUIT_PURPOSE_C_HS_MAX_) {
-     return 1;
-   }
-
-   /* Service-side purpose */
-   if (purpose >= CIRCUIT_PURPOSE_S_HS_MIN_ &&
-       purpose <= CIRCUIT_PURPOSE_S_HS_MAX_) {
-     return 1;
-   }
-
-   return 0;
+  /* HS Vanguard purpose. */
+  if (circuit_purpose_is_hs_vanguards(purpose)) {
+    return 1;
+  }
+
+  /* Client-side purpose */
+  if (circuit_purpose_is_hs_client(purpose)) {
+    return 1;
+  }
+
+  /* Service-side purpose */
+  if (circuit_purpose_is_hs_service(purpose)) {
+    return 1;
+  }
+
+  return 0;
+}
+
+/** Retrun true iff the given circuit is an HS client circuit. */
+bool
+circuit_purpose_is_hs_client(const uint8_t purpose)
+{
+  return (purpose >= CIRCUIT_PURPOSE_C_HS_MIN_ &&
+          purpose <= CIRCUIT_PURPOSE_C_HS_MAX_);
+}
+
+/** Retrun true iff the given circuit is an HS service circuit. */
+bool
+circuit_purpose_is_hs_service(const uint8_t purpose)
+{
+  return (purpose >= CIRCUIT_PURPOSE_S_HS_MIN_ &&
+          purpose <= CIRCUIT_PURPOSE_S_HS_MAX_);
+}
+
+/** Retrun true iff the given circuit is an HS Vanguards circuit. */
+bool
+circuit_purpose_is_hs_vanguards(const uint8_t purpose)
+{
+  return (purpose == CIRCUIT_PURPOSE_HS_VANGUARDS);
+}
+
+/** Retrun true iff the given circuit is an HS v2 circuit. */
+bool
+circuit_is_hs_v2(const circuit_t *circ)
+{
+  return (CIRCUIT_IS_ORIGIN(circ) &&
+          (CONST_TO_ORIGIN_CIRCUIT(circ)->rend_data != NULL));
+}
+
+/** Retrun true iff the given circuit is an HS v3 circuit. */
+bool
+circuit_is_hs_v3(const circuit_t *circ)
+{
+  return (CIRCUIT_IS_ORIGIN(circ) &&
+          (CONST_TO_ORIGIN_CIRCUIT(circ)->hs_ident != NULL));
 }
 
 /**
@@ -2523,8 +2568,7 @@ circuit_get_open_circ_or_launch(entry_connection_t *conn,
           circ->rend_data = rend_data_dup(edge_conn->rend_data);
         } else if (edge_conn->hs_ident) {
           circ->hs_ident =
-            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk,
-                                 HS_IDENT_CIRCUIT_INTRO);
+            hs_ident_circuit_new(&edge_conn->hs_ident->identity_pk);
         }
         if (circ->base_.purpose == CIRCUIT_PURPOSE_C_ESTABLISH_REND &&
             circ->base_.state == CIRCUIT_STATE_OPEN)
@@ -2596,6 +2640,12 @@ link_apconn_to_circ(entry_connection_t *apconn, origin_circuit_t *circ,
   /* add it into the linked list of streams on this circuit */
   log_debug(LD_APP|LD_CIRC, "attaching new conn to circ. n_circ_id %u.",
             (unsigned)circ->base_.n_circ_id);
+
+  /* If this is the first stream on this circuit, tell circpad
+   * that streams are attached */
+  if (!circ->p_streams)
+    circpad_machine_event_circ_has_streams(circ);
+
   /* reset it, so we can measure circ timeouts */
   ENTRY_TO_CONN(apconn)->timestamp_last_read_allowed = time(NULL);
   ENTRY_TO_EDGE_CONN(apconn)->next_stream = circ->p_streams;
@@ -3070,7 +3120,7 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
     /* Take specific actions if we are repurposing a hidden service circuit. */
     if (circuit_purpose_is_hidden_service(circ->purpose) &&
         !circuit_purpose_is_hidden_service(new_purpose)) {
-      hs_circ_cleanup(circ);
+      hs_circ_cleanup_on_repurpose(circ);
     }
   }
 
@@ -3080,6 +3130,8 @@ circuit_change_purpose(circuit_t *circ, uint8_t new_purpose)
   if (CIRCUIT_IS_ORIGIN(circ)) {
     control_event_circuit_purpose_changed(TO_ORIGIN_CIRCUIT(circ),
                                           old_purpose);
+
+    circpad_machine_event_circ_purpose_changed(TO_ORIGIN_CIRCUIT(circ));
   }
 }
 
@@ -3113,7 +3165,9 @@ circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len)
 {
   if (!circ) return;
 
-  tor_assert_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE);
+  tor_assertf_nonfatal(relay_body_len <= RELAY_PAYLOAD_SIZE,
+                       "Wrong relay_body_len: %d (should be at most %d)",
+                       relay_body_len, RELAY_PAYLOAD_SIZE);
 
   circ->n_delivered_written_circ_bw =
       tor_add_u32_nowrap(circ->n_delivered_written_circ_bw, relay_body_len);
diff --git a/src/core/or/circuituse.h b/src/core/or/circuituse.h
index 25588dbb11..95d36d6474 100644
--- a/src/core/or/circuituse.h
+++ b/src/core/or/circuituse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -64,6 +64,15 @@ int hostname_in_track_host_exits(const or_options_t *options,
 void mark_circuit_unusable_for_new_conns(origin_circuit_t *circ);
 
 int circuit_purpose_is_hidden_service(uint8_t);
+
+/* Series of helper functions for hidden services. */
+bool circuit_purpose_is_hs_client(const uint8_t purpose);
+bool circuit_purpose_is_hs_service(const uint8_t purpose);
+bool circuit_purpose_is_hs_vanguards(const uint8_t purpose);
+
+bool circuit_is_hs_v2(const circuit_t *circ);
+bool circuit_is_hs_v3(const circuit_t *circ);
+
 int circuit_should_use_vanguards(uint8_t);
 void circuit_sent_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
 void circuit_read_valid_data(origin_circuit_t *circ, uint16_t relay_body_len);
diff --git a/src/core/or/command.c b/src/core/or/command.c
index 5fb6640c22..8a1d2066cc 100644
--- a/src/core/or/command.c
+++ b/src/core/or/command.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,11 +49,12 @@
 #include "core/or/dos.h"
 #include "core/or/onion.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/relay/circuitbuild_relay.h"
 #include "feature/relay/routermode.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -182,7 +183,7 @@ command_process_cell(channel_t *chan, cell_t *cell)
     command_time_process_cell(cl, cn, & tp ## time ,            \
                               command_process_ ## tp ## _cell);  \
   } STMT_END
-#else /* !(defined(KEEP_TIMING_STATS)) */
+#else /* !defined(KEEP_TIMING_STATS) */
 #define PROCESS_CELL(tp, cl, cn) command_process_ ## tp ## _cell(cl, cn)
 #endif /* defined(KEEP_TIMING_STATS) */
 
@@ -217,23 +218,6 @@ command_process_cell(channel_t *chan, cell_t *cell)
   }
 }
 
-/** Process an incoming var_cell from a channel; in the current protocol all
- * the var_cells are handshake-related and handled below the channel layer,
- * so this just logs a warning and drops the cell.
- */
-
-void
-command_process_var_cell(channel_t *chan, var_cell_t *var_cell)
-{
-  tor_assert(chan);
-  tor_assert(var_cell);
-
-  log_info(LD_PROTOCOL,
-           "Received unexpected var_cell above the channel layer of type %d"
-           "; dropping it.",
-           var_cell->command);
-}
-
 /** Process a 'create' <b>cell</b> that just arrived from <b>chan</b>. Make a
  * new circuit with the p_circ_id specified in cell. Put the circuit in state
  * onionskin_pending, and pass the onionskin to the cpuworker. Circ will get
@@ -685,8 +669,7 @@ command_setup_channel(channel_t *chan)
   tor_assert(chan);
 
   channel_set_cell_handlers(chan,
-                            command_process_cell,
-                            command_process_var_cell);
+                            command_process_cell);
 }
 
 /** Given a listener, install the right handler to process incoming
diff --git a/src/core/or/command.h b/src/core/or/command.h
index 8c90e1de6f..14ebb4a339 100644
--- a/src/core/or/command.h
+++ b/src/core/or/command.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,6 @@
 #include "core/or/channel.h"
 
 void command_process_cell(channel_t *chan, cell_t *cell);
-void command_process_var_cell(channel_t *chan, var_cell_t *cell);
 void command_setup_channel(channel_t *chan);
 void command_setup_listener(channel_listener_t *chan_l);
 
diff --git a/src/core/or/connection_edge.c b/src/core/or/connection_edge.c
index 84b80313ce..1394a41c73 100644
--- a/src/core/or/connection_edge.c
+++ b/src/core/or/connection_edge.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -62,21 +62,24 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
 #include "core/or/policies.h"
 #include "core/or/reasons.h"
 #include "core/or/relay.h"
+#include "core/or/sendme.h"
 #include "core/proto/proto_http.h"
 #include "core/proto/proto_socks.h"
 #include "feature/client/addressmap.h"
 #include "feature/client/circpathbias.h"
 #include "feature/client/dnsserv.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hibernate/hibernate.h"
@@ -97,7 +100,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "core/or/cell_st.h"
@@ -300,6 +303,11 @@ connection_edge_process_inbuf(edge_connection_t *conn, int package_partial)
       }
       return 0;
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
       if (connection_edge_package_raw_inbuf(conn, package_partial, NULL) < 0) {
         /* (We already sent an end cell if possible) */
@@ -424,6 +432,21 @@ warn_if_hs_unreachable(const edge_connection_t *conn, uint8_t reason)
   }
 }
 
+/** Given a TTL (in seconds) from a DNS response or from a relay, determine
+ * what TTL clients and relays should actually use for caching it. */
+uint32_t
+clip_dns_ttl(uint32_t ttl)
+{
+  /* This logic is a defense against "DefectTor" DNS-based traffic
+   * confirmation attacks, as in https://nymity.ch/tor-dns/tor-dns.pdf .
+   * We only give two values: a "low" value and a "high" value.
+   */
+  if (ttl < MIN_DNS_TTL)
+    return MIN_DNS_TTL;
+  else
+    return MAX_DNS_TTL;
+}
+
 /** Send a relay end cell from stream <b>conn</b> down conn's circuit, and
  * remember that we've done so.  If this is not a client connection, set the
  * relay end cell's reason for closing as <b>reason</b>.
@@ -472,7 +495,7 @@ connection_edge_end(edge_connection_t *conn, uint8_t reason)
       memcpy(payload+1, tor_addr_to_in6_addr8(&conn->base_.addr), 16);
       addrlen = 16;
     }
-    set_uint32(payload+1+addrlen, htonl(dns_clip_ttl(conn->address_ttl)));
+    set_uint32(payload+1+addrlen, htonl(clip_dns_ttl(conn->address_ttl)));
     payload_len += 4+addrlen;
   }
 
@@ -754,8 +777,13 @@ connection_edge_flushed_some(edge_connection_t *conn)
 {
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
+      if (! conn->base_.linked) {
+        note_user_activity(approx_time());
+      }
+
+      FALLTHROUGH;
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       break;
   }
   return 0;
@@ -779,7 +807,7 @@ connection_edge_finished_flushing(edge_connection_t *conn)
   switch (conn->base_.state) {
     case AP_CONN_STATE_OPEN:
     case EXIT_CONN_STATE_OPEN:
-      connection_edge_consider_sending_sendme(conn);
+      sendme_connection_edge_consider_sending(conn);
       return 0;
     case AP_CONN_STATE_SOCKS_WAIT:
     case AP_CONN_STATE_NATD_WAIT:
@@ -832,7 +860,7 @@ connected_cell_format_payload(uint8_t *payload_out,
     return -1;
   }
 
-  set_uint32(payload_out + connected_payload_len, htonl(dns_clip_ttl(ttl)));
+  set_uint32(payload_out + connected_payload_len, htonl(clip_dns_ttl(ttl)));
   connected_payload_len += 4;
 
   tor_assert(connected_payload_len <= MAX_CONNECTED_CELL_PAYLOAD_LEN);
@@ -1211,7 +1239,7 @@ connection_ap_rescan_and_attach_pending(void)
     entry_conn->marked_pending_circ_line = 0;   \
     entry_conn->marked_pending_circ_file = 0;   \
   } while (0)
-#else /* !(defined(DEBUGGING_17659)) */
+#else /* !defined(DEBUGGING_17659) */
 #define UNMARK() do { } while (0)
 #endif /* defined(DEBUGGING_17659) */
 
@@ -1540,6 +1568,107 @@ consider_plaintext_ports(entry_connection_t *conn, uint16_t port)
   return 0;
 }
 
+/** Parse the given hostname in address. Returns true if the parsing was
+ * successful and type_out contains the type of the hostname. Else, false is
+ * returned which means it was not recognized and type_out is set to
+ * BAD_HOSTNAME.
+ *
+ * The possible recognized forms are (where true is returned):
+ *
+ *  If address is of the form "y.onion" with a well-formed handle y:
+ *     Put a NUL after y, lower-case it, and return ONION_V2_HOSTNAME or
+ *     ONION_V3_HOSTNAME depending on the HS version.
+ *
+ *  If address is of the form "x.y.onion" with a well-formed handle x:
+ *     Drop "x.", put a NUL after y, lower-case it, and return
+ *     ONION_V2_HOSTNAME or ONION_V3_HOSTNAME depending on the HS version.
+ *
+ * If address is of the form "y.onion" with a badly-formed handle y:
+ *     Return BAD_HOSTNAME and log a message.
+ *
+ * If address is of the form "y.exit":
+ *     Put a NUL after y and return EXIT_HOSTNAME.
+ *
+ * Otherwise:
+ *     Return NORMAL_HOSTNAME and change nothing.
+ */
+STATIC bool
+parse_extended_hostname(char *address, hostname_type_t *type_out)
+{
+  char *s;
+  char *q;
+  char query[HS_SERVICE_ADDR_LEN_BASE32+1];
+
+  s = strrchr(address,'.');
+  if (!s) {
+    *type_out = NORMAL_HOSTNAME; /* no dot, thus normal */
+    goto success;
+  }
+  if (!strcmp(s+1,"exit")) {
+    *s = 0; /* NUL-terminate it */
+    *type_out = EXIT_HOSTNAME; /* .exit */
+    goto success;
+  }
+  if (strcmp(s+1,"onion")) {
+    *type_out = NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */
+    goto success;
+  }
+
+  /* so it is .onion */
+  *s = 0; /* NUL-terminate it */
+  /* locate a 'sub-domain' component, in order to remove it */
+  q = strrchr(address, '.');
+  if (q == address) {
+    *type_out = BAD_HOSTNAME;
+    goto failed; /* reject sub-domain, as DNS does */
+  }
+  q = (NULL == q) ? address : q + 1;
+  if (strlcpy(query, q, HS_SERVICE_ADDR_LEN_BASE32+1) >=
+      HS_SERVICE_ADDR_LEN_BASE32+1) {
+    *type_out = BAD_HOSTNAME;
+    goto failed;
+  }
+  if (q != address) {
+    memmove(address, q, strlen(q) + 1 /* also get \0 */);
+  }
+  /* v2 onion address check. */
+  if (strlen(query) == REND_SERVICE_ID_LEN_BASE32) {
+    *type_out = ONION_V2_HOSTNAME;
+    if (rend_valid_v2_service_id(query)) {
+      goto success;
+    }
+    goto failed;
+  }
+
+  /* v3 onion address check. */
+  if (strlen(query) == HS_SERVICE_ADDR_LEN_BASE32) {
+    *type_out = ONION_V3_HOSTNAME;
+    if (hs_address_is_valid(query)) {
+      goto success;
+    }
+    goto failed;
+  }
+
+  /* Reaching this point, nothing was recognized. */
+  *type_out = BAD_HOSTNAME;
+  goto failed;
+
+ success:
+  return true;
+ failed:
+  /* otherwise, return to previous state and return 0 */
+  *s = '.';
+  const bool is_onion = (*type_out == ONION_V2_HOSTNAME) ||
+    (*type_out == ONION_V3_HOSTNAME);
+  log_warn(LD_APP, "Invalid %shostname %s; rejecting",
+           is_onion ? "onion " : "",
+           safe_str_client(address));
+  if (*type_out == ONION_V3_HOSTNAME) {
+      *type_out = BAD_HOSTNAME;
+  }
+  return false;
+}
+
 /** How many times do we try connecting with an exit configured via
  * TrackHostExits before concluding that it won't work any more and trying a
  * different one? */
@@ -2007,16 +2136,15 @@ connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
   const int automap = rr.automap;
   const addressmap_entry_source_t exit_source = rr.exit_source;
 
-  /* Now, we parse the address to see if it's an .onion or .exit or
-   * other special address.
-   */
-  const hostname_type_t addresstype = parse_extended_hostname(socks->address);
-
   /* Now see whether the hostname is bogus.  This could happen because of an
    * onion hostname whose format we don't recognize. */
-  if (addresstype == BAD_HOSTNAME) {
+  hostname_type_t addresstype;
+  if (!parse_extended_hostname(socks->address, &addresstype)) {
     control_event_client_status(LOG_WARN, "SOCKS_BAD_HOSTNAME HOSTNAME=%s",
                                 escaped(socks->address));
+    if (addresstype == BAD_HOSTNAME) {
+      conn->socks_request->socks_extended_error_code = SOCKS5_HS_BAD_ADDRESS;
+    }
     connection_mark_unattached_ap(conn, END_STREAM_REASON_TORPROTOCOL);
     return -1;
   }
@@ -2803,6 +2931,31 @@ connection_ap_process_natd(entry_connection_t *conn)
   return connection_ap_rewrite_and_attach_if_allowed(conn, NULL, NULL);
 }
 
+static const char HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG[] =
+  "HTTP/1.0 405 Method Not Allowed\r\n"
+  "Content-Type: text/html; charset=iso-8859-1\r\n\r\n"
+  "<html>\n"
+  "<head>\n"
+  "<title>This is an HTTP CONNECT tunnel, not a full HTTP Proxy</title>\n"
+  "</head>\n"
+  "<body>\n"
+  "<h1>This is an HTTP CONNECT tunnel, not an HTTP proxy.</h1>\n"
+  "<p>\n"
+  "It appears you have configured your web browser to use this Tor port as\n"
+  "an HTTP proxy.\n"
+  "</p><p>\n"
+  "This is not correct: This port is configured as a CONNECT tunnel, not\n"
+  "an HTTP proxy. Please configure your client accordingly.  You can also\n"
+  "use HTTPS; then the client should automatically use HTTP CONNECT."
+  "</p>\n"
+  "<p>\n"
+  "See <a href=\"https://www.torproject.org/documentation.html\">"
+  "https://www.torproject.org/documentation.html</a> for more "
+  "information.\n"
+  "</p>\n"
+  "</body>\n"
+  "</html>\n";
+
 /** Called on an HTTP CONNECT entry connection when some bytes have arrived,
  * but we have not yet received a full HTTP CONNECT request.  Try to parse an
  * HTTP CONNECT request from the connection's inbuf.  On success, set up the
@@ -2843,7 +2996,7 @@ connection_ap_process_http_connect(entry_connection_t *conn)
   tor_assert(command);
   tor_assert(addrport);
   if (strcasecmp(command, "connect")) {
-    errmsg = "HTTP/1.0 405 Method Not Allowed\r\n\r\n";
+    errmsg = HTTP_CONNECT_IS_NOT_AN_HTTP_PROXY_MSG;
     goto err;
   }
 
@@ -3309,8 +3462,9 @@ tell_controller_about_resolved_result(entry_connection_t *conn,
   expires = time(NULL) + ttl;
   if (answer_type == RESOLVED_TYPE_IPV4 && answer_len >= 4) {
     char *cp = tor_dup_ip(ntohl(get_uint32(answer)));
-    control_event_address_mapped(conn->socks_request->address,
-                                 cp, expires, NULL, 0);
+    if (cp)
+      control_event_address_mapped(conn->socks_request->address,
+                                   cp, expires, NULL, 0);
     tor_free(cp);
   } else if (answer_type == RESOLVED_TYPE_HOSTNAME && answer_len < 256) {
     char *cp = tor_strndup(answer, answer_len);
@@ -3383,7 +3537,7 @@ connection_ap_handshake_socks_resolved,(entry_connection_t *conn,
       }
     } else if (answer_type == RESOLVED_TYPE_IPV6 && answer_len == 16) {
       tor_addr_t a;
-      tor_addr_from_ipv6_bytes(&a, (char*)answer);
+      tor_addr_from_ipv6_bytes(&a, answer);
       if (! tor_addr_is_null(&a)) {
         client_dns_set_addressmap(conn,
                                   conn->socks_request->address, &a,
@@ -3484,11 +3638,17 @@ connection_ap_handshake_socks_reply(entry_connection_t *conn, char *reply,
                                     size_t replylen, int endreason)
 {
   char buf[256];
-  socks5_reply_status_t status =
-    stream_end_reason_to_socks5_response(endreason);
+  socks5_reply_status_t status;
 
   tor_assert(conn->socks_request); /* make sure it's an AP stream */
 
+  if (conn->socks_request->socks_use_extended_errors &&
+      conn->socks_request->socks_extended_error_code != 0) {
+    status = conn->socks_request->socks_extended_error_code;
+  } else {
+    status = stream_end_reason_to_socks5_response(endreason);
+  }
+
   if (!SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command)) {
     control_event_stream_status(conn, status==SOCKS5_SUCCEEDED ?
                                 STREAM_EVENT_SUCCEEDED : STREAM_EVENT_FAILED,
@@ -3706,6 +3866,10 @@ handle_hs_exit_conn(circuit_t *circ, edge_connection_t *conn)
   /* Link the circuit and the connection crypt path. */
   conn->cpath_layer = origin_circ->cpath->prev;
 
+  /* If this is the first stream on this circuit, tell circpad */
+  if (!origin_circ->p_streams)
+    circpad_machine_event_circ_has_streams(origin_circ);
+
   /* Add it into the linked list of p_streams on this circuit */
   conn->next_stream = origin_circ->p_streams;
   origin_circ->p_streams = conn;
@@ -3796,6 +3960,7 @@ connection_exit_begin_conn(cell_t *cell, circuit_t *circ)
 
   if (! bcell.is_begindir) {
     /* Steal reference */
+    tor_assert(bcell.address);
     address = bcell.address;
     port = bcell.port;
 
@@ -4300,68 +4465,6 @@ connection_ap_can_use_exit(const entry_connection_t *conn,
   return 1;
 }
 
-/** If address is of the form "y.onion" with a well-formed handle y:
- *     Put a NUL after y, lower-case it, and return ONION_V2_HOSTNAME or
- *     ONION_V3_HOSTNAME depending on the HS version.
- *
- *  If address is of the form "x.y.onion" with a well-formed handle x:
- *     Drop "x.", put a NUL after y, lower-case it, and return
- *     ONION_V2_HOSTNAME or ONION_V3_HOSTNAME depending on the HS version.
- *
- * If address is of the form "y.onion" with a badly-formed handle y:
- *     Return BAD_HOSTNAME and log a message.
- *
- * If address is of the form "y.exit":
- *     Put a NUL after y and return EXIT_HOSTNAME.
- *
- * Otherwise:
- *     Return NORMAL_HOSTNAME and change nothing.
- */
-hostname_type_t
-parse_extended_hostname(char *address)
-{
-    char *s;
-    char *q;
-    char query[HS_SERVICE_ADDR_LEN_BASE32+1];
-
-    s = strrchr(address,'.');
-    if (!s)
-      return NORMAL_HOSTNAME; /* no dot, thus normal */
-    if (!strcmp(s+1,"exit")) {
-      *s = 0; /* NUL-terminate it */
-      return EXIT_HOSTNAME; /* .exit */
-    }
-    if (strcmp(s+1,"onion"))
-      return NORMAL_HOSTNAME; /* neither .exit nor .onion, thus normal */
-
-    /* so it is .onion */
-    *s = 0; /* NUL-terminate it */
-    /* locate a 'sub-domain' component, in order to remove it */
-    q = strrchr(address, '.');
-    if (q == address) {
-      goto failed; /* reject sub-domain, as DNS does */
-    }
-    q = (NULL == q) ? address : q + 1;
-    if (strlcpy(query, q, HS_SERVICE_ADDR_LEN_BASE32+1) >=
-        HS_SERVICE_ADDR_LEN_BASE32+1)
-      goto failed;
-    if (q != address) {
-      memmove(address, q, strlen(q) + 1 /* also get \0 */);
-    }
-    if (rend_valid_v2_service_id(query)) {
-      return ONION_V2_HOSTNAME; /* success */
-    }
-    if (hs_address_is_valid(query)) {
-      return ONION_V3_HOSTNAME;
-    }
- failed:
-    /* otherwise, return to previous state and return 0 */
-    *s = '.';
-    log_warn(LD_APP, "Invalid onion hostname %s; rejecting",
-             safe_str_client(address));
-    return BAD_HOSTNAME;
-}
-
 /** Return true iff the (possibly NULL) <b>alen</b>-byte chunk of memory at
  * <b>a</b> is equal to the (possibly NULL) <b>blen</b>-byte chunk of memory
  * at <b>b</b>. */
@@ -4565,6 +4668,25 @@ circuit_clear_isolation(origin_circuit_t *circ)
   circ->socks_username_len = circ->socks_password_len = 0;
 }
 
+/** Send an END and mark for close the given edge connection conn using the
+ * given reason that has to be a stream reason.
+ *
+ * Note: We don't unattached the AP connection (if applicable) because we
+ * don't want to flush the remaining data. This function aims at ending
+ * everything quickly regardless of the connection state.
+ *
+ * This function can't fail and does nothing if conn is NULL. */
+void
+connection_edge_end_close(edge_connection_t *conn, uint8_t reason)
+{
+  if (!conn) {
+    return;
+  }
+
+  connection_edge_end(conn, reason);
+  connection_mark_for_close(TO_CONN(conn));
+}
+
 /** Free all storage held in module-scoped variables for connection_edge.c */
 void
 connection_edge_free_all(void)
diff --git a/src/core/or/connection_edge.h b/src/core/or/connection_edge.h
index 68d8b19a11..8c06af5664 100644
--- a/src/core/or/connection_edge.h
+++ b/src/core/or/connection_edge.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,6 +71,15 @@ entry_connection_t *EDGE_TO_ENTRY_CONN(edge_connection_t *);
 #define connection_mark_unattached_ap(conn, endreason)                  \
   connection_mark_unattached_ap_((conn), (endreason), __LINE__, SHORT_FILE__)
 
+/** Possible return values for parse_extended_hostname. */
+typedef enum hostname_type_t {
+  BAD_HOSTNAME,
+  EXIT_HOSTNAME,
+  NORMAL_HOSTNAME,
+  ONION_V2_HOSTNAME,
+  ONION_V3_HOSTNAME,
+} hostname_type_t;
+
 MOCK_DECL(void,connection_mark_unattached_ap_,
           (entry_connection_t *conn, int endreason,
            int line, const char *file));
@@ -80,6 +89,7 @@ int connection_edge_process_inbuf(edge_connection_t *conn,
 int connection_edge_destroy(circid_t circ_id, edge_connection_t *conn);
 int connection_edge_end(edge_connection_t *conn, uint8_t reason);
 int connection_edge_end_errno(edge_connection_t *conn);
+void connection_edge_end_close(edge_connection_t *conn, uint8_t reason);
 int connection_edge_flushed_some(edge_connection_t *conn);
 int connection_edge_finished_flushing(edge_connection_t *conn);
 int connection_edge_finished_connecting(edge_connection_t *conn);
@@ -154,13 +164,6 @@ int connection_ap_handshake_rewrite_and_attach(entry_connection_t *conn,
                                                origin_circuit_t *circ,
                                                crypt_path_t *cpath);
 
-/** Possible return values for parse_extended_hostname. */
-typedef enum hostname_type_t {
-  NORMAL_HOSTNAME, ONION_V2_HOSTNAME, ONION_V3_HOSTNAME,
-  EXIT_HOSTNAME, BAD_HOSTNAME
-} hostname_type_t;
-hostname_type_t parse_extended_hostname(char *address);
-
 #if defined(HAVE_NET_IF_H) && defined(HAVE_NET_PFVAR_H)
 int get_pf_socket(void);
 #endif
@@ -179,6 +182,21 @@ void connection_ap_warn_and_unmark_if_pending_circ(
                                              entry_connection_t *entry_conn,
                                              const char *where);
 
+/** Lowest value for DNS ttl that a server should give or a client should
+ * believe. */
+#define MIN_DNS_TTL (5*60)
+/** Highest value for DNS ttl that a server should give or a client should
+ * believe. */
+#define MAX_DNS_TTL (60*60)
+/** How long do we keep DNS cache entries before purging them (regardless of
+ * their TTL)? */
+#define MAX_DNS_ENTRY_AGE (3*60*60)
+/** How long do we cache/tell clients to cache DNS records when no TTL is
+ * known? */
+#define DEFAULT_DNS_TTL (30*60)
+
+uint32_t clip_dns_ttl(uint32_t ttl);
+
 int connection_half_edge_is_valid_data(const smartlist_t *half_conns,
                                        streamid_t stream_id);
 int connection_half_edge_is_valid_sendme(const smartlist_t *half_conns,
@@ -218,6 +236,8 @@ void half_edge_free_(struct half_edge_t *he);
 
 #ifdef CONNECTION_EDGE_PRIVATE
 
+STATIC bool parse_extended_hostname(char *address, hostname_type_t *type_out);
+
 /** A parsed BEGIN or BEGIN_DIR cell */
 typedef struct begin_cell_t {
   /** The address the client has asked us to connect to, or NULL if this is
diff --git a/src/core/or/connection_or.c b/src/core/or/connection_or.c
index 67157d8d0b..b88d1b6afb 100644
--- a/src/core/or/connection_or.c
+++ b/src/core/or/connection_or.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,17 +18,19 @@
  * tortls.c) which it uses as its TLS stream.  It is responsible for
  * sending and receiving cells over that TLS.
  *
- * This module also implements the client side of the v3 Tor link handshake,
+ * This module also implements the client side of the v3 (and greater) Tor
+ * link handshake.
  **/
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).
  */
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CONNECTION_OR_PRIVATE
+#define ORCONN_EVENT_PRIVATE
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitbuild.h"
@@ -38,14 +40,14 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
-#include "lib/crypt_ops/crypto_rand.h"
+#include "feature/relay/relay_handshake.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/client/entrynodes.h"
 #include "lib/geoip/geoip.h"
 #include "core/mainloop/mainloop.h"
-#include "trunnel/link_handshake.h"
+#include "trunnel/netinfo.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
@@ -76,7 +78,8 @@
 #include "lib/crypt_ops/crypto_format.h"
 
 #include "lib/tls/tortls.h"
-#include "lib/tls/x509.h"
+
+#include "core/or/orconn_event.h"
 
 static int connection_tls_finish_handshake(or_connection_t *conn);
 static int connection_or_launch_v3_or_handshake(or_connection_t *conn);
@@ -91,13 +94,6 @@ static unsigned int
 connection_or_is_bad_for_new_circs(or_connection_t *or_conn);
 static void connection_or_mark_bad_for_new_circs(or_connection_t *or_conn);
 
-/*
- * Call this when changing connection state, so notifications to the owning
- * channel can be handled.
- */
-
-static void connection_or_change_state(or_connection_t *conn, uint8_t state);
-
 static void connection_or_check_canonicity(or_connection_t *conn,
                                            int started_here);
 
@@ -112,10 +108,6 @@ TO_OR_CONN(connection_t *c)
   return DOWNCAST(or_connection_t, c);
 }
 
-/** Global map between Extended ORPort identifiers and OR
- *  connections. */
-static digestmap_t *orconn_ext_or_id_map = NULL;
-
 /** Clear clear conn->identity_digest and update other data
  * structures as appropriate.*/
 void
@@ -201,71 +193,6 @@ connection_or_set_identity_digest(or_connection_t *conn,
     channel_set_identity_digest(chan, rsa_digest, ed_id);
 }
 
-/** Remove the Extended ORPort identifier of <b>conn</b> from the
- *  global identifier list. Also, clear the identifier from the
- *  connection itself. */
-void
-connection_or_remove_from_ext_or_id_map(or_connection_t *conn)
-{
-  or_connection_t *tmp;
-  if (!orconn_ext_or_id_map)
-    return;
-  if (!conn->ext_or_conn_id)
-    return;
-
-  tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id);
-  if (!tor_digest_is_zero(conn->ext_or_conn_id))
-    tor_assert(tmp == conn);
-
-  memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN);
-}
-
-/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such
- * connection is found. */
-or_connection_t *
-connection_or_get_by_ext_or_id(const char *id)
-{
-  if (!orconn_ext_or_id_map)
-    return NULL;
-  return digestmap_get(orconn_ext_or_id_map, id);
-}
-
-/** Deallocate the global Extended ORPort identifier list */
-void
-connection_or_clear_ext_or_id_map(void)
-{
-  digestmap_free(orconn_ext_or_id_map, NULL);
-  orconn_ext_or_id_map = NULL;
-}
-
-/** Creates an Extended ORPort identifier for <b>conn</b> and deposits
- *  it into the global list of identifiers. */
-void
-connection_or_set_ext_or_identifier(or_connection_t *conn)
-{
-  char random_id[EXT_OR_CONN_ID_LEN];
-  or_connection_t *tmp;
-
-  if (!orconn_ext_or_id_map)
-    orconn_ext_or_id_map = digestmap_new();
-
-  /* Remove any previous identifiers: */
-  if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id))
-      connection_or_remove_from_ext_or_id_map(conn);
-
-  do {
-    crypto_rand(random_id, sizeof(random_id));
-  } while (digestmap_get(orconn_ext_or_id_map, random_id));
-
-  if (!conn->ext_or_conn_id)
-    conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN);
-
-  memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN);
-
-  tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn);
-  tor_assert(!tmp);
-}
-
 /**************************************************************/
 
 /** Map from a string describing what a non-open OR connection was doing when
@@ -400,23 +327,69 @@ connection_or_report_broken_states(int severity, int domain)
   smartlist_free(items);
 }
 
+/**
+ * Helper function to publish an OR connection status event
+ *
+ * Publishes a messages to subscribers of ORCONN messages, and sends
+ * the control event.
+ **/
+void
+connection_or_event_status(or_connection_t *conn, or_conn_status_event_t tp,
+                           int reason)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  msg->status = tp;
+  msg->reason = reason;
+  orconn_status_publish(msg);
+  control_event_or_conn_status(conn, tp, reason);
+}
+
+/**
+ * Helper function to publish a state change message
+ *
+ * connection_or_change_state() calls this to notify subscribers about
+ * a change of an OR connection state.
+ **/
+static void
+connection_or_state_publish(const or_connection_t *conn, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = conn->base_.global_identifier;
+  if (conn->is_pt) {
+    /* Do extra decoding because conn->proxy_type indicates the proxy
+     * protocol that tor uses to talk with the transport plugin,
+     * instead of PROXY_PLUGGABLE. */
+    tor_assert_nonfatal(conn->proxy_type != PROXY_NONE);
+    msg->proxy_type = PROXY_PLUGGABLE;
+  } else {
+    msg->proxy_type = conn->proxy_type;
+  }
+  msg->state = state;
+  if (conn->chan) {
+    msg->chan = TLS_CHAN_TO_BASE(conn->chan)->global_identifier;
+  } else {
+    msg->chan = 0;
+  }
+  orconn_state_publish(msg);
+}
+
 /** Call this to change or_connection_t states, so the owning channel_tls_t can
  * be notified.
  */
 
-static void
-connection_or_change_state(or_connection_t *conn, uint8_t state)
+MOCK_IMPL(STATIC void,
+connection_or_change_state,(or_connection_t *conn, uint8_t state))
 {
-  uint8_t old_state;
-
   tor_assert(conn);
 
-  old_state = conn->base_.state;
   conn->base_.state = state;
 
+  connection_or_state_publish(conn, state);
   if (conn->chan)
-    channel_tls_handle_state_change_on_orconn(conn->chan, conn,
-                                              old_state, state);
+    channel_tls_handle_state_change_on_orconn(conn->chan, conn, state);
 }
 
 /** Return the number of circuits using an or_connection_t; this used to
@@ -676,6 +649,19 @@ connection_or_finished_flushing(or_connection_t *conn)
 
   switch (conn->base_.state) {
     case OR_CONN_STATE_PROXY_HANDSHAKING:
+      /* PROXY_HAPROXY gets connected by receiving an ack. */
+      if (conn->proxy_type == PROXY_HAPROXY) {
+        tor_assert(TO_CONN(conn)->proxy_state == PROXY_HAPROXY_WAIT_FOR_FLUSH);
+        TO_CONN(conn)->proxy_state = PROXY_CONNECTED;
+
+        if (connection_tls_start_handshake(conn, 0) < 0) {
+          /* TLS handshaking error of some kind. */
+          connection_or_close_for_error(conn, 0);
+          return -1;
+        }
+        break;
+      }
+      break;
     case OR_CONN_STATE_OPEN:
     case OR_CONN_STATE_OR_HANDSHAKING_V2:
     case OR_CONN_STATE_OR_HANDSHAKING_V3:
@@ -707,8 +693,6 @@ connection_or_finished_connecting(or_connection_t *or_conn)
 
   log_debug(LD_HANDSHAKE,"OR connect() to router at %s:%u finished.",
             conn->address,conn->port);
-  control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  control_event_boot_first_orconn();
 
   if (proxy_type != PROXY_NONE) {
     /* start proxy handshake */
@@ -717,8 +701,9 @@ connection_or_finished_connecting(or_connection_t *or_conn)
       return -1;
     }
 
-    connection_start_reading(conn);
     connection_or_change_state(or_conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+    connection_start_reading(conn);
+
     return 0;
   }
 
@@ -758,21 +743,27 @@ connection_or_about_to_close(or_connection_t *or_conn)
       entry_guard_chan_failed(TLS_CHAN_TO_BASE(or_conn->chan));
       if (conn->state >= OR_CONN_STATE_TLS_HANDSHAKING) {
         int reason = tls_error_to_orconn_end_reason(or_conn->tls_error);
-        control_event_or_conn_status(or_conn, OR_CONN_EVENT_FAILED,
-                                     reason);
-        if (!authdir_mode_tests_reachability(options))
-          control_event_bootstrap_prob_or(
-                orconn_end_reason_to_control_string(reason),
-                reason, or_conn);
+        connection_or_event_status(or_conn, OR_CONN_EVENT_FAILED,
+                                   reason);
+        if (!authdir_mode_tests_reachability(options)) {
+          const char *warning = NULL;
+          if (reason == END_OR_CONN_REASON_TLS_ERROR && or_conn->tls) {
+            warning = tor_tls_get_last_error_msg(or_conn->tls);
+          }
+          if (warning == NULL) {
+            warning = orconn_end_reason_to_control_string(reason);
+          }
+          control_event_bootstrap_prob_or(warning, reason, or_conn);
+        }
       }
     }
   } else if (conn->hold_open_until_flushed) {
     /* We only set hold_open_until_flushed when we're intentionally
      * closing a connection. */
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   } else if (!tor_digest_is_zero(or_conn->identity_digest)) {
-    control_event_or_conn_status(or_conn, OR_CONN_EVENT_CLOSED,
+    connection_or_event_status(or_conn, OR_CONN_EVENT_CLOSED,
                 tls_error_to_orconn_end_reason(or_conn->tls_error));
   }
 }
@@ -918,12 +909,21 @@ connection_or_check_canonicity(or_connection_t *conn, int started_here)
   }
 
   if (r) {
-    tor_addr_port_t node_ap;
-    node_get_pref_orport(r, &node_ap);
-    /* XXXX proposal 186 is making this more complex.  For now, a conn
-       is canonical when it uses the _preferred_ address. */
-    if (tor_addr_eq(&conn->base_.addr, &node_ap.addr))
+    tor_addr_port_t node_ipv4_ap;
+    tor_addr_port_t node_ipv6_ap;
+    node_get_prim_orport(r, &node_ipv4_ap);
+    node_get_pref_ipv6_orport(r, &node_ipv6_ap);
+    if (tor_addr_eq(&conn->base_.addr, &node_ipv4_ap.addr) ||
+        tor_addr_eq(&conn->base_.addr, &node_ipv6_ap.addr)) {
       connection_or_set_canonical(conn, 1);
+    }
+    /* Choose the correct canonical address and port. */
+    tor_addr_port_t *node_ap;
+    if (tor_addr_family(&conn->base_.addr) == AF_INET) {
+      node_ap = &node_ipv4_ap;
+    } else {
+      node_ap = &node_ipv6_ap;
+    }
     if (!started_here) {
       /* Override the addr/port, so our log messages will make sense.
        * This is dangerous, since if we ever try looking up a conn by
@@ -935,13 +935,14 @@ connection_or_check_canonicity(or_connection_t *conn, int started_here)
        * right IP address and port 56244, that wouldn't be as helpful. now we
        * log the "right" port too, so we know if it's moria1 or moria2.
        */
-      tor_addr_copy(&conn->base_.addr, &node_ap.addr);
-      conn->base_.port = node_ap.port;
+      /* See #33898 for a ticket that resolves this technical debt. */
+      tor_addr_copy(&conn->base_.addr, &node_ap->addr);
+      conn->base_.port = node_ap->port;
     }
     tor_free(conn->nickname);
     conn->nickname = tor_strdup(node_get_nickname(r));
     tor_free(conn->base_.address);
-    conn->base_.address = tor_addr_to_str_dup(&node_ap.addr);
+    conn->base_.address = tor_addr_to_str_dup(&node_ap->addr);
   } else {
     tor_free(conn->nickname);
     conn->nickname = tor_malloc(HEX_DIGEST_LEN+2);
@@ -1229,11 +1230,11 @@ or_connect_failure_ht_hash(const or_connect_failure_entry_t *entry)
 }
 
 HT_PROTOTYPE(or_connect_failure_ht, or_connect_failure_entry_t, node,
-             or_connect_failure_ht_hash, or_connect_failure_ht_eq)
+             or_connect_failure_ht_hash, or_connect_failure_ht_eq);
 
 HT_GENERATE2(or_connect_failure_ht, or_connect_failure_entry_t, node,
              or_connect_failure_ht_hash, or_connect_failure_ht_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 /* Initialize a given connect failure entry with the given identity_digest,
  * addr and port. All field are optional except ocf. */
@@ -1364,7 +1365,7 @@ void
 connection_or_connect_failed(or_connection_t *conn,
                              int reason, const char *msg)
 {
-  control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED, reason);
+  connection_or_event_status(conn, OR_CONN_EVENT_FAILED, reason);
   if (!authdir_mode_tests_reachability(get_options()))
     control_event_bootstrap_prob_or(msg, reason, conn);
   note_or_connect_failed(conn);
@@ -1430,7 +1431,7 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
   int r;
   tor_addr_t proxy_addr;
   uint16_t proxy_port;
-  int proxy_type;
+  int proxy_type, is_pt = 0;
 
   tor_assert(_addr);
   tor_assert(id_digest);
@@ -1471,21 +1472,27 @@ connection_or_connect, (const tor_addr_t *_addr, uint16_t port,
     return NULL;
   }
 
-  connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
-
   conn->is_outgoing = 1;
 
   /* If we are using a proxy server, find it and use it. */
-  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, TO_CONN(conn));
+  r = get_proxy_addrport(&proxy_addr, &proxy_port, &proxy_type, &is_pt,
+                         TO_CONN(conn));
   if (r == 0) {
     conn->proxy_type = proxy_type;
     if (proxy_type != PROXY_NONE) {
       tor_addr_copy(&addr, &proxy_addr);
       port = proxy_port;
       conn->base_.proxy_state = PROXY_INFANT;
+      conn->is_pt = is_pt;
     }
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
   } else {
+    /* This duplication of state change calls is necessary in case we
+     * run into an error condition below */
+    connection_or_change_state(conn, OR_CONN_STATE_CONNECTING);
+    connection_or_event_status(conn, OR_CONN_EVENT_LAUNCHED, 0);
+
     /* get_proxy_addrport() might fail if we have a Bridge line that
        references a transport, but no ClientTransportPlugin lines
        defining its transport proxy. If this is the case, let's try to
@@ -1691,7 +1698,8 @@ connection_tls_continue_handshake(or_connection_t *conn)
 
   switch (result) {
     CASE_TOR_TLS_ERROR_ANY:
-    log_info(LD_OR,"tls error [%s]. breaking connection.",
+      conn->tls_error = result;
+      log_info(LD_OR,"tls error [%s]. breaking connection.",
              tor_tls_err_to_string(result));
       return -1;
     case TOR_TLS_DONE:
@@ -1723,6 +1731,7 @@ connection_tls_continue_handshake(or_connection_t *conn)
       log_debug(LD_OR,"wanted read");
       return 0;
     case TOR_TLS_CLOSE:
+      conn->tls_error = result;
       log_info(LD_OR,"tls closed. breaking connection.");
       return -1;
   }
@@ -1981,8 +1990,8 @@ connection_or_client_learned_peer_id(or_connection_t *conn,
 
     /* Tell the new guard API about the channel failure */
     entry_guard_chan_failed(TLS_CHAN_TO_BASE(conn->chan));
-    control_event_or_conn_status(conn, OR_CONN_EVENT_FAILED,
-                                 END_OR_CONN_REASON_OR_IDENTITY);
+    connection_or_event_status(conn, OR_CONN_EVENT_FAILED,
+                               END_OR_CONN_REASON_OR_IDENTITY);
     if (!authdir_mode_tests_reachability(options))
       control_event_bootstrap_prob_or(
                                 "Unexpected identity in router certificate",
@@ -2222,7 +2231,7 @@ int
 connection_or_set_state_open(or_connection_t *conn)
 {
   connection_or_change_state(conn, OR_CONN_STATE_OPEN);
-  control_event_or_conn_status(conn, OR_CONN_EVENT_CONNECTED, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_CONNECTED, 0);
 
   /* Link protocol 3 appeared in Tor 0.2.3.6-alpha, so any connection
    * that uses an earlier link protocol should not be treated as a relay. */
@@ -2252,6 +2261,8 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
 
   cell_pack(&networkcell, cell, conn->wide_circ_ids);
 
+  /* We need to count padding cells from this non-packed code path
+   * since they are sent via chan->write_cell() (which is not packed) */
   rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   if (cell->command == CELL_PADDING)
     rep_hist_padding_count_write(PADDING_TYPE_CELL);
@@ -2262,7 +2273,7 @@ connection_or_write_cell_to_buf(const cell_t *cell, or_connection_t *conn)
   if (conn->chan) {
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
 
-    if (TLS_CHAN_TO_BASE(conn->chan)->currently_padding) {
+    if (TLS_CHAN_TO_BASE(conn->chan)->padding_enabled) {
       rep_hist_padding_count_write(PADDING_TYPE_ENABLED_TOTAL);
       if (cell->command == CELL_PADDING)
         rep_hist_padding_count_write(PADDING_TYPE_ENABLED_CELL);
@@ -2292,6 +2303,7 @@ connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
   if (conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3)
     or_handshake_state_record_var_cell(conn, conn->handshake_state, cell, 0);
 
+  rep_hist_padding_count_write(PADDING_TYPE_TOTAL);
   /* Touch the channel's active timestamp if there is one */
   if (conn->chan)
     channel_timestamp_active(TLS_CHAN_TO_BASE(conn->chan));
@@ -2428,6 +2440,31 @@ connection_or_send_versions(or_connection_t *conn, int v3_plus)
   return 0;
 }
 
+static netinfo_addr_t *
+netinfo_addr_from_tor_addr(const tor_addr_t *tor_addr)
+{
+  sa_family_t addr_family = tor_addr_family(tor_addr);
+
+  if (BUG(addr_family != AF_INET && addr_family != AF_INET6))
+    return NULL;
+
+  netinfo_addr_t *netinfo_addr = netinfo_addr_new();
+
+  if (addr_family == AF_INET) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV4);
+    netinfo_addr_set_len(netinfo_addr, 4);
+    netinfo_addr_set_addr_ipv4(netinfo_addr, tor_addr_to_ipv4h(tor_addr));
+  } else if (addr_family == AF_INET6) {
+    netinfo_addr_set_addr_type(netinfo_addr, NETINFO_ADDR_TYPE_IPV6);
+    netinfo_addr_set_len(netinfo_addr, 16);
+    uint8_t *ipv6_buf = netinfo_addr_getarray_addr_ipv6(netinfo_addr);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(tor_addr);
+    memcpy(ipv6_buf, in6_addr, 16);
+  }
+
+  return netinfo_addr;
+}
+
 /** Send a NETINFO cell on <b>conn</b>, telling the other server what we know
  * about their address, our address, and the current time. */
 MOCK_IMPL(int,
@@ -2436,8 +2473,7 @@ connection_or_send_netinfo,(or_connection_t *conn))
   cell_t cell;
   time_t now = time(NULL);
   const routerinfo_t *me;
-  int len;
-  uint8_t *out;
+  int r = -1;
 
   tor_assert(conn->handshake_state);
 
@@ -2450,20 +2486,21 @@ connection_or_send_netinfo,(or_connection_t *conn))
   memset(&cell, 0, sizeof(cell_t));
   cell.command = CELL_NETINFO;
 
+  netinfo_cell_t *netinfo_cell = netinfo_cell_new();
+
   /* Timestamp, if we're a relay. */
   if (public_server_mode(get_options()) || ! conn->is_outgoing)
-    set_uint32(cell.payload, htonl((uint32_t)now));
+    netinfo_cell_set_timestamp(netinfo_cell, (uint32_t)now);
 
   /* Their address. */
-  out = cell.payload + 4;
+  const tor_addr_t *remote_tor_addr =
+    !tor_addr_is_null(&conn->real_addr) ? &conn->real_addr : &conn->base_.addr;
   /* We use &conn->real_addr below, unless it hasn't yet been set. If it
    * hasn't yet been set, we know that base_.addr hasn't been tampered with
    * yet either. */
-  len = append_address_to_payload(out, !tor_addr_is_null(&conn->real_addr)
-                                       ? &conn->real_addr : &conn->base_.addr);
-  if (len<0)
-    return -1;
-  out += len;
+  netinfo_addr_t *their_addr = netinfo_addr_from_tor_addr(remote_tor_addr);
+
+  netinfo_cell_set_other_addr(netinfo_cell, their_addr);
 
   /* My address -- only include it if I'm a public relay, or if I'm a
    * bridge and this is an incoming connection. If I'm a bridge and this
@@ -2471,556 +2508,40 @@ connection_or_send_netinfo,(or_connection_t *conn))
   if ((public_server_mode(get_options()) || !conn->is_outgoing) &&
       (me = router_get_my_routerinfo())) {
     tor_addr_t my_addr;
-    *out++ = 1 + !tor_addr_is_null(&me->ipv6_addr);
-
     tor_addr_from_ipv4h(&my_addr, me->addr);
-    len = append_address_to_payload(out, &my_addr);
-    if (len < 0)
-      return -1;
-    out += len;
-
-    if (!tor_addr_is_null(&me->ipv6_addr)) {
-      len = append_address_to_payload(out, &me->ipv6_addr);
-      if (len < 0)
-        return -1;
-    }
-  } else {
-    *out = 0;
-  }
-
-  conn->handshake_state->digest_sent_data = 0;
-  conn->handshake_state->sent_netinfo = 1;
-  connection_or_write_cell_to_buf(&cell, conn);
-
-  return 0;
-}
-
-/** Helper used to add an encoded certs to a cert cell */
-static void
-add_certs_cell_cert_helper(certs_cell_t *certs_cell,
-                           uint8_t cert_type,
-                           const uint8_t *cert_encoded,
-                           size_t cert_len)
-{
-  tor_assert(cert_len <= UINT16_MAX);
-  certs_cell_cert_t *ccc = certs_cell_cert_new();
-  ccc->cert_type = cert_type;
-  ccc->cert_len = cert_len;
-  certs_cell_cert_setlen_body(ccc, cert_len);
-  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
-
-  certs_cell_add_certs(certs_cell, ccc);
-}
-
-/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
- * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
- * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
- * (If <b>cert</b> is NULL, take no action.) */
-static void
-add_x509_cert(certs_cell_t *certs_cell,
-              uint8_t cert_type,
-              const tor_x509_cert_t *cert)
-{
-  if (NULL == cert)
-    return;
 
-  const uint8_t *cert_encoded = NULL;
-  size_t cert_len;
-  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+    uint8_t n_my_addrs = 1 + !tor_addr_is_null(&me->ipv6_addr);
+    netinfo_cell_set_n_my_addrs(netinfo_cell, n_my_addrs);
 
-  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
-}
+    netinfo_cell_add_my_addrs(netinfo_cell,
+                              netinfo_addr_from_tor_addr(&my_addr));
 
-/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
- * that we are building in <b>certs_cell</b>.  Set its type field to
- * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
-static void
-add_ed25519_cert(certs_cell_t *certs_cell,
-                 uint8_t cert_type,
-                 const tor_cert_t *cert)
-{
-  if (NULL == cert)
-    return;
-
-  add_certs_cell_cert_helper(certs_cell, cert_type,
-                             cert->encoded, cert->encoded_len);
-}
-
-#ifdef TOR_UNIT_TESTS
-int certs_cell_ed25519_disabled_for_testing = 0;
-#else
-#define certs_cell_ed25519_disabled_for_testing 0
-#endif
-
-/** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
- * on failure. */
-int
-connection_or_send_certs_cell(or_connection_t *conn)
-{
-  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
-  tor_x509_cert_t *own_link_cert = NULL;
-  var_cell_t *cell;
-
-  certs_cell_t *certs_cell = NULL;
-
-  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
-
-  if (! conn->handshake_state)
-    return -1;
-
-  const int conn_in_server_mode = ! conn->handshake_state->started_here;
-
-  /* Get the encoded values of the X509 certificates */
-  if (tor_tls_get_my_certs(conn_in_server_mode,
-                           &global_link_cert, &id_cert) < 0)
-    return -1;
-
-  if (conn_in_server_mode) {
-    own_link_cert = tor_tls_get_own_cert(conn->tls);
-  }
-  tor_assert(id_cert);
-
-  certs_cell = certs_cell_new();
-
-  /* Start adding certs.  First the link cert or auth1024 cert. */
-  if (conn_in_server_mode) {
-    tor_assert_nonfatal(own_link_cert);
-    add_x509_cert(certs_cell,
-                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
-  } else {
-    tor_assert(global_link_cert);
-    add_x509_cert(certs_cell,
-                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
-  }
-
-  /* Next the RSA->RSA ID cert */
-  add_x509_cert(certs_cell,
-                OR_CERT_TYPE_ID_1024, id_cert);
-
-  /* Next the Ed25519 certs */
-  add_ed25519_cert(certs_cell,
-                   CERTTYPE_ED_ID_SIGN,
-                   get_master_signing_key_cert());
-  if (conn_in_server_mode) {
-    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
-                        certs_cell_ed25519_disabled_for_testing);
-    add_ed25519_cert(certs_cell,
-                     CERTTYPE_ED_SIGN_LINK,
-                     conn->handshake_state->own_link_cert);
-  } else {
-    add_ed25519_cert(certs_cell,
-                     CERTTYPE_ED_SIGN_AUTH,
-                     get_current_auth_key_cert());
-  }
-
-  /* And finally the crosscert. */
-  {
-    const uint8_t *crosscert=NULL;
-    size_t crosscert_len;
-    get_master_rsa_crosscert(&crosscert, &crosscert_len);
-    if (crosscert) {
-      add_certs_cell_cert_helper(certs_cell,
-                               CERTTYPE_RSA1024_ID_EDID,
-                               crosscert, crosscert_len);
+    if (!tor_addr_is_null(&me->ipv6_addr)) {
+      netinfo_cell_add_my_addrs(netinfo_cell,
+                                netinfo_addr_from_tor_addr(&me->ipv6_addr));
     }
   }
 
-  /* We've added all the certs; make the cell. */
-  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
-
-  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
-  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
-  cell = var_cell_new(alloc_len);
-  cell->command = CELL_CERTS;
-  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
-  tor_assert(enc_len > 0 && enc_len <= alloc_len);
-  cell->payload_len = enc_len;
-
-  connection_or_write_var_cell_to_buf(cell, conn);
-  var_cell_free(cell);
-  certs_cell_free(certs_cell);
-  tor_x509_cert_free(own_link_cert);
-
-  return 0;
-}
-
-#ifdef TOR_UNIT_TESTS
-int testing__connection_or_pretend_TLSSECRET_is_supported = 0;
-#else
-#define testing__connection_or_pretend_TLSSECRET_is_supported 0
-#endif
-
-/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
- * we can send and receive. */
-int
-authchallenge_type_is_supported(uint16_t challenge_type)
-{
-  switch (challenge_type) {
-     case AUTHTYPE_RSA_SHA256_TLSSECRET:
-#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
-       return 1;
-#else
-       return testing__connection_or_pretend_TLSSECRET_is_supported;
-#endif
-     case AUTHTYPE_ED25519_SHA256_RFC5705:
-       return 1;
-     case AUTHTYPE_RSA_SHA256_RFC5705:
-     default:
-       return 0;
+  const char *errmsg = NULL;
+  if ((errmsg = netinfo_cell_check(netinfo_cell))) {
+    log_warn(LD_OR, "Failed to validate NETINFO cell with error: %s",
+                    errmsg);
+    goto cleanup;
   }
-}
-
-/** Return true iff <b>challenge_type_a</b> is one that we would rather
- * use than <b>challenge_type_b</b>. */
-int
-authchallenge_type_is_better(uint16_t challenge_type_a,
-                             uint16_t challenge_type_b)
-{
-  /* Any supported type is better than an unsupported one;
-   * all unsupported types are equally bad. */
-  if (!authchallenge_type_is_supported(challenge_type_a))
-    return 0;
-  if (!authchallenge_type_is_supported(challenge_type_b))
-    return 1;
-  /* It happens that types are superior in numerically ascending order.
-   * If that ever changes, this must change too. */
-  return (challenge_type_a > challenge_type_b);
-}
-
-/** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
- * on success, -1 on failure. */
-int
-connection_or_send_auth_challenge_cell(or_connection_t *conn)
-{
-  var_cell_t *cell = NULL;
-  int r = -1;
-  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
-
-  if (! conn->handshake_state)
-    return -1;
-
-  auth_challenge_cell_t *ac = auth_challenge_cell_new();
 
-  tor_assert(sizeof(ac->challenge) == 32);
-  crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
-
-  if (authchallenge_type_is_supported(AUTHTYPE_RSA_SHA256_TLSSECRET))
-    auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
-  /* Disabled, because everything that supports this method also supports
-   * the much-superior ED25519_SHA256_RFC5705 */
-  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
-  if (authchallenge_type_is_supported(AUTHTYPE_ED25519_SHA256_RFC5705))
-    auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
-  auth_challenge_cell_set_n_methods(ac,
-                                    auth_challenge_cell_getlen_methods(ac));
-
-  cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
-  ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
-                                           ac);
-  if (len != cell->payload_len) {
-    /* LCOV_EXCL_START */
-    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
-    goto done;
-    /* LCOV_EXCL_STOP */
+  if (netinfo_cell_encode(cell.payload, CELL_PAYLOAD_SIZE,
+                          netinfo_cell) < 0) {
+    log_warn(LD_OR, "Failed generating NETINFO cell");
+    goto cleanup;
   }
-  cell->command = CELL_AUTH_CHALLENGE;
 
-  connection_or_write_var_cell_to_buf(cell, conn);
-  r = 0;
+  conn->handshake_state->digest_sent_data = 0;
+  conn->handshake_state->sent_netinfo = 1;
+  connection_or_write_cell_to_buf(&cell, conn);
 
- done:
-  var_cell_free(cell);
-  auth_challenge_cell_free(ac);
+  r = 0;
+ cleanup:
+  netinfo_cell_free(netinfo_cell);
 
   return r;
 }
-
-/** Compute the main body of an AUTHENTICATE cell that a client can use
- * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
- * in a var_cell_t.
- *
- * If <b>server</b> is true, only calculate the first
- * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
- * determined by the rest of the handshake, and which match the provided value
- * exactly.
- *
- * If <b>server</b> is false and <b>signing_key</b> is NULL, calculate the
- * first V3_AUTH_BODY_LEN bytes of the authenticator (that is, everything
- * that should be signed), but don't actually sign it.
- *
- * If <b>server</b> is false and <b>signing_key</b> is provided, calculate the
- * entire authenticator, signed with <b>signing_key</b>.
- *
- * Return the length of the cell body on success, and -1 on failure.
- */
-var_cell_t *
-connection_or_compute_authenticate_cell_body(or_connection_t *conn,
-                                             const int authtype,
-                                             crypto_pk_t *signing_key,
-                                      const ed25519_keypair_t *ed_signing_key,
-                                      int server)
-{
-  auth1_t *auth = NULL;
-  auth_ctx_t *ctx = auth_ctx_new();
-  var_cell_t *result = NULL;
-  int old_tlssecrets_algorithm = 0;
-  const char *authtype_str = NULL;
-
-  int is_ed = 0;
-
-  /* assert state is reasonable XXXX */
-  switch (authtype) {
-  case AUTHTYPE_RSA_SHA256_TLSSECRET:
-    authtype_str = "AUTH0001";
-    old_tlssecrets_algorithm = 1;
-    break;
-  case AUTHTYPE_RSA_SHA256_RFC5705:
-    authtype_str = "AUTH0002";
-    break;
-  case AUTHTYPE_ED25519_SHA256_RFC5705:
-    authtype_str = "AUTH0003";
-    is_ed = 1;
-    break;
-  default:
-    tor_assert(0);
-    break;
-  }
-
-  auth = auth1_new();
-  ctx->is_ed = is_ed;
-
-  /* Type: 8 bytes. */
-  memcpy(auth1_getarray_type(auth), authtype_str, 8);
-
-  {
-    const tor_x509_cert_t *id_cert=NULL;
-    const common_digests_t *my_digests, *their_digests;
-    const uint8_t *my_id, *their_id, *client_id, *server_id;
-    if (tor_tls_get_my_certs(server, NULL, &id_cert))
-      goto err;
-    my_digests = tor_x509_cert_get_id_digests(id_cert);
-    their_digests =
-      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
-    tor_assert(my_digests);
-    tor_assert(their_digests);
-    my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
-    their_id = (uint8_t*)their_digests->d[DIGEST_SHA256];
-
-    client_id = server ? their_id : my_id;
-    server_id = server ? my_id : their_id;
-
-    /* Client ID digest: 32 octets. */
-    memcpy(auth->cid, client_id, 32);
-
-    /* Server ID digest: 32 octets. */
-    memcpy(auth->sid, server_id, 32);
-  }
-
-  if (is_ed) {
-    const ed25519_public_key_t *my_ed_id, *their_ed_id;
-    if (!conn->handshake_state->certs->ed_id_sign) {
-      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
-      goto err;
-    }
-    my_ed_id = get_master_identity_key();
-    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
-
-    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
-    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
-
-    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
-    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
-  }
-
-  {
-    crypto_digest_t *server_d, *client_d;
-    if (server) {
-      server_d = conn->handshake_state->digest_sent;
-      client_d = conn->handshake_state->digest_received;
-    } else {
-      client_d = conn->handshake_state->digest_sent;
-      server_d = conn->handshake_state->digest_received;
-    }
-
-    /* Server log digest : 32 octets */
-    crypto_digest_get_digest(server_d, (char*)auth->slog, 32);
-
-    /* Client log digest : 32 octets */
-    crypto_digest_get_digest(client_d, (char*)auth->clog, 32);
-  }
-
-  {
-    /* Digest of cert used on TLS link : 32 octets. */
-    tor_x509_cert_t *cert = NULL;
-    if (server) {
-      cert = tor_tls_get_own_cert(conn->tls);
-    } else {
-      cert = tor_tls_get_peer_cert(conn->tls);
-    }
-    if (!cert) {
-      log_warn(LD_OR, "Unable to find cert when making %s data.",
-               authtype_str);
-      goto err;
-    }
-
-    memcpy(auth->scert,
-           tor_x509_cert_get_cert_digests(cert)->d[DIGEST_SHA256], 32);
-
-    tor_x509_cert_free(cert);
-  }
-
-  /* HMAC of clientrandom and serverrandom using master key : 32 octets */
-  if (old_tlssecrets_algorithm) {
-    if (tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets) < 0) {
-      log_fn(LOG_PROTOCOL_WARN, LD_OR, "Somebody asked us for an older TLS "
-         "authentication method (AUTHTYPE_RSA_SHA256_TLSSECRET) "
-         "which we don't support.");
-    }
-  } else {
-    char label[128];
-    tor_snprintf(label, sizeof(label),
-                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
-    int r = tor_tls_export_key_material(conn->tls, auth->tlssecrets,
-                                        auth->cid, sizeof(auth->cid),
-                                        label);
-    if (r < 0) {
-      if (r != -2)
-        log_warn(LD_BUG, "TLS key export failed for unknown reason.");
-      // If r == -2, this was openssl bug 7712.
-      goto err;
-    }
-  }
-
-  /* 8 octets were reserved for the current time, but we're trying to get out
-   * of the habit of sending time around willynilly.  Fortunately, nothing
-   * checks it.  That's followed by 16 bytes of nonce. */
-  crypto_rand((char*)auth->rand, 24);
-
-  ssize_t maxlen = auth1_encoded_len(auth, ctx);
-  if (ed_signing_key && is_ed) {
-    maxlen += ED25519_SIG_LEN;
-  } else if (signing_key && !is_ed) {
-    maxlen += crypto_pk_keysize(signing_key);
-  }
-
-  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
-  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
-  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
-  const size_t outlen = maxlen;
-  ssize_t len;
-
-  result->command = CELL_AUTHENTICATE;
-  set_uint16(result->payload, htons(authtype));
-
-  if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
-    /* LCOV_EXCL_START */
-    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
-    goto err;
-    /* LCOV_EXCL_STOP */
-  }
-
-  if (server) {
-    auth1_t *tmp = NULL;
-    ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
-    if (!tmp) {
-      /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
-               "we just encoded");
-      goto err;
-      /* LCOV_EXCL_STOP */
-    }
-    result->payload_len = (tmp->end_of_signed - result->payload);
-
-    auth1_free(tmp);
-    if (len2 != len) {
-      /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
-      goto err;
-      /* LCOV_EXCL_STOP */
-    }
-    goto done;
-  }
-
-  if (ed_signing_key && is_ed) {
-    ed25519_signature_t sig;
-    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
-      /* LCOV_EXCL_START */
-      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
-      goto err;
-      /* LCOV_EXCL_STOP */
-    }
-    auth1_setlen_sig(auth, ED25519_SIG_LEN);
-    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
-
-  } else if (signing_key && !is_ed) {
-    auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
-
-    char d[32];
-    crypto_digest256(d, (char*)out, len, DIGEST_SHA256);
-    int siglen = crypto_pk_private_sign(signing_key,
-                                    (char*)auth1_getarray_sig(auth),
-                                    auth1_getlen_sig(auth),
-                                    d, 32);
-    if (siglen < 0) {
-      log_warn(LD_OR, "Unable to sign AUTH1 data.");
-      goto err;
-    }
-
-    auth1_setlen_sig(auth, siglen);
-  }
-
-  len = auth1_encode(out, outlen, auth, ctx);
-  if (len < 0) {
-    /* LCOV_EXCL_START */
-    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
-    goto err;
-    /* LCOV_EXCL_STOP */
-  }
-  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
-  result->payload_len = len + AUTH_CELL_HEADER_LEN;
-  set_uint16(result->payload+2, htons(len));
-
-  goto done;
-
- err:
-  var_cell_free(result);
-  result = NULL;
- done:
-  auth1_free(auth);
-  auth_ctx_free(ctx);
-  return result;
-}
-
-/** Send an AUTHENTICATE cell on the connection <b>conn</b>.  Return 0 on
- * success, -1 on failure */
-MOCK_IMPL(int,
-connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
-{
-  var_cell_t *cell;
-  crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
-  /* XXXX make sure we're actually supposed to send this! */
-
-  if (!pk) {
-    log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
-    return -1;
-  }
-  if (! authchallenge_type_is_supported(authtype)) {
-    log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
-             "authentication type %d", authtype);
-    return -1;
-  }
-
-  cell = connection_or_compute_authenticate_cell_body(conn,
-                                                 authtype,
-                                                 pk,
-                                                 get_current_auth_keypair(),
-                                                 0 /* not server */);
-  if (! cell) {
-    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unable to compute authenticate cell!");
-    return -1;
-  }
-  connection_or_write_var_cell_to_buf(cell, conn);
-  var_cell_free(cell);
-
-  return 0;
-}
diff --git a/src/core/or/connection_or.h b/src/core/or/connection_or.h
index 817bcdd317..e9ace56ab4 100644
--- a/src/core/or/connection_or.h
+++ b/src/core/or/connection_or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,40 +17,11 @@ struct ed25519_keypair_t;
 
 or_connection_t *TO_OR_CONN(connection_t *);
 
-#define OR_CONN_STATE_MIN_ 1
-/** State for a connection to an OR: waiting for connect() to finish. */
-#define OR_CONN_STATE_CONNECTING 1
-/** State for a connection to an OR: waiting for proxy handshake to complete */
-#define OR_CONN_STATE_PROXY_HANDSHAKING 2
-/** State for an OR connection client: SSL is handshaking, not done
- * yet. */
-#define OR_CONN_STATE_TLS_HANDSHAKING 3
-/** State for a connection to an OR: We're doing a second SSL handshake for
- * renegotiation purposes. (V2 handshake only.) */
-#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
-/** State for a connection at an OR: We're waiting for the client to
- * renegotiate (to indicate a v2 handshake) or send a versions cell (to
- * indicate a v3 handshake) */
-#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
-/** State for an OR connection: We're done with our SSL handshake, we've done
- * renegotiation, but we haven't yet negotiated link protocol versions and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
-/** State for an OR connection: We're done with our SSL handshake, but we
- * haven't yet negotiated link protocol versions, done a V3 handshake, and
- * sent a netinfo cell. */
-#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
-/** State for an OR connection: Ready to send/receive cells. */
-#define OR_CONN_STATE_OPEN 8
-#define OR_CONN_STATE_MAX_ 8
+#include "core/or/orconn_event.h"
 
 void connection_or_clear_identity(or_connection_t *conn);
 void connection_or_clear_identity_map(void);
 void clear_broken_connection_map(int disable);
-or_connection_t *connection_or_get_for_extend(const char *digest,
-                                              const tor_addr_t *target_addr,
-                                              const char **msg_out,
-                                              int *launch_out);
 
 void connection_or_block_renegotiation(or_connection_t *conn);
 int connection_or_reached_eof(or_connection_t *conn);
@@ -81,6 +52,9 @@ MOCK_DECL(void,connection_or_close_for_error,
 
 void connection_or_report_broken_states(int severity, int domain);
 
+void connection_or_event_status(or_connection_t *conn,
+                                or_conn_status_event_t tp, int reason);
+
 MOCK_DECL(int,connection_tls_start_handshake,(or_connection_t *conn,
                                               int receiving));
 int connection_tls_continue_handshake(or_connection_t *conn);
@@ -119,19 +93,6 @@ MOCK_DECL(void,connection_or_write_var_cell_to_buf,(const var_cell_t *cell,
                                                    or_connection_t *conn));
 int connection_or_send_versions(or_connection_t *conn, int v3_plus);
 MOCK_DECL(int,connection_or_send_netinfo,(or_connection_t *conn));
-int connection_or_send_certs_cell(or_connection_t *conn);
-int connection_or_send_auth_challenge_cell(or_connection_t *conn);
-int authchallenge_type_is_supported(uint16_t challenge_type);
-int authchallenge_type_is_better(uint16_t challenge_type_a,
-                                 uint16_t challenge_type_b);
-var_cell_t *connection_or_compute_authenticate_cell_body(
-                              or_connection_t *conn,
-                              const int authtype,
-                              crypto_pk_t *signing_key,
-                              const struct ed25519_keypair_t *ed_signing_key,
-                              int server);
-MOCK_DECL(int,connection_or_send_authenticate_cell,
-          (or_connection_t *conn, int type));
 
 int is_or_protocol_version_known(uint16_t version);
 
@@ -156,10 +117,16 @@ void connection_or_group_set_badness_(smartlist_t *group, int force);
 #ifdef CONNECTION_OR_PRIVATE
 STATIC int should_connect_to_relay(const or_connection_t *or_conn);
 STATIC void note_or_connect_failed(const or_connection_t *or_conn);
-#endif
+
+/*
+ * Call this when changing connection state, so notifications to the owning
+ * channel can be handled.
+ */
+MOCK_DECL(STATIC void,connection_or_change_state,
+          (or_connection_t *conn, uint8_t state));
+#endif /* defined(CONNECTION_OR_PRIVATE) */
 
 #ifdef TOR_UNIT_TESTS
-extern int certs_cell_ed25519_disabled_for_testing;
 extern int testing__connection_or_pretend_TLSSECRET_is_supported;
 #endif
 
diff --git a/src/core/or/connection_st.h b/src/core/or/connection_st.h
index c197a81340..685c9f89f4 100644
--- a/src/core/or/connection_st.h
+++ b/src/core/or/connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file connection_st.h
+ * @brief Base connection structure.
+ **/
+
 #ifndef CONNECTION_ST_H
 #define CONNECTION_ST_H
 
@@ -149,4 +154,4 @@ struct connection_t {
  * directory connection. */
 #define DIR_CONN_IS_SERVER(conn) ((conn)->purpose == DIR_PURPOSE_SERVER)
 
-#endif
+#endif /* !defined(CONNECTION_ST_H) */
diff --git a/src/core/or/core_or.md b/src/core/or/core_or.md
new file mode 100644
index 0000000000..0b4d430a00
--- /dev/null
+++ b/src/core/or/core_or.md
@@ -0,0 +1,62 @@
+@dir core/or
+@brief core/or: **Onion routing happens here!**
+
+This is the central part of Tor that handles the core tasks of onion routing:
+building circuit, handling circuits, attaching circuit to streams, moving
+data around, and so forth.
+
+Some aspects of this module should probably be refactored into others.
+
+Notable files here include:
+
+`channel.c`
+: Generic channel implementation. Channels handle sending and receiving cells
+among tor nodes.
+
+`channeltls.c`
+: Channel implementation for TLS-based OR connections. Uses `connection_or.c`.
+
+`circuitbuild.c`
+: Code for constructing circuits and choosing their paths.  (*Note*:
+this module could plausibly be split into handling the client side,
+the server side, and the path generation aspects of circuit building.)
+
+`circuitlist.c`
+: Code for maintaining and navigating the global list of circuits.
+
+`circuitmux.c`
+: Generic circuitmux implementation. A circuitmux handles deciding, for a
+particular channel, which circuit should write next.
+
+`circuitmux_ewma.c`
+: A circuitmux implementation based on the EWMA (exponentially
+weighted moving average) algorithm.
+
+`circuituse.c`
+: Code to actually send and receive data on circuits.
+
+`command.c`
+: Handles incoming cells on channels.
+
+`connection.c`
+: Generic and common connection tools, and implementation for the simpler
+connection types.
+
+`connection_edge.c`
+: Implementation for entry and exit connections.
+
+`connection_or.c`
+: Implementation for OR connections (the ones that send cells over TLS).
+
+`onion.c`
+: Generic code for generating and responding to CREATE and CREATED
+cells, and performing the appropriate onion handshakes. Also contains
+code to manage the server-side onion queue.
+
+`relay.c`
+: Handles particular types of relay cells, and provides code to receive,
+encrypt, route, and interpret relay cells.
+
+`scheduler.c`
+: Decides which channel/circuit pair is ready to receive the next cell.
+
diff --git a/src/core/or/cpath_build_state_st.h b/src/core/or/cpath_build_state_st.h
index dbe596d851..ee9a0d972c 100644
--- a/src/core/or/cpath_build_state_st.h
+++ b/src/core/or/cpath_build_state_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cpath_build_state_st.h
+ * @brief Circuit-build-stse structure
+ **/
+
 #ifndef CIRCUIT_BUILD_STATE_ST_ST_H
 #define CIRCUIT_BUILD_STATE_ST_ST_H
 
@@ -34,5 +39,4 @@ struct cpath_build_state_t {
   time_t expiry_time;
 };
 
-#endif
-
+#endif /* !defined(CIRCUIT_BUILD_STATE_ST_ST_H) */
diff --git a/src/core/or/crypt_path.c b/src/core/or/crypt_path.c
new file mode 100644
index 0000000000..8f41540848
--- /dev/null
+++ b/src/core/or/crypt_path.c
@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypt_path.c
+ *
+ * \brief Functions dealing with layered circuit encryption. This file aims to
+ *   provide an API around the crypt_path_t structure which holds crypto
+ *   information about a specific hop of a circuit.
+ *
+ * TODO: We should eventually move all functions dealing and manipulating
+ *   crypt_path_t to this file, so that eventually we encapsulate more and more
+ *   of crypt_path_t. Here are some more functions that can be moved here with
+ *   some more effort:
+ *
+ *   - circuit_list_path_impl()
+ *   - Functions dealing with cpaths in HSv2 create_rend_cpath() and
+ *     create_rend_cpath_legacy()
+ *   - The cpath related parts of rend_service_receive_introduction() and
+ *     rend_client_send_introduction().
+ **/
+
+#define CRYPT_PATH_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/crypt_path.h"
+
+#include "core/crypto/relay_crypto.h"
+#include "core/crypto/onion_crypto.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+
+#include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_util.h"
+
+#include "core/or/crypt_path_st.h"
+#include "core/or/cell_st.h"
+
+/** Add <b>new_hop</b> to the end of the doubly-linked-list <b>head_ptr</b>.
+ * This function is used to extend cpath by another hop.
+ */
+void
+cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop)
+{
+  if (*head_ptr) {
+    new_hop->next = (*head_ptr);
+    new_hop->prev = (*head_ptr)->prev;
+    (*head_ptr)->prev->next = new_hop;
+    (*head_ptr)->prev = new_hop;
+  } else {
+    *head_ptr = new_hop;
+    new_hop->prev = new_hop->next = new_hop;
+  }
+}
+
+/** Create a new hop, annotate it with information about its
+ * corresponding router <b>choice</b>, and append it to the
+ * end of the cpath <b>head_ptr</b>. */
+int
+cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice)
+{
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+
+  /* link hop into the cpath, at the end. */
+  cpath_extend_linked_list(head_ptr, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_CLOSED;
+
+  hop->extend_info = extend_info_dup(choice);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  return 0;
+}
+
+/** Verify that cpath <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_ok(const crypt_path_t *cp)
+{
+  const crypt_path_t *start = cp;
+
+  do {
+    cpath_assert_layer_ok(cp);
+    /* layers must be in sequence of: "open* awaiting? closed*" */
+    if (cp != start) {
+      if (cp->state == CPATH_STATE_AWAITING_KEYS) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      } else if (cp->state == CPATH_STATE_OPEN) {
+        tor_assert(cp->prev->state == CPATH_STATE_OPEN);
+      }
+    }
+    cp = cp->next;
+    tor_assert(cp);
+  } while (cp != start);
+}
+
+/** Verify that cpath layer <b>cp</b> has all of its invariants
+ * correct. Trigger an assert if anything is invalid.
+ */
+void
+cpath_assert_layer_ok(const crypt_path_t *cp)
+{
+//  tor_assert(cp->addr); /* these are zero for rendezvous extra-hops */
+//  tor_assert(cp->port);
+  tor_assert(cp);
+  tor_assert(cp->magic == CRYPT_PATH_MAGIC);
+  switch (cp->state)
+    {
+    case CPATH_STATE_OPEN:
+      relay_crypto_assert_ok(&cp->pvt_crypto);
+      FALLTHROUGH;
+    case CPATH_STATE_CLOSED:
+      /*XXXX Assert that there's no handshake_state either. */
+      tor_assert(!cp->rend_dh_handshake_state);
+      break;
+    case CPATH_STATE_AWAITING_KEYS:
+      /* tor_assert(cp->dh_handshake_state); */
+      break;
+    default:
+      log_fn(LOG_ERR, LD_BUG, "Unexpected state %d", cp->state);
+      tor_assert(0);
+    }
+  tor_assert(cp->package_window >= 0);
+  tor_assert(cp->deliver_window >= 0);
+}
+
+/** Initialize cpath-\>{f|b}_{crypto|digest} from the key material in key_data.
+ *
+ * If <b>is_hs_v3</b> is set, this cpath will be used for next gen hidden
+ * service circuits and <b>key_data</b> must be at least
+ * HS_NTOR_KEY_EXPANSION_KDF_OUT_LEN bytes in length.
+ *
+ * If <b>is_hs_v3</b> is not set, key_data must contain CPATH_KEY_MATERIAL_LEN
+ * bytes, which are used as follows:
+ *   - 20 to initialize f_digest
+ *   - 20 to initialize b_digest
+ *   - 16 to key f_crypto
+ *   - 16 to key b_crypto
+ *
+ * (If 'reverse' is true, then f_XX and b_XX are swapped.)
+ *
+ * Return 0 if init was successful, else -1 if it failed.
+ */
+int
+cpath_init_circuit_crypto(crypt_path_t *cpath,
+                          const char *key_data, size_t key_data_len,
+                          int reverse, int is_hs_v3)
+{
+
+  tor_assert(cpath);
+  return relay_crypto_init(&cpath->pvt_crypto, key_data, key_data_len,
+                           reverse, is_hs_v3);
+}
+
+/** Deallocate space associated with the cpath node <b>victim</b>. */
+void
+cpath_free(crypt_path_t *victim)
+{
+  if (!victim)
+    return;
+
+  relay_crypto_clear(&victim->pvt_crypto);
+  onion_handshake_state_release(&victim->handshake_state);
+  crypto_dh_free(victim->rend_dh_handshake_state);
+  extend_info_free(victim->extend_info);
+
+  memwipe(victim, 0xBB, sizeof(crypt_path_t)); /* poison memory */
+  tor_free(victim);
+}
+
+/********************** cpath crypto API *******************************/
+
+/** Encrypt or decrypt <b>payload</b> using the crypto of <b>cpath</b>. Actual
+ *  operation decided by <b>is_decrypt</b>.  */
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt)
+{
+  if (is_decrypt) {
+    relay_crypt_one_payload(cpath->pvt_crypto.b_crypto, payload);
+  } else {
+    relay_crypt_one_payload(cpath->pvt_crypto.f_crypto, payload);
+  }
+}
+
+/** Getter for the incoming digest of <b>cpath</b>. */
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath)
+{
+  return cpath->pvt_crypto.b_digest;
+}
+
+/** Set the right integrity digest on the outgoing <b>cell</b> based on the
+ *  cell payload and update the forward digest of <b>cpath</b>. */
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell)
+{
+  relay_set_digest(cpath->pvt_crypto.f_digest, cell);
+}
+
+/************ cpath sendme API ***************************/
+
+/** Return the sendme_digest of this <b>cpath</b>. */
+uint8_t *
+cpath_get_sendme_digest(crypt_path_t *cpath)
+{
+  return relay_crypto_get_sendme_digest(&cpath->pvt_crypto);
+}
+
+/** Record the cell digest, indicated by is_foward_digest or not, as the
+ * SENDME cell digest. */
+void
+cpath_sendme_record_cell_digest(crypt_path_t *cpath, bool is_foward_digest)
+{
+  tor_assert(cpath);
+  relay_crypto_record_sendme_digest(&cpath->pvt_crypto, is_foward_digest);
+}
+
+/************ other cpath functions ***************************/
+
+/** Return the first non-open hop in cpath, or return NULL if all
+ * hops are open. */
+crypt_path_t *
+cpath_get_next_non_open_hop(crypt_path_t *cpath)
+{
+  crypt_path_t *hop = cpath;
+  do {
+    if (hop->state != CPATH_STATE_OPEN)
+      return hop;
+    hop = hop->next;
+  } while (hop != cpath);
+  return NULL;
+}
+
+#ifdef TOR_UNIT_TESTS
+
+/** Unittest helper function: Count number of hops in cpath linked list. */
+unsigned int
+cpath_get_n_hops(crypt_path_t **head_ptr)
+{
+  unsigned int n_hops = 0;
+  crypt_path_t *tmp;
+
+  if (!*head_ptr) {
+    return 0;
+  }
+
+  tmp = *head_ptr;
+  do {
+    n_hops++;
+    tmp = tmp->next;
+  } while (tmp != *head_ptr);
+
+  return n_hops;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
diff --git a/src/core/or/crypt_path.h b/src/core/or/crypt_path.h
new file mode 100644
index 0000000000..7a95fec2b4
--- /dev/null
+++ b/src/core/or/crypt_path.h
@@ -0,0 +1,46 @@
+/**
+ * \file crypt_path.h
+ * \brief Header file for crypt_path.c.
+ **/
+
+#ifndef CRYPT_PATH_H
+#define CRYPT_PATH_H
+
+void cpath_assert_layer_ok(const crypt_path_t *cp);
+
+void cpath_assert_ok(const crypt_path_t *cp);
+
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+
+int cpath_init_circuit_crypto(crypt_path_t *cpath,
+                              const char *key_data, size_t key_data_len,
+                              int reverse, int is_hs_v3);
+
+void
+cpath_free(crypt_path_t *victim);
+
+void cpath_extend_linked_list(crypt_path_t **head_ptr, crypt_path_t *new_hop);
+
+void
+cpath_crypt_cell(const crypt_path_t *cpath, uint8_t *payload, bool is_decrypt);
+
+struct crypto_digest_t *
+cpath_get_incoming_digest(const crypt_path_t *cpath);
+
+void cpath_sendme_record_cell_digest(crypt_path_t *cpath,
+                                     bool is_foward_digest);
+
+void
+cpath_set_cell_forward_digest(crypt_path_t *cpath, cell_t *cell);
+
+crypt_path_t *cpath_get_next_non_open_hop(crypt_path_t *cpath);
+
+void cpath_sendme_circuit_record_inbound_cell(crypt_path_t *cpath);
+
+uint8_t *cpath_get_sendme_digest(crypt_path_t *cpath);
+
+#if defined(TOR_UNIT_TESTS)
+unsigned int cpath_get_n_hops(crypt_path_t **head_ptr);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(CRYPT_PATH_H) */
diff --git a/src/core/or/crypt_path_reference_st.h b/src/core/or/crypt_path_reference_st.h
index 3d79f26c1c..71f9cb8c36 100644
--- a/src/core/or/crypt_path_reference_st.h
+++ b/src/core/or/crypt_path_reference_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file crypt_path_reference_st.h
+ * @brief reference-counting structure for crypt_path_t
+ **/
+
 #ifndef CRYPT_PATH_REFERENCE_ST_H
 #define CRYPT_PATH_REFERENCE_ST_H
 
@@ -19,5 +24,4 @@ struct crypt_path_reference_t {
   crypt_path_t *cpath;
 };
 
-#endif
-
+#endif /* !defined(CRYPT_PATH_REFERENCE_ST_H) */
diff --git a/src/core/or/crypt_path_st.h b/src/core/or/crypt_path_st.h
index 429480f8ab..2b69728a6d 100644
--- a/src/core/or/crypt_path_st.h
+++ b/src/core/or/crypt_path_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file crypt_path_st.h
+ * @brief Path structures for origin circuits.
+ **/
+
 #ifndef CRYPT_PATH_ST_H
 #define CRYPT_PATH_ST_H
 
@@ -24,15 +29,24 @@ struct onion_handshake_state_t {
   } u;
 };
 
+/** Macro to encapsulate private members of a struct.
+ *
+ *  Renames 'x' to 'x_crypt_path_private_field'.
+ */
+#define CRYPT_PATH_PRIV_FIELD(x) x ## _crypt_path_private_field
+
+#ifdef CRYPT_PATH_PRIVATE
+
+/* Helper macro to access private members of a struct. */
+#define pvt_crypto CRYPT_PATH_PRIV_FIELD(crypto)
+
+#endif /* defined(CRYPT_PATH_PRIVATE) */
+
 /** Holds accounting information for a single step in the layered encryption
  * performed by a circuit.  Used only at the client edge of a circuit. */
 struct crypt_path_t {
   uint32_t magic;
 
-  /** Cryptographic state used for encrypting and authenticating relay
-   * cells to and from this hop. */
-  relay_crypto_t crypto;
-
   /** Current state of the handshake as performed with the OR at this
    * step. */
   onion_handshake_state_t handshake_state;
@@ -65,6 +79,12 @@ struct crypt_path_t {
                        * at this step? */
   int deliver_window; /**< How many cells are we willing to deliver originating
                        * at this step? */
+
+  /*********************** Private members ****************************/
+
+  /** Private member: Cryptographic state used for encrypting and
+   * authenticating relay cells to and from this hop. */
+  relay_crypto_t CRYPT_PATH_PRIV_FIELD(crypto);
 };
 
-#endif
+#endif /* !defined(CRYPT_PATH_ST_H) */
diff --git a/src/core/or/dataflow.md b/src/core/or/dataflow.md
new file mode 100644
index 0000000000..1098701780
--- /dev/null
+++ b/src/core/or/dataflow.md
@@ -0,0 +1,236 @@
+@tableofcontents
+
+@page dataflow Data flow in the Tor process
+
+We read bytes from the network, we write bytes to the network.  For the
+most part, the bytes we write correspond roughly to bytes we have read,
+with bits of cryptography added in.
+
+The rest is a matter of details.
+
+### Connections and buffers: reading, writing, and interpreting.
+
+At a low level, Tor's networking code is based on "connections".  Each
+connection represents an object that can send or receive network-like
+events.  For the most part, each connection has a single underlying TCP
+stream (I'll discuss counterexamples below).
+
+A connection that behaves like a TCP stream has an input buffer and an
+output buffer.  Incoming data is
+written into the input buffer ("inbuf"); data to be written to the
+network is queued on an output buffer ("outbuf").
+
+Buffers are implemented in buffers.c.  Each of these buffers is
+implemented as a linked queue of memory extents, in the style of classic
+BSD mbufs, or Linux skbufs.
+
+A connection's reading and writing can be enabled or disabled.  Under
+the hood, this functionality is implemented using libevent events: one
+for reading, one for writing.  These events are turned on/off in
+main.c, in the functions connection_{start,stop}_{reading,writing}.
+
+When a read or write event is turned on, the main libevent loop polls
+the kernel, asking which sockets are ready to read or write.  (This
+polling happens in the event_base_loop() call in run_main_loop_once()
+in main.c.)  When libevent finds a socket that's ready to read or write,
+it invokes conn_{read,write}_callback(), also in main.c
+
+These callback functions delegate to connection_handle_read() and
+connection_handle_write() in connection.c, which read or write on the
+network as appropriate, possibly delegating to openssl.
+
+After data is read or written, or other event occurs, these
+connection_handle_read_write() functions call logic functions whose job is
+to respond to the information.  Some examples included:
+
+   * connection_flushed_some() -- called after a connection writes any
+     amount of data from its outbuf.
+   * connection_finished_flushing() -- called when a connection has
+     emptied its outbuf.
+   * connection_finished_connecting() -- called when an in-process connection
+     finishes making a remote connection.
+   * connection_reached_eof() -- called after receiving a FIN from the
+     remote server.
+   * connection_process_inbuf() -- called when more data arrives on
+     the inbuf.
+
+These functions then call into specific implementations depending on
+the type of the connection.  For example, if the connection is an
+edge_connection_t, connection_reached_eof() will call
+connection_edge_reached_eof().
+
+> **Note:** "Also there are bufferevents!"  We have vestigial
+> code for an alternative low-level networking
+> implementation, based on Libevent's evbuffer and bufferevent
+> code.  These two object types take on (most of) the roles of
+> buffers and connections respectively. It isn't working in today's
+> Tor, due to code rot and possible lingering libevent bugs.  More
+> work is needed; it would be good to get this working efficiently
+> again, to have IOCP support on Windows.
+
+
+#### Controlling connections ####
+
+A connection can have reading or writing enabled or disabled for a
+wide variety of reasons, including:
+
+   * Writing is disabled when there is no more data to write
+   * For some connection types, reading is disabled when the inbuf is
+     too full.
+   * Reading/writing is temporarily disabled on connections that have
+     recently read/written enough data up to their bandwidth
+   * Reading is disabled on connections when reading more data from them
+     would require that data to be buffered somewhere else that is
+     already full.
+
+Currently, these conditions are checked in a diffuse set of
+increasingly complex conditional expressions.  In the future, it could
+be helpful to transition to a unified model for handling temporary
+read/write suspensions.
+
+#### Kinds of connections ####
+
+Today Tor has the following connection and pseudoconnection types.
+For the most part, each type of channel has an associated C module
+that implements its underlying logic.
+
+**Edge connections** receive data from and deliver data to points
+outside the onion routing network.  See `connection_edge.c`. They fall into two types:
+
+**Entry connections** are a type of edge connection. They receive data
+from the user running a Tor client, and deliver data to that user.
+They are used to implement SOCKSPort, TransPort, NATDPort, and so on.
+Sometimes they are called "AP" connections for historical reasons (it
+used to stand for "Application Proxy").
+
+**Exit connections** are a type of edge connection. They exist at an
+exit node, and transmit traffic to and from the network.
+
+(Entry connections and exit connections are also used as placeholders
+when performing a remote DNS request; they are not decoupled from the
+notion of "stream" in the Tor protocol. This is implemented partially
+in `connection_edge.c`, and partially in `dnsserv.c` and `dns.c`.)
+
+**OR connections** send and receive Tor cells over TLS, using some
+version of the Tor link protocol.  Their implementation is spread
+across `connection_or.c`, with a bit of logic in `command.c`,
+`relay.c`, and `channeltls.c`.
+
+**Extended OR connections** are a type of OR connection for use on
+bridges using pluggable transports, so that the PT can tell the bridge
+some information about the incoming connection before passing on its
+data.  They are implemented in `ext_orport.c`.
+
+**Directory connections** are server-side or client-side connections
+that implement Tor's HTTP-based directory protocol.  These are
+instantiated using a socket when Tor is making an unencrypted HTTP
+connection.  When Tor is tunneling a directory request over a Tor
+circuit, directory connections are implemented using a linked
+connection pair (see below).  Directory connections are implemented in
+`directory.c`; some of the server-side logic is implemented in
+`dirserver.c`.
+
+**Controller connections** are local connections to a controller
+process implementing the controller protocol from
+control-spec.txt. These are in `control.c`.
+
+**Listener connections** are not stream oriented!  Rather, they wrap a
+listening socket in order to detect new incoming connections.  They
+bypass most of stream logic.  They don't have associated buffers.
+They are implemented in `connection.c`.
+
+![structure hierarchy for connection types](./diagrams/02/02-connection-types.png "structure hierarchy for connection types")
+
+>**Note**: "History Time!" You might occasionally find reference to a couple types of connections
+> which no longer exist in modern Tor.  A *CPUWorker connection*
+>connected the main Tor process to a thread or process used for
+>computation.  (Nowadays we use in-process communication.)  Even more
+>anciently, a *DNSWorker connection* connected the main tor process to
+>a separate thread or process used for running `gethostbyname()` or
+>`getaddrinfo()`.  (Nowadays we use Libevent's evdns facility to
+>perform DNS requests asynchronously.)
+
+#### Linked connections ####
+
+Sometimes two channels are joined together, such that data which the
+Tor process sends on one should immediately be received by the same
+Tor process on the other.  (For example, when Tor makes a tunneled
+directory connection, this is implemented on the client side as a
+directory connection whose output goes, not to the network, but to a
+local entry connection. And when a directory receives a tunnelled
+directory connection, this is implemented as an exit connection whose
+output goes, not to the network, but to a local directory connection.)
+
+The earliest versions of Tor to support linked connections used
+socketpairs for the purpose.  But using socketpairs forced us to copy
+data through kernelspace, and wasted limited file descriptors.  So
+instead, a pair of connections can be linked in-process.  Each linked
+connection has a pointer to the other, such that data written on one
+is immediately readable on the other, and vice versa.
+
+### From connections to channels ###
+
+There's an abstraction layer above OR connections (the ones that
+handle cells) and below cells called **Channels**.  A channel's
+purpose is to transmit authenticated cells from one Tor instance
+(relay or client) to another.
+
+Currently, only one implementation exists: Channel_tls, which sends
+and receiveds cells over a TLS-based OR connection.
+
+Cells are sent on a channel using
+`channel_write_{,packed_,var_}cell()`. Incoming cells arrive on a
+channel from its backend using `channel_queue*_cell()`, and are
+immediately processed using `channel_process_cells()`.
+
+Some cell types are handled below the channel layer, such as those
+that affect handshaking only.  And some others are passed up to the
+generic cross-channel code in `command.c`: cells like `DESTROY` and
+`CREATED` are all trivial to handle.  But relay cells
+require special handling...
+
+### From channels through circuits ###
+
+When a relay cell arrives on an existing circuit, it is handled in
+`circuit_receive_relay_cell()` -- one of the innermost functions in
+Tor.  This function encrypts or decrypts the relay cell as
+appropriate, and decides whether the cell is intended for the current
+hop of the circuit.
+
+If the cell *is* intended for the current hop, we pass it to
+`connection_edge_process_relay_cell()` in `relay.c`, which acts on it
+based on its relay command, and (possibly) queues its data on an
+`edge_connection_t`.
+
+If the cell *is not* intended for the current hop, we queue it for the
+next channel in sequence with `append cell_to_circuit_queue()`.  This
+places the cell on a per-circuit queue for cells headed out on that
+particular channel.
+
+### Sending cells on circuits: the complicated bit.
+
+Relay cells are queued onto circuits from one of two (main) sources:
+reading data from edge connections, and receiving a cell to be relayed
+on a circuit.  Both of these sources place their cells on cell queue:
+each circuit has one cell queue for each direction that it travels.
+
+A naive implementation would skip using cell queues, and instead write
+each outgoing relay cell.  (Tor did this in its earlier versions.)
+But such an approach tends to give poor performance, because it allows
+high-volume circuits to clog channels, and it forces the Tor server to
+send data queued on a circuit even after that circuit has been closed.
+
+So by using queues on each circuit, we can add cells to each channel
+on a just-in-time basis, choosing the cell at each moment based on
+a performance-aware algorithm.
+
+This logic is implemented in two main modules: `scheduler.c` and
+`circuitmux*.c`.  The scheduler code is responsible for determining
+globally, across all channels that could write cells, which one should
+next receive queued cells.  The circuitmux code determines, for all
+of the circuits with queued cells for a channel, which one should
+queue the next cell.
+
+(This logic applies to outgoing relay cells only; incoming relay cells
+are processed as they arrive.)
+
diff --git a/src/core/or/destroy_cell_queue_st.h b/src/core/or/destroy_cell_queue_st.h
index 56630670ba..aa28289be5 100644
--- a/src/core/or/destroy_cell_queue_st.h
+++ b/src/core/or/destroy_cell_queue_st.h
@@ -1,12 +1,19 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file destroy_cell_queue_st.h
+ * @brief Destroy-cell queue structures
+ **/
+
 #ifndef DESTROY_CELL_QUEUE_ST_H
 #define DESTROY_CELL_QUEUE_ST_H
 
+#include "core/or/cell_queue_st.h"
+
 /** A single queued destroy cell. */
 struct destroy_cell_t {
   TOR_SIMPLEQ_ENTRY(destroy_cell_t) next;
@@ -19,9 +26,8 @@ struct destroy_cell_t {
 /** A queue of destroy cells on a channel. */
 struct destroy_cell_queue_t {
   /** Linked list of packed_cell_t */
-  TOR_SIMPLEQ_HEAD(dcell_simpleq, destroy_cell_t) head;
+  TOR_SIMPLEQ_HEAD(dcell_simpleq_t, destroy_cell_t) head;
   int n; /**< The number of cells in the queue. */
 };
 
-#endif
-
+#endif /* !defined(DESTROY_CELL_QUEUE_ST_H) */
diff --git a/src/core/or/dos.c b/src/core/or/dos.c
index d06eaa6d05..5f99280030 100644
--- a/src/core/or/dos.c
+++ b/src/core/or/dos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
@@ -15,6 +15,7 @@
 #include "core/or/channel.h"
 #include "core/or/connection_or.h"
 #include "core/or/relay.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/relay/routermode.h"
@@ -629,6 +630,7 @@ dos_log_heartbeat(void)
   char *cc_msg = NULL;
   char *single_hop_client_msg = NULL;
   char *circ_stats_msg = NULL;
+  char *hs_dos_intro2_msg = NULL;
 
   /* Stats number coming from relay.c append_cell_to_circuit_queue(). */
   tor_asprintf(&circ_stats_msg,
@@ -654,17 +656,24 @@ dos_log_heartbeat(void)
                  num_single_hop_client_refused);
   }
 
+  /* HS DoS stats. */
+  tor_asprintf(&hs_dos_intro2_msg,
+               " %" PRIu64 " INTRODUCE2 rejected.",
+               hs_dos_get_intro2_rejected_count());
+
   log_notice(LD_HEARTBEAT,
-             "DoS mitigation since startup:%s%s%s%s",
+             "DoS mitigation since startup:%s%s%s%s%s",
              circ_stats_msg,
              (cc_msg != NULL) ? cc_msg : " [cc not enabled]",
              (conn_msg != NULL) ? conn_msg : " [conn not enabled]",
-             (single_hop_client_msg != NULL) ? single_hop_client_msg : "");
+             (single_hop_client_msg != NULL) ? single_hop_client_msg : "",
+             (hs_dos_intro2_msg != NULL) ? hs_dos_intro2_msg : "");
 
   tor_free(conn_msg);
   tor_free(cc_msg);
   tor_free(single_hop_client_msg);
   tor_free(circ_stats_msg);
+  tor_free(hs_dos_intro2_msg);
   return;
 }
 
diff --git a/src/core/or/dos.h b/src/core/or/dos.h
index 058b7afce6..b3eca058b8 100644
--- a/src/core/or/dos.h
+++ b/src/core/or/dos.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
@@ -135,7 +135,7 @@ MOCK_DECL(STATIC unsigned int, get_param_cc_enabled,
 MOCK_DECL(STATIC unsigned int, get_param_conn_enabled,
           (const networkstatus_t *ns));
 
-#endif /* TOR_DOS_PRIVATE */
+#endif /* defined(DOS_PRIVATE) */
 
-#endif /* TOR_DOS_H */
+#endif /* !defined(TOR_DOS_H) */
 
diff --git a/src/core/or/edge_connection_st.h b/src/core/or/edge_connection_st.h
index 1665b8589f..9b2f031b9d 100644
--- a/src/core/or/edge_connection_st.h
+++ b/src/core/or/edge_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file edge_connection_st.h
+ * @brief Edge-connection structure.
+ **/
+
 #ifndef EDGE_CONNECTION_ST_H
 #define EDGE_CONNECTION_ST_H
 
@@ -73,5 +78,4 @@ struct edge_connection_t {
   uint64_t dirreq_id;
 };
 
-#endif
-
+#endif /* !defined(EDGE_CONNECTION_ST_H) */
diff --git a/src/core/or/entry_connection_st.h b/src/core/or/entry_connection_st.h
index 45621fadbf..495ffd85dd 100644
--- a/src/core/or/entry_connection_st.h
+++ b/src/core/or/entry_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file entry_connection_st.h
+ * @brief Entry connection structure.
+ **/
+
 #ifndef ENTRY_CONNECTION_ST_H
 #define ENTRY_CONNECTION_ST_H
 
@@ -96,5 +101,4 @@ struct entry_connection_t {
 /** Cast a entry_connection_t subtype pointer to a edge_connection_t **/
 #define ENTRY_TO_EDGE_CONN(c) (&(((c))->edge_))
 
-#endif
-
+#endif /* !defined(ENTRY_CONNECTION_ST_H) */
diff --git a/src/core/or/entry_port_cfg_st.h b/src/core/or/entry_port_cfg_st.h
index 87dfb331e5..ef1095086d 100644
--- a/src/core/or/entry_port_cfg_st.h
+++ b/src/core/or/entry_port_cfg_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file entry_port_cfg_st.h
+ * @brief Configuration structure for client ports.
+ **/
+
 #ifndef ENTRY_PORT_CFG_ST_H
 #define ENTRY_PORT_CFG_ST_H
 
@@ -48,7 +53,9 @@ struct entry_port_cfg_t {
    * do we prefer IPv6? */
   unsigned int prefer_ipv6_virtaddr : 1;
 
-};
+  /** For socks listeners: can we send back the extended SOCKS5 error code? */
+  unsigned int extended_socks5_codes : 1;
 
-#endif
+};
 
+#endif /* !defined(ENTRY_PORT_CFG_ST_H) */
diff --git a/src/core/or/extend_info_st.h b/src/core/or/extend_info_st.h
index bc7a77b1b2..a66ce24cfa 100644
--- a/src/core/or/extend_info_st.h
+++ b/src/core/or/extend_info_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file extend_info_st.h
+ * @brief Extend-info structure.
+ **/
+
 #ifndef EXTEND_INFO_ST_H
 #define EXTEND_INFO_ST_H
 
@@ -27,4 +32,4 @@ struct extend_info_t {
   curve25519_public_key_t curve25519_onion_key;
 };
 
-#endif
+#endif /* !defined(EXTEND_INFO_ST_H) */
diff --git a/src/core/or/half_edge_st.h b/src/core/or/half_edge_st.h
index d4617be108..c6b6e518fc 100644
--- a/src/core/or/half_edge_st.h
+++ b/src/core/or/half_edge_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file half_edge_st.h
+ * @brief Half-open connection structure.
+ **/
+
 #ifndef HALF_EDGE_ST_H
 #define HALF_EDGE_ST_H
 
@@ -30,5 +35,4 @@ typedef struct half_edge_t {
   int connected_pending : 1;
 } half_edge_t;
 
-#endif
-
+#endif /* !defined(HALF_EDGE_ST_H) */
diff --git a/src/core/or/include.am b/src/core/or/include.am
new file mode 100644
index 0000000000..3626e76bed
--- /dev/null
+++ b/src/core/or/include.am
@@ -0,0 +1,96 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/or/address_set.c		\
+	src/core/or/channel.c			\
+	src/core/or/channelpadding.c		\
+	src/core/or/channeltls.c		\
+	src/core/or/circuitbuild.c		\
+	src/core/or/circuitlist.c		\
+	src/core/or/circuitmux.c		\
+	src/core/or/circuitmux_ewma.c		\
+	src/core/or/circuitpadding.c		\
+	src/core/or/circuitpadding_machines.c	\
+	src/core/or/circuitstats.c		\
+	src/core/or/circuituse.c		\
+	src/core/or/crypt_path.c		\
+	src/core/or/command.c			\
+	src/core/or/connection_edge.c		\
+	src/core/or/connection_or.c		\
+	src/core/or/dos.c			\
+	src/core/or/onion.c			\
+	src/core/or/ocirc_event.c		\
+	src/core/or/or_periodic.c		\
+	src/core/or/or_sys.c			\
+	src/core/or/orconn_event.c		\
+	src/core/or/policies.c			\
+	src/core/or/protover.c			\
+	src/core/or/protover_rust.c		\
+	src/core/or/reasons.c			\
+	src/core/or/relay.c			\
+	src/core/or/scheduler.c			\
+	src/core/or/scheduler_kist.c		\
+	src/core/or/scheduler_vanilla.c		\
+	src/core/or/sendme.c			\
+	src/core/or/status.c			\
+	src/core/or/versions.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/or/addr_policy_st.h			\
+	src/core/or/address_set.h			\
+	src/core/or/cell_queue_st.h			\
+	src/core/or/cell_st.h				\
+	src/core/or/channel.h				\
+	src/core/or/channelpadding.h			\
+	src/core/or/channeltls.h			\
+	src/core/or/circuit_st.h			\
+	src/core/or/circuitbuild.h			\
+	src/core/or/circuitlist.h			\
+	src/core/or/circuitmux.h			\
+	src/core/or/circuitmux_ewma.h			\
+	src/core/or/circuitstats.h			\
+	src/core/or/circuitpadding.h			\
+	src/core/or/circuitpadding_machines.h		\
+	src/core/or/circuituse.h			\
+	src/core/or/command.h				\
+	src/core/or/connection_edge.h			\
+	src/core/or/connection_or.h			\
+	src/core/or/connection_st.h			\
+	src/core/or/crypt_path.h			\
+	src/core/or/cpath_build_state_st.h		\
+	src/core/or/crypt_path_reference_st.h		\
+	src/core/or/crypt_path_st.h			\
+	src/core/or/destroy_cell_queue_st.h		\
+	src/core/or/dos.h				\
+	src/core/or/edge_connection_st.h		\
+	src/core/or/half_edge_st.h			\
+	src/core/or/entry_connection_st.h		\
+	src/core/or/entry_port_cfg_st.h			\
+	src/core/or/extend_info_st.h			\
+	src/core/or/listener_connection_st.h		\
+	src/core/or/onion.h				\
+	src/core/or/or.h				\
+	src/core/or/or_periodic.h			\
+	src/core/or/or_sys.h				\
+	src/core/or/orconn_event.h			\
+	src/core/or/or_circuit_st.h			\
+	src/core/or/or_connection_st.h			\
+	src/core/or/or_handshake_certs_st.h		\
+	src/core/or/or_handshake_state_st.h		\
+	src/core/or/ocirc_event.h			\
+	src/core/or/origin_circuit_st.h			\
+	src/core/or/policies.h				\
+	src/core/or/port_cfg_st.h			\
+	src/core/or/protover.h				\
+	src/core/or/reasons.h				\
+	src/core/or/relay.h				\
+	src/core/or/relay_crypto_st.h			\
+	src/core/or/scheduler.h				\
+	src/core/or/sendme.h				\
+	src/core/or/server_port_cfg_st.h		\
+	src/core/or/socks_request_st.h			\
+	src/core/or/status.h				\
+	src/core/or/tor_version_st.h			\
+	src/core/or/var_cell_st.h			\
+	src/core/or/versions.h
diff --git a/src/core/or/listener_connection_st.h b/src/core/or/listener_connection_st.h
index 8989a39dc8..78175ea061 100644
--- a/src/core/or/listener_connection_st.h
+++ b/src/core/or/listener_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file listener_connection_st.h
+ * @brief Listener connection structure.
+ **/
+
 #ifndef LISTENER_CONNECTION_ST_H
 #define LISTENER_CONNECTION_ST_H
 
@@ -21,5 +26,4 @@ struct listener_connection_t {
 
 };
 
-#endif
-
+#endif /* !defined(LISTENER_CONNECTION_ST_H) */
diff --git a/src/core/or/ocirc_event.c b/src/core/or/ocirc_event.c
new file mode 100644
index 0000000000..fa16459175
--- /dev/null
+++ b/src/core/or/ocirc_event.c
@@ -0,0 +1,121 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.c
+ * \brief Publish state change messages for origin circuits
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of origin circuits.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/or_sys.h"
+#include "core/or/origin_circuit_st.h"
+#include "lib/subsys/subsys.h"
+
+DECLARE_PUBLISH(ocirc_state);
+DECLARE_PUBLISH(ocirc_chan);
+DECLARE_PUBLISH(ocirc_cevent);
+
+static void
+ocirc_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+ocirc_state_fmt(msg_aux_data_t u)
+{
+  ocirc_state_msg_t *msg = (ocirc_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" state=%d onehop=%d>",
+               msg->gid, msg->state, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_chan_fmt(msg_aux_data_t u)
+{
+  ocirc_chan_msg_t *msg = (ocirc_chan_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" chan=%"PRIu64" onehop=%d>",
+               msg->gid, msg->chan, msg->onehop);
+  return s;
+}
+
+static char *
+ocirc_cevent_fmt(msg_aux_data_t u)
+{
+  ocirc_cevent_msg_t *msg = (ocirc_cevent_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu32" evtype=%d reason=%d onehop=%d>",
+               msg->gid, msg->evtype, msg->reason, msg->onehop);
+  return s;
+}
+
+static dispatch_typefns_t ocirc_state_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_state_fmt,
+};
+
+static dispatch_typefns_t ocirc_chan_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_chan_fmt,
+};
+
+static dispatch_typefns_t ocirc_cevent_fns = {
+  .free_fn = ocirc_event_free,
+  .fmt_fn = ocirc_cevent_fmt,
+};
+
+int
+ocirc_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_state, &ocirc_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_chan, &ocirc_chan_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, ocirc_cevent, &ocirc_cevent_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_state))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  return 0;
+}
+
+void
+ocirc_state_publish(ocirc_state_msg_t *msg)
+{
+  PUBLISH(ocirc_state, msg);
+}
+
+void
+ocirc_chan_publish(ocirc_chan_msg_t *msg)
+{
+  PUBLISH(ocirc_chan, msg);
+}
+
+void
+ocirc_cevent_publish(ocirc_cevent_msg_t *msg)
+{
+  PUBLISH(ocirc_cevent, msg);
+}
diff --git a/src/core/or/ocirc_event.h b/src/core/or/ocirc_event.h
new file mode 100644
index 0000000000..10307a3664
--- /dev/null
+++ b/src/core/or/ocirc_event.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ocirc_event.h
+ * \brief Header file for ocirc_event.c
+ **/
+
+#ifndef TOR_OCIRC_EVENT_H
+#define TOR_OCIRC_EVENT_H
+
+#include <stdbool.h>
+
+#include "lib/cc/torint.h"
+#include "lib/pubsub/pubsub.h"
+
+/** Used to indicate the type of a circuit event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum circuit_status_event_t {
+  CIRC_EVENT_LAUNCHED = 0,
+  CIRC_EVENT_BUILT    = 1,
+  CIRC_EVENT_EXTENDED = 2,
+  CIRC_EVENT_FAILED   = 3,
+  CIRC_EVENT_CLOSED   = 4,
+} circuit_status_event_t;
+
+/** Message for origin circuit state update */
+typedef struct ocirc_state_msg_t {
+  uint32_t gid;       /**< global ID (only origin circuits have them) */
+  int state;          /**< new circuit state */
+  bool onehop;        /**< one-hop circuit? */
+} ocirc_state_msg_t;
+
+DECLARE_MESSAGE(ocirc_state, ocirc_state, ocirc_state_msg_t *);
+
+/**
+ * Message when a channel gets associated to a circuit.
+ *
+ * This doesn't always correspond to something in circuitbuild.c
+ * setting the n_chan field in the circuit.  For some reason, if
+ * circuit_handle_first_hop() launches a new circuit, it doesn't set
+ * the n_chan field.
+ */
+typedef struct ocirc_chan_msg_t {
+  uint32_t gid;                 /**< global ID */
+  uint64_t chan;                /**< channel ID */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_chan_msg_t;
+
+DECLARE_MESSAGE(ocirc_chan, ocirc_chan, ocirc_chan_msg_t *);
+
+/**
+ * Message for origin circuit status event
+ *
+ * This contains information that ends up in CIRC control protocol events.
+ */
+typedef struct ocirc_cevent_msg_t {
+  uint32_t gid;                 /**< global ID */
+  int evtype;                   /**< event type */
+  int reason;                   /**< reason */
+  bool onehop;                  /**< one-hop circuit? */
+} ocirc_cevent_msg_t;
+
+DECLARE_MESSAGE(ocirc_cevent, ocirc_cevent, ocirc_cevent_msg_t *);
+
+#ifdef OCIRC_EVENT_PRIVATE
+void ocirc_state_publish(ocirc_state_msg_t *msg);
+void ocirc_chan_publish(ocirc_chan_msg_t *msg);
+void ocirc_cevent_publish(ocirc_cevent_msg_t *msg);
+#endif
+
+#endif /* !defined(TOR_OCIRC_EVENT_H) */
diff --git a/src/core/or/onion.c b/src/core/or/onion.c
index aeddedd807..a3b5c6922d 100644
--- a/src/core/or/onion.c
+++ b/src/core/or/onion.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -240,11 +240,21 @@ created_cell_parse(created_cell_t *cell_out, const cell_t *cell_in)
 static int
 check_extend_cell(const extend_cell_t *cell)
 {
+  const bool is_extend2 = (cell->cell_type == RELAY_COMMAND_EXTEND2);
+
   if (tor_digest_is_zero((const char*)cell->node_id))
     return -1;
-  /* We don't currently allow EXTEND2 cells without an IPv4 address */
-  if (tor_addr_family(&cell->orport_ipv4.addr) == AF_UNSPEC)
-    return -1;
+  if (!tor_addr_port_is_valid_ap(&cell->orport_ipv4, 0)) {
+    /* EXTEND cells must have an IPv4 address. */
+    if (!is_extend2) {
+      return -1;
+    }
+    /* EXTEND2 cells must have at least one IP address.
+     * It can be IPv4 or IPv6. */
+    if (!tor_addr_port_is_valid_ap(&cell->orport_ipv6, 0)) {
+      return -1;
+    }
+  }
   if (cell->create_cell.cell_type == CELL_CREATE) {
     if (cell->cell_type != RELAY_COMMAND_EXTEND)
       return -1;
@@ -343,7 +353,7 @@ extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
           continue;
         found_ipv6 = 1;
         tor_addr_from_ipv6_bytes(&cell_out->orport_ipv6.addr,
-                                 (const char *)ls->un_ipv6_addr);
+                                 ls->un_ipv6_addr);
         cell_out->orport_ipv6.port = ls->un_ipv6_port;
         break;
       case LS_LEGACY_ID:
@@ -364,7 +374,12 @@ extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
     }
   }
 
-  if (!found_rsa_id || !found_ipv4) /* These are mandatory */
+  /* EXTEND2 cells must have an RSA ID */
+  if (!found_rsa_id)
+    return -1;
+
+  /* EXTEND2 cells must have at least one IP address */
+  if (!found_ipv4 && !found_ipv6)
     return -1;
 
   return create_cell_from_create2_cell_body(&cell_out->create_cell,
@@ -374,9 +389,11 @@ extend_cell_from_extend2_cell_body(extend_cell_t *cell_out,
 /** Parse an EXTEND or EXTEND2 cell (according to <b>command</b>) from the
  * <b>payload_length</b> bytes of <b>payload</b> into <b>cell_out</b>. Return
  * 0 on success, -1 on failure. */
-int
-extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
-                  const uint8_t *payload, size_t payload_length)
+MOCK_IMPL(int,
+extend_cell_parse,(extend_cell_t *cell_out,
+                   const uint8_t command,
+                   const uint8_t *payload,
+                   size_t payload_length))
 {
 
   tor_assert(cell_out);
@@ -618,12 +635,13 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
     break;
   case RELAY_COMMAND_EXTEND2:
     {
-      uint8_t n_specifiers = 2;
+      uint8_t n_specifiers = 1;
       *command_out = RELAY_COMMAND_EXTEND2;
       extend2_cell_body_t *cell = extend2_cell_body_new();
       link_specifier_t *ls;
-      {
-        /* IPv4 specifier first. */
+      if (tor_addr_port_is_valid_ap(&cell_in->orport_ipv4, 0)) {
+        /* Maybe IPv4 specifier first. */
+        ++n_specifiers;
         ls = link_specifier_new();
         extend2_cell_body_add_ls(cell, ls);
         ls->ls_type = LS_IPV4;
@@ -649,6 +667,17 @@ extend_cell_format(uint8_t *command_out, uint16_t *len_out,
         ls->ls_len = 32;
         memcpy(ls->un_ed25519_id, cell_in->ed_pubkey.pubkey, 32);
       }
+      if (tor_addr_port_is_valid_ap(&cell_in->orport_ipv6, 0)) {
+        /* Then maybe IPv6 specifier. */
+        ++n_specifiers;
+        ls = link_specifier_new();
+        extend2_cell_body_add_ls(cell, ls);
+        ls->ls_type = LS_IPV6;
+        ls->ls_len = 18;
+        tor_addr_copy_ipv6_bytes(ls->un_ipv6_addr,
+                                 &cell_in->orport_ipv6.addr);
+        ls->un_ipv6_port = cell_in->orport_ipv6.port;
+      }
       cell->n_spec = n_specifiers;
 
       /* Now, the handshake */
diff --git a/src/core/or/onion.h b/src/core/or/onion.h
index bb0b5b8dfd..256f0a3f31 100644
--- a/src/core/or/onion.h
+++ b/src/core/or/onion.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,8 +56,8 @@ typedef struct extend_cell_t {
   /** Ed25519 public identity key. Zero if not set. */
   struct ed25519_public_key_t ed_pubkey;
   /** The "create cell" embedded in this extend cell. Note that unlike the
-   * create cells we generate ourself, this once can have a handshake type we
-   * don't recognize. */
+   * create cells we generate ourselves, this create cell can have a handshake
+   * type we don't recognize. */
   create_cell_t create_cell;
 } extend_cell_t;
 
@@ -74,8 +74,10 @@ void create_cell_init(create_cell_t *cell_out, uint8_t cell_type,
                       const uint8_t *onionskin);
 int create_cell_parse(create_cell_t *cell_out, const cell_t *cell_in);
 int created_cell_parse(created_cell_t *cell_out, const cell_t *cell_in);
-int extend_cell_parse(extend_cell_t *cell_out, const uint8_t command,
-                      const uint8_t *payload_in, size_t payload_len);
+MOCK_DECL(int,extend_cell_parse,(extend_cell_t *cell_out,
+                                 const uint8_t command,
+                                 const uint8_t *payload_in,
+                                 size_t payload_len));
 int extended_cell_parse(extended_cell_t *cell_out, const uint8_t command,
                         const uint8_t *payload_in, size_t payload_len);
 
diff --git a/src/core/or/or.h b/src/core/or/or.h
index 7c601e49b3..8758a2ec6f 100644
--- a/src/core/or/or.h
+++ b/src/core/or/or.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,9 +24,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/compat_compiler.h"
-#include "lib/cc/torint.h"
 #include "lib/container/map.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rsa.h"
@@ -97,6 +96,8 @@ struct curve25519_public_key_t;
 #define SIGNEWNYM 129
 #define SIGCLEARDNSCACHE 130
 #define SIGHEARTBEAT 131
+#define SIGACTIVE 132
+#define SIGDORMANT 133
 
 #if (SIZEOF_CELL_T != 0)
 /* On Irix, stdlib.h defines a cell_t type, so we need to make sure
@@ -166,12 +167,13 @@ struct curve25519_public_key_t;
 #define PROXY_CONNECT 1
 #define PROXY_SOCKS4 2
 #define PROXY_SOCKS5 3
-/* !!!! If there is ever a PROXY_* type over 3, we must grow the proxy_type
+#define PROXY_HAPROXY 4
+/* !!!! If there is ever a PROXY_* type over 7, we must grow the proxy_type
  * field in or_connection_t */
 
 /* Pluggable transport proxy type. Don't use this in or_connection_t,
  * instead use the actual underlying proxy type (see above).  */
-#define PROXY_PLUGGABLE 4
+#define PROXY_PLUGGABLE 5
 
 /** How many circuits do we want simultaneously in-progress to handle
  * a given stream? */
@@ -205,6 +207,9 @@ struct curve25519_public_key_t;
 #define RELAY_COMMAND_RENDEZVOUS_ESTABLISHED 39
 #define RELAY_COMMAND_INTRODUCE_ACK 40
 
+#define RELAY_COMMAND_PADDING_NEGOTIATE 41
+#define RELAY_COMMAND_PADDING_NEGOTIATED 42
+
 /* Reasons why an OR connection is closed. */
 #define END_OR_CONN_REASON_DONE           1
 #define END_OR_CONN_REASON_REFUSED        2 /* connection refused */
@@ -215,7 +220,8 @@ struct curve25519_public_key_t;
 #define END_OR_CONN_REASON_IO_ERROR       7 /* read/write error */
 #define END_OR_CONN_REASON_RESOURCE_LIMIT 8 /* sockets, buffers, etc */
 #define END_OR_CONN_REASON_PT_MISSING     9 /* PT failed or not available */
-#define END_OR_CONN_REASON_MISC           10
+#define END_OR_CONN_REASON_TLS_ERROR      10 /* Problem in TLS protocol */
+#define END_OR_CONN_REASON_MISC           11
 
 /* Reasons why we (or a remote OR) might close a stream. See tor-spec.txt for
  * documentation of these.  The values must match. */
@@ -604,21 +610,21 @@ typedef uint32_t circid_t;
 /** Identifies a stream on a circuit */
 typedef uint16_t streamid_t;
 
-/* channel_t typedef; struct channel_s is in channel.h */
+/* channel_t typedef; struct channel_t is in channel.h */
 
-typedef struct channel_s channel_t;
+typedef struct channel_t channel_t;
 
-/* channel_listener_t typedef; struct channel_listener_s is in channel.h */
+/* channel_listener_t typedef; struct channel_listener_t is in channel.h */
 
-typedef struct channel_listener_s channel_listener_t;
+typedef struct channel_listener_t channel_listener_t;
 
 /* TLS channel stuff */
 
-typedef struct channel_tls_s channel_tls_t;
+typedef struct channel_tls_t channel_tls_t;
 
-/* circuitmux_t typedef; struct circuitmux_s is in circuitmux.h */
+/* circuitmux_t typedef; struct circuitmux_t is in circuitmux.h */
 
-typedef struct circuitmux_s circuitmux_t;
+typedef struct circuitmux_t circuitmux_t;
 
 typedef struct cell_t cell_t;
 typedef struct var_cell_t var_cell_t;
@@ -834,6 +840,14 @@ typedef struct protover_summary_flags_t {
    * service rendezvous point supporting version 3 as seen in proposal 224.
    * This requires HSRend=2. */
   unsigned int supports_v3_rendezvous_point: 1;
+
+  /** True iff this router has a protocol list that allows clients to
+   * negotiate hs circuit setup padding. Requires Padding>=2. */
+  unsigned int supports_hs_setup_padding : 1;
+
+  /** True iff this router has a protocol list that allows it to support the
+   * ESTABLISH_INTRO DoS cell extension. Requires HSIntro>=5. */
+  unsigned int supports_establish_intro_dos_extension : 1;
 } protover_summary_flags_t;
 
 typedef struct routerinfo_t routerinfo_t;
@@ -982,8 +996,6 @@ typedef struct routerset_t routerset_t;
 
 typedef struct or_options_t or_options_t;
 
-#define LOG_PROTOCOL_WARN (get_protocol_warning_severity_level())
-
 typedef struct or_state_t or_state_t;
 
 #define MAX_SOCKS_ADDR_LEN 256
@@ -1000,7 +1012,7 @@ typedef struct or_state_t or_state_t;
 #define BW_MIN_WEIGHT_SCALE 1
 #define BW_MAX_WEIGHT_SCALE INT32_MAX
 
-typedef struct circuit_build_times_s circuit_build_times_t;
+typedef struct circuit_build_times_t circuit_build_times_t;
 
 /********************************* config.c ***************************/
 
diff --git a/src/core/or/or_circuit_st.h b/src/core/or/or_circuit_st.h
index 6b6feb9d89..4e17b1c143 100644
--- a/src/core/or/or_circuit_st.h
+++ b/src/core/or/or_circuit_st.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef OR_CIRCUIT_ST_H
@@ -12,6 +12,8 @@
 #include "core/or/circuit_st.h"
 #include "core/or/crypt_path_st.h"
 
+#include "lib/evloop/token_bucket.h"
+
 struct onion_queue_t;
 
 /** An or_circuit_t holds information needed to implement a circuit at an
@@ -25,7 +27,7 @@ struct or_circuit_t {
   /** Pointer to a workqueue entry, if this circuit has given an onionskin to
    * a cpuworker and is waiting for a response. Used to decide whether it is
    * safe to free a circuit or if it is still in use by a cpuworker. */
-  struct workqueue_entry_s *workqueue_entry;
+  struct workqueue_entry_t *workqueue_entry;
 
   /** The circuit_id used in the previous (backward) hop of this circuit. */
   circid_t p_circ_id;
@@ -33,11 +35,6 @@ struct or_circuit_t {
   cell_queue_t p_chan_cells;
   /** The channel that is previous in this circuit. */
   channel_t *p_chan;
-  /**
-   * Circuit mux associated with p_chan to which this circuit is attached;
-   * NULL if we have no p_chan.
-   */
-  circuitmux_t *p_mux;
   /** Linked list of Exit streams associated with this circuit. */
   edge_connection_t *n_streams;
   /** Linked list of Exit streams associated with this circuit that are
@@ -74,7 +71,20 @@ struct or_circuit_t {
    * exit-ward queues of this circuit; reset every time when writing
    * buffer stats to disk. */
   uint64_t total_cell_waiting_time;
+
+  /** If set, the DoS defenses are enabled on this circuit meaning that the
+   * introduce2_bucket is initialized and used. */
+  unsigned int introduce2_dos_defense_enabled : 1;
+  /** If set, the DoS defenses were explicitly enabled through the
+   * ESTABLISH_INTRO cell extension. If unset, the consensus is used to learn
+   * if the defenses can be enabled or not. */
+  unsigned int introduce2_dos_defense_explicit : 1;
+
+  /** INTRODUCE2 cell bucket controlling how much can go on this circuit. Only
+   * used if this is a service introduction circuit at the intro point
+   * (purpose = CIRCUIT_PURPOSE_INTRO_POINT). */
+  token_bucket_ctr_t introduce2_bucket;
 };
 
-#endif
+#endif /* !defined(OR_CIRCUIT_ST_H) */
 
diff --git a/src/core/or/or_connection_st.h b/src/core/or/or_connection_st.h
index d5db5e8694..92956c2847 100644
--- a/src/core/or/or_connection_st.h
+++ b/src/core/or/or_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file or_connection_st.h
+ * @brief OR connection structure.
+ **/
+
 #ifndef OR_CONNECTION_ST_H
 #define OR_CONNECTION_ST_H
 
@@ -58,7 +63,7 @@ struct or_connection_t {
 
   /** True iff this is an outgoing connection. */
   unsigned int is_outgoing:1;
-  unsigned int proxy_type:2; /**< One of PROXY_NONE...PROXY_SOCKS5 */
+  unsigned int proxy_type:3; /**< One of PROXY_NONE...PROXY_HAPROXY */
   unsigned int wide_circ_ids:1;
   /** True iff this connection has had its bootstrap failure logged with
    * control_event_bootstrap_problem. */
@@ -67,6 +72,8 @@ struct or_connection_t {
    * geoip cache and handled by the DoS mitigation subsystem. We use this to
    * insure we have a coherent count of concurrent connection. */
   unsigned int tracked_for_dos_mitigation : 1;
+  /** True iff this connection is using a pluggable transport */
+  unsigned int is_pt : 1;
 
   uint16_t link_proto; /**< What protocol version are we using? 0 for
                         * "none negotiated yet." */
@@ -89,4 +96,4 @@ struct or_connection_t {
   uint64_t bytes_xmitted, bytes_xmitted_by_tls;
 };
 
-#endif
+#endif /* !defined(OR_CONNECTION_ST_H) */
diff --git a/src/core/or/or_handshake_certs_st.h b/src/core/or/or_handshake_certs_st.h
index a93b7104aa..31755f04a1 100644
--- a/src/core/or/or_handshake_certs_st.h
+++ b/src/core/or/or_handshake_certs_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file or_handshake_certs_st.h
+ * @brief OR handshake certs structure
+ **/
+
 #ifndef OR_HANDSHAKE_CERTS_ST
 #define OR_HANDSHAKE_CERTS_ST
 
@@ -37,4 +42,4 @@ struct or_handshake_certs_t {
   size_t ed_rsa_crosscert_len;
 };
 
-#endif
+#endif /* !defined(OR_HANDSHAKE_CERTS_ST) */
diff --git a/src/core/or/or_handshake_state_st.h b/src/core/or/or_handshake_state_st.h
index 09a8a34179..050404046d 100644
--- a/src/core/or/or_handshake_state_st.h
+++ b/src/core/or/or_handshake_state_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file or_handshake_state_st.h
+ * @brief OR handshake state structure
+ **/
+
 #ifndef OR_HANDSHAKE_STATE_ST
 #define OR_HANDSHAKE_STATE_ST
 
@@ -74,5 +79,4 @@ struct or_handshake_state_t {
   or_handshake_certs_t *certs;
 };
 
-#endif
-
+#endif /* !defined(OR_HANDSHAKE_STATE_ST) */
diff --git a/src/core/or/or_periodic.c b/src/core/or/or_periodic.c
new file mode 100644
index 0000000000..4dfdce14ca
--- /dev/null
+++ b/src/core/or/or_periodic.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.c
+ * @brief Periodic callbacks for the onion routing subsystem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+
+#include "core/or/channel.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_periodic.h"
+
+#include "feature/relay/routermode.h"
+
+#ifndef COCCI
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+#endif /* !defined(COCCI) */
+
+#define FL(name) (PERIODIC_EVENT_FLAG_ ## name)
+
+#define CHANNEL_CHECK_INTERVAL (60*60)
+static int
+check_canonical_channels_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  if (public_server_mode(options))
+    channel_check_for_duplicates();
+
+  return CHANNEL_CHECK_INTERVAL;
+}
+
+DECLARE_EVENT(check_canonical_channels, RELAY, FL(NEED_NET));
+
+/**
+ * Periodic callback: as a server, see if we have any old unused circuits
+ * that should be expired */
+static int
+expire_old_circuits_serverside_callback(time_t now,
+                                        const or_options_t *options)
+{
+  (void)options;
+  /* every 11 seconds, so not usually the same second as other such events */
+  circuit_expire_old_circuits_serverside(now);
+  return 11;
+}
+
+DECLARE_EVENT(expire_old_circuits_serverside, ROUTER, FL(NEED_NET));
+
+void
+or_register_periodic_events(void)
+{
+  // These are router-only events, but they're owned by the OR subsystem. */
+  periodic_events_register(&check_canonical_channels_event);
+  periodic_events_register(&expire_old_circuits_serverside_event);
+}
diff --git a/src/core/or/or_periodic.h b/src/core/or/or_periodic.h
new file mode 100644
index 0000000000..080573a838
--- /dev/null
+++ b/src/core/or/or_periodic.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_periodic.h
+ * @brief Header for core/or/or_periodic.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_PERIODIC_H
+#define TOR_CORE_OR_OR_PERIODIC_H
+
+void or_register_periodic_events(void);
+
+#endif /* !defined(TOR_CORE_OR_OR_PERIODIC_H) */
diff --git a/src/core/or/or_sys.c b/src/core/or/or_sys.c
new file mode 100644
index 0000000000..73c6087dce
--- /dev/null
+++ b/src/core/or/or_sys.c
@@ -0,0 +1,56 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.c
+ * @brief Subsystem definitions for OR module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/or/or_periodic.h"
+#include "core/or/or_sys.h"
+#include "core/or/policies.h"
+#include "core/or/protover.h"
+#include "core/or/versions.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_or_initialize(void)
+{
+  or_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_or_shutdown(void)
+{
+  protover_free_all();
+  protover_summary_cache_free_all();
+  policies_free_all();
+}
+
+static int
+subsys_or_add_pubsub(struct pubsub_connector_t *connector)
+{
+  int rv = 0;
+  if (orconn_add_pubsub(connector) < 0)
+    rv = -1;
+  if (ocirc_add_pubsub(connector) < 0)
+    rv = -1;
+  return rv;
+}
+
+const struct subsys_fns_t sys_or = {
+  .name = "or",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = 20,
+  .initialize = subsys_or_initialize,
+  .shutdown = subsys_or_shutdown,
+  .add_pubsub = subsys_or_add_pubsub,
+};
diff --git a/src/core/or/or_sys.h b/src/core/or/or_sys.h
new file mode 100644
index 0000000000..7ee56c8682
--- /dev/null
+++ b/src/core/or/or_sys.h
@@ -0,0 +1,21 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file or_sys.h
+ * @brief Header for core/or/or_sys.c
+ **/
+
+#ifndef TOR_CORE_OR_OR_SYS_H
+#define TOR_CORE_OR_OR_SYS_H
+
+extern const struct subsys_fns_t sys_or;
+
+struct pubsub_connector_t;
+int ocirc_add_pubsub(struct pubsub_connector_t *connector);
+int orconn_add_pubsub(struct pubsub_connector_t *connector);
+
+#endif /* !defined(TOR_CORE_OR_OR_SYS_H) */
diff --git a/src/core/or/orconn_event.c b/src/core/or/orconn_event.c
new file mode 100644
index 0000000000..c30e2dd22f
--- /dev/null
+++ b/src/core/or/orconn_event.c
@@ -0,0 +1,92 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.c
+ * \brief Publish state change messages for OR connections
+ *
+ * Implements a basic publish-subscribe framework for messages about
+ * the state of OR connections.  The publisher calls the subscriber
+ * callback functions synchronously.
+ *
+ * Although the synchronous calls might not simplify the call graph,
+ * this approach improves data isolation because the publisher doesn't
+ * need knowledge about the internals of subscribing subsystems.  It
+ * also avoids race conditions that might occur in asynchronous
+ * frameworks.
+ **/
+
+#include "core/or/or.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/orconn_event.h"
+#include "core/or/or_sys.h"
+
+DECLARE_PUBLISH(orconn_state);
+DECLARE_PUBLISH(orconn_status);
+
+static void
+orconn_event_free(msg_aux_data_t u)
+{
+  tor_free_(u.ptr);
+}
+
+static char *
+orconn_state_fmt(msg_aux_data_t u)
+{
+  orconn_state_msg_t *msg = (orconn_state_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" chan=%"PRIu64" proxy_type=%d state=%d>",
+               msg->gid, msg->chan, msg->proxy_type, msg->state);
+  return s;
+}
+
+static char *
+orconn_status_fmt(msg_aux_data_t u)
+{
+  orconn_status_msg_t *msg = (orconn_status_msg_t *)u.ptr;
+  char *s = NULL;
+
+  tor_asprintf(&s, "<gid=%"PRIu64" status=%d reason=%d>",
+               msg->gid, msg->status, msg->reason);
+  return s;
+}
+
+static dispatch_typefns_t orconn_state_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_state_fmt,
+};
+
+static dispatch_typefns_t orconn_status_fns = {
+  .free_fn = orconn_event_free,
+  .fmt_fn = orconn_status_fmt,
+};
+
+int
+orconn_add_pubsub(struct pubsub_connector_t *connector)
+{
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_state, &orconn_state_fns))
+    return -1;
+  if (DISPATCH_REGISTER_TYPE(connector, orconn_status, &orconn_status_fns))
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_state) != 0)
+    return -1;
+  if (DISPATCH_ADD_PUB(connector, orconn, orconn_status) != 0)
+    return -1;
+  return 0;
+}
+
+void
+orconn_state_publish(orconn_state_msg_t *msg)
+{
+  PUBLISH(orconn_state, msg);
+}
+
+void
+orconn_status_publish(orconn_status_msg_t *msg)
+{
+  PUBLISH(orconn_status, msg);
+}
diff --git a/src/core/or/orconn_event.h b/src/core/or/orconn_event.h
new file mode 100644
index 0000000000..2653b20d6e
--- /dev/null
+++ b/src/core/or/orconn_event.h
@@ -0,0 +1,103 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file orconn_event.h
+ * \brief Header file for orconn_event.c
+ *
+ * The OR_CONN_STATE_* symbols are here to make it easier for
+ * subscribers to make decisions based on the messages that they
+ * receive.
+ **/
+
+#ifndef TOR_ORCONN_EVENT_H
+#define TOR_ORCONN_EVENT_H
+
+#include "lib/pubsub/pubsub.h"
+
+/**
+ * @name States of OR connections
+ *
+ * These must be in a partial ordering such that usually no OR
+ * connection will transition from a higher-numbered state to a
+ * lower-numbered one.  Code such as bto_update_best() depends on this
+ * ordering to determine the best state it's seen so far.
+ * @{ */
+#define OR_CONN_STATE_MIN_ 1
+/** State for a connection to an OR: waiting for connect() to finish. */
+#define OR_CONN_STATE_CONNECTING 1
+/** State for a connection to an OR: waiting for proxy handshake to complete */
+#define OR_CONN_STATE_PROXY_HANDSHAKING 2
+/** State for an OR connection client: SSL is handshaking, not done
+ * yet. */
+#define OR_CONN_STATE_TLS_HANDSHAKING 3
+/** State for a connection to an OR: We're doing a second SSL handshake for
+ * renegotiation purposes. (V2 handshake only.) */
+#define OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING 4
+/** State for a connection at an OR: We're waiting for the client to
+ * renegotiate (to indicate a v2 handshake) or send a versions cell (to
+ * indicate a v3 handshake) */
+#define OR_CONN_STATE_TLS_SERVER_RENEGOTIATING 5
+/** State for an OR connection: We're done with our SSL handshake, we've done
+ * renegotiation, but we haven't yet negotiated link protocol versions and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V2 6
+/** State for an OR connection: We're done with our SSL handshake, but we
+ * haven't yet negotiated link protocol versions, done a V3 handshake, and
+ * sent a netinfo cell. */
+#define OR_CONN_STATE_OR_HANDSHAKING_V3 7
+/** State for an OR connection: Ready to send/receive cells. */
+#define OR_CONN_STATE_OPEN 8
+#define OR_CONN_STATE_MAX_ 8
+/** @} */
+
+/** Used to indicate the type of an OR connection event passed to the
+ * controller.  The various types are defined in control-spec.txt */
+typedef enum or_conn_status_event_t {
+  OR_CONN_EVENT_LAUNCHED     = 0,
+  OR_CONN_EVENT_CONNECTED    = 1,
+  OR_CONN_EVENT_FAILED       = 2,
+  OR_CONN_EVENT_CLOSED       = 3,
+  OR_CONN_EVENT_NEW          = 4,
+} or_conn_status_event_t;
+
+/**
+ * Message for orconn state update
+ *
+ * This contains information about internal state changes of
+ * or_connection_t objects.  The chan and proxy_type fields are
+ * additional information that a subscriber may need to make
+ * decisions.
+ **/
+typedef struct orconn_state_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  uint64_t chan;                /**< associated channel ID */
+  int proxy_type;               /**< connection's proxy type */
+  uint8_t state;                /**< new connection state */
+} orconn_state_msg_t;
+
+DECLARE_MESSAGE(orconn_state, orconn_state, orconn_state_msg_t *);
+
+/**
+ * Message for orconn status event
+ *
+ * This contains information that ends up in ORCONN control protocol
+ * events.
+ **/
+typedef struct orconn_status_msg_t {
+  uint64_t gid;                 /**< connection's global ID */
+  int status;                   /**< or_conn_status_event_t */
+  int reason;                   /**< reason */
+} orconn_status_msg_t;
+
+DECLARE_MESSAGE(orconn_status, orconn_status, orconn_status_msg_t *);
+
+#ifdef ORCONN_EVENT_PRIVATE
+void orconn_state_publish(orconn_state_msg_t *);
+void orconn_status_publish(orconn_status_msg_t *);
+#endif
+
+#endif /* !defined(TOR_ORCONN_EVENT_H) */
diff --git a/src/core/or/origin_circuit_st.h b/src/core/or/origin_circuit_st.h
index f55416db14..79e250cd59 100644
--- a/src/core/or/origin_circuit_st.h
+++ b/src/core/or/origin_circuit_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file origin_circuit_st.h
+ * @brief Origin circuit structure.
+ **/
+
 #ifndef ORIGIN_CIRCUIT_ST_H
 #define ORIGIN_CIRCUIT_ST_H
 
@@ -161,6 +166,10 @@ struct origin_circuit_t {
    * connections to this circuit. */
   unsigned int unusable_for_new_conns : 1;
 
+  /* If this flag is set (due to padding negotiation failure), we should
+   * not try to negotiate further circuit padding. */
+  unsigned padding_negotiation_failed : 1;
+
   /**
    * Tristate variable to guard against pathbias miscounting
    * due to circuit purpose transitions changing the decision
@@ -291,4 +300,4 @@ struct origin_circuit_t {
 
 };
 
-#endif
+#endif /* !defined(ORIGIN_CIRCUIT_ST_H) */
diff --git a/src/core/or/policies.c b/src/core/or/policies.c
index 3ed282d785..2bf2dc7005 100644
--- a/src/core/or/policies.c
+++ b/src/core/or/policies.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,9 @@
 #include "feature/relay/routermode.h"
 #include "lib/geoip/geoip.h"
 #include "ht.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/encoding/confline.h"
+#include "trunnel/ed25519_cert.h"
 
 #include "core/or/addr_policy_st.h"
 #include "feature/dirclient/dir_server_st.h"
@@ -165,7 +167,7 @@ policy_expand_unspec(smartlist_t **policy)
       }
       tor_addr_from_ipv4h(&newpolicy_ipv4.addr, 0);
       tor_addr_from_ipv6_bytes(&newpolicy_ipv6.addr,
-                               "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
+          (const uint8_t *)"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
       smartlist_add(tmp, addr_policy_get_canonical_entry(&newpolicy_ipv4));
       smartlist_add(tmp, addr_policy_get_canonical_entry(&newpolicy_ipv6));
       addr_policy_free(p);
@@ -918,49 +920,6 @@ fascist_firewall_choose_address_ipv4h(uint32_t ipv4h_addr,
                                        pref_ipv6, ap);
 }
 
-/* Some microdescriptor consensus methods have no IPv6 addresses in rs: they
- * are in the microdescriptors. For these consensus methods, we can't rely on
- * the node's IPv6 address until its microdescriptor is available (when using
- * microdescs).
- * But for bridges, rewrite_node_address_for_bridge() updates node->ri with
- * the configured address, so we can trust bridge addresses.
- * (Bridges could gain an IPv6 address if their microdescriptor arrives, but
- * this will never be their preferred address: that is in the config.)
- * Returns true if the node needs a microdescriptor for its IPv6 address, and
- * false if the addresses in the node are already up-to-date.
- */
-static int
-node_awaiting_ipv6(const or_options_t* options, const node_t *node)
-{
-  tor_assert(node);
-
-  /* There's no point waiting for an IPv6 address if we'd never use it */
-  if (!fascist_firewall_use_ipv6(options)) {
-    return 0;
-  }
-
-  /* If the node has an IPv6 address, we're not waiting */
-  if (node_has_ipv6_addr(node)) {
-    return 0;
-  }
-
-  /* If the current consensus method and flavour has IPv6 addresses, we're not
-   * waiting */
-  if (networkstatus_consensus_has_ipv6(options)) {
-    return 0;
-  }
-
-  /* Bridge clients never use the address from a bridge's md, so there's no
-   * need to wait for it. */
-  if (node_is_a_configured_bridge(node)) {
-    return 0;
-  }
-
-  /* We are waiting if we_use_microdescriptors_for_circuits() and we have no
-   * md. */
-  return (!node->md && we_use_microdescriptors_for_circuits(options));
-}
-
 /** Like fascist_firewall_choose_address_base(), but takes <b>rs</b>.
  * Consults the corresponding node, then falls back to rs if node is NULL.
  * This should only happen when there's no valid consensus, and rs doesn't
@@ -983,7 +942,7 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
   const or_options_t *options = get_options();
   const node_t *node = node_get_by_id(rs->identity_digest);
 
-  if (node && !node_awaiting_ipv6(options, node)) {
+  if (node) {
     fascist_firewall_choose_address_node(node, fw_connection, pref_only, ap);
   } else {
     /* There's no node-specific IPv6 preference, so use the generic IPv6
@@ -1001,6 +960,83 @@ fascist_firewall_choose_address_rs(const routerstatus_t *rs,
   }
 }
 
+/** Like fascist_firewall_choose_address_base(), but takes in a smartlist
+ * <b>lspecs</b> consisting of one or more link specifiers. We assume
+ * fw_connection is FIREWALL_OR_CONNECTION as link specifiers cannot
+ * contain DirPorts.
+ */
+void
+fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                   int pref_only, tor_addr_port_t* ap)
+{
+  int have_v4 = 0, have_v6 = 0;
+  uint16_t port_v4 = 0, port_v6 = 0;
+  tor_addr_t addr_v4, addr_v6;
+
+  tor_assert(ap);
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Unknown or missing link specifiers");
+    return;
+  }
+  if (smartlist_len(lspecs) == 0) {
+    log_warn(LD_PROTOCOL, "Link specifiers are empty");
+    return;
+  }
+
+  tor_addr_make_null(&ap->addr, AF_UNSPEC);
+  ap->port = 0;
+
+  tor_addr_make_null(&addr_v4, AF_INET);
+  tor_addr_make_null(&addr_v6, AF_INET6);
+
+  SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
+    switch (link_specifier_get_ls_type(ls)) {
+    case LS_IPV4:
+      /* Skip if we already seen a v4. */
+      if (have_v4) continue;
+      tor_addr_from_ipv4h(&addr_v4,
+                          link_specifier_get_un_ipv4_addr(ls));
+      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      have_v4 = 1;
+      break;
+    case LS_IPV6:
+      /* Skip if we already seen a v6, or deliberately skip it if we're not a
+       * direct connection. */
+      if (have_v6) continue;
+      tor_addr_from_ipv6_bytes(&addr_v6,
+          link_specifier_getconstarray_un_ipv6_addr(ls));
+      port_v6 = link_specifier_get_un_ipv6_port(ls);
+      have_v6 = 1;
+      break;
+    default:
+      /* Ignore unknown. */
+      break;
+    }
+  } SMARTLIST_FOREACH_END(ls);
+
+  /* If we don't have IPv4 or IPv6 in link specifiers, log a bug and return. */
+  if (!have_v4 && !have_v6) {
+    if (!have_v6) {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4 or IPv6");
+    } else {
+      log_warn(LD_PROTOCOL, "None of our link specifiers have IPv4");
+    }
+    return;
+  }
+
+  /* Here, don't check for DirPorts as link specifiers are only used for
+   * ORPorts. */
+  const or_options_t *options = get_options();
+  int pref_ipv6 = fascist_firewall_prefer_ipv6_orport(options);
+  /* Assume that the DirPorts are zero as link specifiers only use ORPorts. */
+  fascist_firewall_choose_address_base(&addr_v4, port_v4, 0,
+                                       &addr_v6, port_v6, 0,
+                                       FIREWALL_OR_CONNECTION,
+                                       pref_only, pref_ipv6,
+                                       ap);
+}
+
 /** Like fascist_firewall_choose_address_base(), but takes <b>node</b>, and
  * looks up the node's IPv6 preference rather than taking an argument
  * for pref_ipv6. */
@@ -1019,17 +1055,6 @@ fascist_firewall_choose_address_node(const node_t *node,
   }
 
   node_assert_ok(node);
-  /* Calling fascist_firewall_choose_address_node() when the node is missing
-   * IPv6 information breaks IPv6-only clients.
-   * If the node is a hard-coded fallback directory or authority, call
-   * fascist_firewall_choose_address_rs() on the fake (hard-coded) routerstatus
-   * for the node.
-   * If it is not hard-coded, check that the node has a microdescriptor, full
-   * descriptor (routerinfo), or is one of our configured bridges before
-   * calling this function. */
-  if (BUG(node_awaiting_ipv6(get_options(), node))) {
-    return;
-  }
 
   const int pref_ipv6_node = (fw_connection == FIREWALL_OR_CONNECTION
                               ? node_ipv6_or_preferred(node)
@@ -1150,6 +1175,15 @@ authdir_policy_badexit_address(uint32_t addr, uint16_t port)
 #define REJECT(arg) \
   STMT_BEGIN *msg = tor_strdup(arg); goto err; STMT_END
 
+/** Check <b>or_options</b> to determine whether or not we are using the
+ * default options for exit policy. Return true if so, false otherwise. */
+static int
+policy_using_default_exit_options(const or_options_t *or_options)
+{
+  return (or_options->ExitPolicy == NULL && or_options->ExitRelay == -1 &&
+          or_options->ReducedExitPolicy == 0 && or_options->IPv6Exit == 0);
+}
+
 /** Config helper: If there's any problem with the policy configuration
  * options in <b>options</b>, return -1 and set <b>msg</b> to a newly
  * allocated description of the error. Else return 0. */
@@ -1168,9 +1202,8 @@ validate_addr_policies(const or_options_t *options, char **msg)
 
   static int warned_about_nonexit = 0;
 
-  if (public_server_mode(options) &&
-      !warned_about_nonexit && options->ExitPolicy == NULL &&
-      options->ExitRelay == -1 && options->ReducedExitPolicy == 0) {
+  if (public_server_mode(options) && !warned_about_nonexit &&
+      policy_using_default_exit_options(options)) {
     warned_about_nonexit = 1;
     log_notice(LD_CONFIG, "By default, Tor does not run as an exit relay. "
                "If you want to be an exit relay, "
@@ -1359,9 +1392,9 @@ policy_hash(const policy_map_ent_t *ent)
 }
 
 HT_PROTOTYPE(policy_map, policy_map_ent_t, node, policy_hash,
-             policy_eq)
+             policy_eq);
 HT_GENERATE2(policy_map, policy_map_ent_t, node, policy_hash,
-             policy_eq, 0.6, tor_reallocarray_, tor_free_)
+             policy_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** Given a pointer to an addr_policy_t, return a copy of the pointer to the
  * "canonical" copy of that addr_policy_t; the canonical copy is a single
@@ -2127,9 +2160,9 @@ policies_parse_exit_policy_from_options(const or_options_t *or_options,
   int rv = 0;
 
   /* Short-circuit for non-exit relays, or for relays where we didn't specify
-   * ExitPolicy or ReducedExitPolicy and ExitRelay is auto. */
-  if (or_options->ExitRelay == 0 || (or_options->ExitPolicy == NULL &&
-      or_options->ExitRelay == -1 && or_options->ReducedExitPolicy == 0)) {
+   * ExitPolicy or ReducedExitPolicy or IPv6Exit and ExitRelay is auto. */
+  if (or_options->ExitRelay == 0 ||
+      policy_using_default_exit_options(or_options)) {
     append_exit_policy_string(result, "reject *4:*");
     append_exit_policy_string(result, "reject *6:*");
     return 0;
@@ -2706,7 +2739,7 @@ parse_short_policy(const char *summary)
   int is_accept;
   int n_entries;
   short_policy_entry_t entries[MAX_EXITPOLICY_SUMMARY_LEN]; /* overkill */
-  const char *next;
+  char *next;
 
   if (!strcmpstart(summary, "accept ")) {
     is_accept = 1;
@@ -2721,57 +2754,56 @@ parse_short_policy(const char *summary)
 
   n_entries = 0;
   for ( ; *summary; summary = next) {
-    const char *comma = strchr(summary, ',');
-    unsigned low, high;
-    char dummy;
-    char ent_buf[32];
-    size_t len;
-
-    next = comma ? comma+1 : strchr(summary, '\0');
-    len = comma ? (size_t)(comma - summary) : strlen(summary);
-
     if (n_entries == MAX_EXITPOLICY_SUMMARY_LEN) {
       log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Impossibly long policy summary %s",
              escaped(orig_summary));
       return NULL;
     }
 
-    if (! TOR_ISDIGIT(*summary) || len > (sizeof(ent_buf)-1)) {
-      /* unrecognized entry format. skip it. */
-      continue;
-    }
-    if (len < 1) {
-      /* empty; skip it. */
-      /* XXX This happens to be unreachable, since if len==0, then *summary is
-       * ',' or '\0', and the TOR_ISDIGIT test above would have failed. */
-      continue;
+    unsigned low, high;
+    int ok;
+    low = (unsigned) tor_parse_ulong(summary, 10, 1, 65535, &ok, &next);
+    if (!ok) {
+      if (! TOR_ISDIGIT(*summary) || *summary == ',') {
+        /* Unrecognized format: skip it. */
+        goto skip_ent;
+      } else {
+        goto bad_ent;
+      }
     }
 
-    memcpy(ent_buf, summary, len);
-    ent_buf[len] = '\0';
+    switch (*next) {
+      case ',':
+        ++next;
+        FALLTHROUGH;
+      case '\0':
+        high = low;
+        break;
+      case '-':
+        high = (unsigned) tor_parse_ulong(next+1, 10, low, 65535, &ok, &next);
+        if (!ok)
+          goto bad_ent;
 
-    if (tor_sscanf(ent_buf, "%u-%u%c", &low, &high, &dummy) == 2) {
-      if (low<1 || low>65535 || high<1 || high>65535 || low>high) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-    } else if (tor_sscanf(ent_buf, "%u%c", &low, &dummy) == 1) {
-      if (low<1 || low>65535) {
-        log_fn(LOG_PROTOCOL_WARN, LD_DIR,
-               "Found bad entry in policy summary %s", escaped(orig_summary));
-        return NULL;
-      }
-      high = low;
-    } else {
-      log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
-             escaped(orig_summary));
-      return NULL;
+        if (*next == ',')
+          ++next;
+        else if (*next != '\0')
+          goto bad_ent;
+
+        break;
+      default:
+        goto bad_ent;
     }
 
     entries[n_entries].min_port = low;
     entries[n_entries].max_port = high;
     n_entries++;
+
+    continue;
+  skip_ent:
+    next = strchr(next, ',');
+    if (!next)
+      break;
+    ++next;
   }
 
   if (n_entries == 0) {
@@ -2792,6 +2824,11 @@ parse_short_policy(const char *summary)
   result->n_entries = n_entries;
   memcpy(result->entries, entries, sizeof(short_policy_entry_t)*n_entries);
   return result;
+
+ bad_ent:
+  log_fn(LOG_PROTOCOL_WARN, LD_DIR,"Found bad entry in policy summary %s",
+         escaped(orig_summary));
+  return NULL;
 }
 
 /** Write <b>policy</b> back out into a string. */
diff --git a/src/core/or/policies.h b/src/core/or/policies.h
index 2c38de362f..72a37d62b0 100644
--- a/src/core/or/policies.h
+++ b/src/core/or/policies.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -91,6 +91,8 @@ int fascist_firewall_allows_dir_server(const dir_server_t *ds,
 void fascist_firewall_choose_address_rs(const routerstatus_t *rs,
                                         firewall_connection_t fw_connection,
                                         int pref_only, tor_addr_port_t* ap);
+void fascist_firewall_choose_address_ls(const smartlist_t *lspecs,
+                                        int pref_only, tor_addr_port_t* ap);
 void fascist_firewall_choose_address_node(const node_t *node,
                                           firewall_connection_t fw_connection,
                                           int pref_only, tor_addr_port_t* ap);
diff --git a/src/core/or/port_cfg_st.h b/src/core/or/port_cfg_st.h
index b67091ce32..064e679d78 100644
--- a/src/core/or/port_cfg_st.h
+++ b/src/core/or/port_cfg_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file port_cfg_st.h
+ * @brief Listener port configuration structure.
+ **/
+
 #ifndef PORT_CFG_ST_H
 #define PORT_CFG_ST_H
 
@@ -31,5 +36,4 @@ struct port_cfg_t {
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
-
+#endif /* !defined(PORT_CFG_ST_H) */
diff --git a/src/core/or/protover.c b/src/core/or/protover.c
index dfb0e9e303..0d03e9a06b 100644
--- a/src/core/or/protover.c
+++ b/src/core/or/protover.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -39,6 +39,9 @@ static int protocol_list_contains(const smartlist_t *protos,
 static const struct {
   protocol_type_t protover_type;
   const char *name;
+/* If you add a new protocol here, you probably also want to add
+ * parsing for it in summarize_protover_flags(), so that it has a
+ * summary flag in routerstatus_t */
 } PROTOCOL_NAMES[] = {
   { PRT_LINK, "Link" },
   { PRT_LINKAUTH, "LinkAuth" },
@@ -49,7 +52,9 @@ static const struct {
   { PRT_HSREND, "HSRend" },
   { PRT_DESC, "Desc" },
   { PRT_MICRODESC, "Microdesc"},
-  { PRT_CONS, "Cons" }
+  { PRT_PADDING, "Padding"},
+  { PRT_CONS, "Cons" },
+  { PRT_FLOWCTRL, "FlowCtrl"},
 };
 
 #define N_PROTOCOL_NAMES ARRAY_LENGTH(PROTOCOL_NAMES)
@@ -386,8 +391,9 @@ protover_get_supported_protocols(void)
     "Cons=1-2 "
     "Desc=1-2 "
     "DirCache=1-2 "
+    "FlowCtrl=1 "
     "HSDir=1-2 "
-    "HSIntro=3-4 "
+    "HSIntro=3-5 "
     "HSRend=1-2 "
     "Link=1-5 "
 #ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
@@ -396,6 +402,7 @@ protover_get_supported_protocols(void)
     "LinkAuth=3 "
 #endif
     "Microdesc=1-2 "
+    "Padding=2 "
     "Relay=1-2";
 }
 
@@ -815,6 +822,8 @@ protover_all_supported(const char *s, char **missing_out)
        * ones and, if so, add them to unsupported->ranges. */
       if (versions->low != 0 && versions->high != 0) {
         smartlist_add(unsupported->ranges, versions);
+      } else {
+        tor_free(versions);
       }
       /* Finally, if we had something unsupported, add it to the list of
        * missing_some things and mark that there was something missing. */
@@ -823,7 +832,6 @@ protover_all_supported(const char *s, char **missing_out)
         all_supported = 0;
       } else {
         proto_entry_free(unsupported);
-        tor_free(versions);
       }
     } SMARTLIST_FOREACH_END(range);
 
diff --git a/src/core/or/protover.h b/src/core/or/protover.h
index 7e181ba97a..9509f3e8a3 100644
--- a/src/core/or/protover.h
+++ b/src/core/or/protover.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,21 +28,25 @@ struct smartlist_t;
 #define PROTOVER_HS_INTRO_V3 4
 /** The protover version number that signifies HSv3 rendezvous point support */
 #define PROTOVER_HS_RENDEZVOUS_POINT_V3 2
+/** The protover that signals support for HS circuit setup padding machines */
+#define PROTOVER_HS_SETUP_PADDING 2
 
 /** List of recognized subprotocols. */
 /// C_RUST_COUPLED: src/rust/protover/ffi.rs `translate_to_rust`
 /// C_RUST_COUPLED: src/rust/protover/protover.rs `Proto`
 typedef enum protocol_type_t {
-  PRT_LINK,
-  PRT_LINKAUTH,
-  PRT_RELAY,
-  PRT_DIRCACHE,
-  PRT_HSDIR,
-  PRT_HSINTRO,
-  PRT_HSREND,
-  PRT_DESC,
-  PRT_MICRODESC,
-  PRT_CONS,
+  PRT_LINK      = 0,
+  PRT_LINKAUTH  = 1,
+  PRT_RELAY     = 2,
+  PRT_DIRCACHE  = 3,
+  PRT_HSDIR     = 4,
+  PRT_HSINTRO   = 5,
+  PRT_HSREND    = 6,
+  PRT_DESC      = 7,
+  PRT_MICRODESC = 8,
+  PRT_CONS      = 9,
+  PRT_PADDING   = 10,
+  PRT_FLOWCTRL  = 11,
 } protocol_type_t;
 
 bool protover_contains_long_protocol_names(const char *s);
diff --git a/src/core/or/protover_rust.c b/src/core/or/protover_rust.c
index bc56ea11d0..f44746b6da 100644
--- a/src/core/or/protover_rust.c
+++ b/src/core/or/protover_rust.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /*
diff --git a/src/core/or/reasons.c b/src/core/or/reasons.c
index e21bfa670a..708f43a689 100644
--- a/src/core/or/reasons.c
+++ b/src/core/or/reasons.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -244,6 +244,8 @@ orconn_end_reason_to_control_string(int r)
       return "IOERROR";
     case END_OR_CONN_REASON_RESOURCE_LIMIT:
       return "RESOURCELIMIT";
+    case END_OR_CONN_REASON_TLS_ERROR:
+      return "TLS_ERROR";
     case END_OR_CONN_REASON_MISC:
       return "MISC";
     case END_OR_CONN_REASON_PT_MISSING:
@@ -276,6 +278,8 @@ tls_error_to_orconn_end_reason(int e)
     case TOR_TLS_CLOSE:
     case TOR_TLS_DONE:
       return END_OR_CONN_REASON_DONE;
+    case TOR_TLS_ERROR_MISC:
+      return END_OR_CONN_REASON_TLS_ERROR;
     default:
       return END_OR_CONN_REASON_MISC;
   }
diff --git a/src/core/or/reasons.h b/src/core/or/reasons.h
index c45a8bc38d..2e534aab73 100644
--- a/src/core/or/reasons.h
+++ b/src/core/or/reasons.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/core/or/relay.c b/src/core/or/relay.c
index 00353f47a9..892dbe2d0a 100644
--- a/src/core/or/relay.c
+++ b/src/core/or/relay.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,22 +49,24 @@
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "feature/client/circpathbias.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
+#include "core/or/circuitpadding.h"
 #include "lib/compress/compress.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dircommon/directory.h"
 #include "feature/relay/dns.h"
+#include "feature/relay/circuitbuild_relay.h"
 #include "feature/stats/geoip_stats.h"
 #include "feature/hs/hs_cache.h"
 #include "core/mainloop/mainloop.h"
@@ -80,7 +82,6 @@
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/routerlist.h"
 #include "core/or/scheduler.h"
-#include "feature/stats/rephist.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/cell_queue_st.h"
@@ -93,15 +94,12 @@
 #include "core/or/origin_circuit_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "core/or/socks_request_st.h"
-
-#include "lib/intmath/weakrng.h"
+#include "core/or/sendme.h"
 
 static edge_connection_t *relay_lookup_conn(circuit_t *circ, cell_t *cell,
                                             cell_direction_t cell_direction,
                                             crypt_path_t *layer_hint);
 
-static void circuit_consider_sending_sendme(circuit_t *circ,
-                                            crypt_path_t *layer_hint);
 static void circuit_resume_edge_reading(circuit_t *circ,
                                         crypt_path_t *layer_hint);
 static int circuit_resume_edge_reading_helper(edge_connection_t *conn,
@@ -134,9 +132,6 @@ uint64_t stats_n_relay_cells_delivered = 0;
  * reached (see append_cell_to_circuit_queue()) */
 uint64_t stats_n_circ_max_cell_reached = 0;
 
-/** Used to tell which stream to read from first on a circuit. */
-static tor_weak_rng_t stream_choice_rng = TOR_WEAK_RNG_INIT;
-
 /**
  * Update channel usage state based on the type of relay cell and
  * circuit properties.
@@ -253,6 +248,10 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
   if (recognized) {
     edge_connection_t *conn = NULL;
 
+    /* Recognized cell, the cell digest has been updated, we'll record it for
+     * the SENDME if need be. */
+    sendme_record_received_cell_digest(circ, layer_hint);
+
     if (circ->purpose == CIRCUIT_PURPOSE_PATH_BIAS_TESTING) {
       if (pathbias_check_probe_response(circ, cell) == -1) {
         pathbias_count_valid_cells(circ, cell);
@@ -267,8 +266,8 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_OUT) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending away from origin.");
-      if ((reason=connection_edge_process_relay_cell(cell, circ, conn, NULL))
-          < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn, NULL);
+      if (reason < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "connection_edge_process_relay_cell (away from origin) "
                "failed.");
@@ -278,8 +277,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     if (cell_direction == CELL_DIRECTION_IN) {
       ++stats_n_relay_cells_delivered;
       log_debug(LD_OR,"Sending to origin.");
-      if ((reason = connection_edge_process_relay_cell(cell, circ, conn,
-                                                       layer_hint)) < 0) {
+      reason = connection_edge_process_relay_cell(cell, circ, conn,
+                                                  layer_hint);
+      if (reason < 0) {
         /* If a client is trying to connect to unknown hidden service port,
          * END_CIRC_AT_ORIGIN is sent back so we can then close the circuit.
          * Do not log warn as this is an expected behavior for a service. */
@@ -293,7 +293,9 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
     return 0;
   }
 
-  /* not recognized. pass it on. */
+  /* not recognized. inform circpad and pass it on. */
+  circpad_deliver_unrecognized_cell_events(circ, cell_direction);
+
   if (cell_direction == CELL_DIRECTION_OUT) {
     cell->circ_id = circ->n_circ_id; /* switch it */
     chan = circ->n_chan;
@@ -353,11 +355,11 @@ circuit_receive_relay_cell(cell_t *cell, circuit_t *circ,
  *  - Encrypt it to the right layer
  *  - Append it to the appropriate cell_queue on <b>circ</b>.
  */
-static int
-circuit_package_relay_cell(cell_t *cell, circuit_t *circ,
+MOCK_IMPL(int,
+circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
                            cell_direction_t cell_direction,
                            crypt_path_t *layer_hint, streamid_t on_stream,
-                           const char *filename, int lineno)
+                           const char *filename, int lineno))
 {
   channel_t *chan; /* where to send the cell */
 
@@ -524,6 +526,8 @@ relay_command_to_string(uint8_t command)
     case RELAY_COMMAND_INTRODUCE_ACK: return "INTRODUCE_ACK";
     case RELAY_COMMAND_EXTEND2: return "EXTEND2";
     case RELAY_COMMAND_EXTENDED2: return "EXTENDED2";
+    case RELAY_COMMAND_PADDING_NEGOTIATE: return "PADDING_NEGOTIATE";
+    case RELAY_COMMAND_PADDING_NEGOTIATED: return "PADDING_NEGOTIATED";
     default:
       tor_snprintf(buf, sizeof(buf), "Unrecognized relay command %u",
                    (unsigned)command);
@@ -531,6 +535,64 @@ relay_command_to_string(uint8_t command)
   }
 }
 
+/** When padding a cell with randomness, leave this many zeros after the
+ * payload. */
+#define CELL_PADDING_GAP 4
+
+/** Return the offset where the padding should start. The <b>data_len</b> is
+ * the relay payload length expected to be put in the cell. It can not be
+ * bigger than RELAY_PAYLOAD_SIZE else this function assert().
+ *
+ * Value will always be smaller than CELL_PAYLOAD_SIZE because this offset is
+ * for the entire cell length not just the data payload length. Zero is
+ * returned if there is no room for padding.
+ *
+ * This function always skips the first 4 bytes after the payload because
+ * having some unused zero bytes has saved us a lot of times in the past. */
+
+STATIC size_t
+get_pad_cell_offset(size_t data_len)
+{
+  /* This is never supposed to happen but in case it does, stop right away
+   * because if tor is tricked somehow into not adding random bytes to the
+   * payload with this function returning 0 for a bad data_len, the entire
+   * authenticated SENDME design can be bypassed leading to bad denial of
+   * service attacks. */
+  tor_assert(data_len <= RELAY_PAYLOAD_SIZE);
+
+  /* If the offset is larger than the cell payload size, we return an offset
+   * of zero indicating that no padding needs to be added. */
+  size_t offset = RELAY_HEADER_SIZE + data_len + CELL_PADDING_GAP;
+  if (offset >= CELL_PAYLOAD_SIZE) {
+    return 0;
+  }
+  return offset;
+}
+
+/* Add random bytes to the unused portion of the payload, to foil attacks
+ * where the other side can predict all of the bytes in the payload and thus
+ * compute the authenticated SENDME cells without seeing the traffic. See
+ * proposal 289. */
+static void
+pad_cell_payload(uint8_t *cell_payload, size_t data_len)
+{
+  size_t pad_offset, pad_len;
+
+  tor_assert(cell_payload);
+
+  pad_offset = get_pad_cell_offset(data_len);
+  if (pad_offset == 0) {
+    /* We can't add padding so we are done. */
+    return;
+  }
+
+  /* Remember here that the cell_payload is the length of the header and
+   * payload size so we offset it using the full length of the cell. */
+  pad_len = CELL_PAYLOAD_SIZE - pad_offset;
+  crypto_fast_rng_getbytes(get_thread_fast_rng(),
+                           cell_payload + pad_offset, pad_len);
+}
+
 /** Make a relay cell out of <b>relay_command</b> and <b>payload</b>, and send
  * it onto the open circuit <b>circ</b>. <b>stream_id</b> is the ID on
  * <b>circ</b> for the stream that's sending the relay cell, or 0 if it's a
@@ -574,11 +636,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
   if (payload_len)
     memcpy(cell.payload+RELAY_HEADER_SIZE, payload, payload_len);
 
+  /* Add random padding to the cell if we can. */
+  pad_cell_payload(cell.payload, payload_len);
+
   log_debug(LD_OR,"delivering %d cell %s.", relay_command,
             cell_direction == CELL_DIRECTION_OUT ? "forward" : "backward");
 
-  if (relay_command == RELAY_COMMAND_DROP)
-    rep_hist_padding_count_write(PADDING_TYPE_DROP);
+  /* Tell circpad we're sending a relay cell */
+  circpad_deliver_sent_relay_cell_events(circ, relay_command);
 
   /* If we are sending an END cell and this circuit is used for a tunneled
    * directory request, advance its state. */
@@ -602,7 +667,9 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
        * one of them.  Don't worry about the conn protocol version:
        * append_cell_to_circuit_queue will fix it up. */
       cell.command = CELL_RELAY_EARLY;
-      --origin_circ->remaining_relay_early_cells;
+      /* If we're out of relay early cells, tell circpad */
+      if (--origin_circ->remaining_relay_early_cells == 0)
+        circpad_machine_event_circ_has_no_relay_early(origin_circ);
       log_debug(LD_OR, "Sending a RELAY_EARLY cell; %d remaining.",
                 (int)origin_circ->remaining_relay_early_cells);
       /* Memorize the command that is sent as RELAY_EARLY cell; helps debug
@@ -639,6 +706,14 @@ relay_send_command_from_edge_,(streamid_t stream_id, circuit_t *circ,
     circuit_mark_for_close(circ, END_CIRC_REASON_INTERNAL);
     return -1;
   }
+
+  /* If applicable, note the cell digest for the SENDME version 1 purpose if
+   * we need to. This call needs to be after the circuit_package_relay_cell()
+   * because the cell digest is set within that function. */
+  if (relay_command == RELAY_COMMAND_DATA) {
+    sendme_record_cell_digest_on_circ(circ, cpath_layer);
+  }
+
   return 0;
 }
 
@@ -792,7 +867,7 @@ connection_ap_process_end_not_open(
               ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+5));
           } else if (rh->length == 17 || rh->length == 21) {
             tor_addr_from_ipv6_bytes(&addr,
-                                (char*)(cell->payload+RELAY_HEADER_SIZE+1));
+                                     (cell->payload+RELAY_HEADER_SIZE+1));
             if (rh->length == 21)
               ttl = (int)ntohl(get_uint32(cell->payload+RELAY_HEADER_SIZE+17));
           }
@@ -1017,7 +1092,7 @@ connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
       return -1;
     if (get_uint8(payload + 4) != 6)
       return -1;
-    tor_addr_from_ipv6_bytes(addr_out, (char*)(payload + 5));
+    tor_addr_from_ipv6_bytes(addr_out, (payload + 5));
     bytes = ntohl(get_uint32(payload + 21));
     if (bytes <= INT32_MAX)
       *ttl_out = (int) bytes;
@@ -1090,7 +1165,7 @@ resolved_cell_parse(const cell_t *cell, const relay_header_t *rh,
       if (answer_len != 16)
         goto err;
       addr = tor_malloc_zero(sizeof(*addr));
-      tor_addr_from_ipv6_bytes(&addr->addr, (const char*) cp);
+      tor_addr_from_ipv6_bytes(&addr->addr, cp);
       cp += 16;
       addr->ttl = ntohl(get_uint32(cp));
       cp += 4;
@@ -1447,85 +1522,108 @@ relay_crypt_from_last_hop(origin_circuit_t *circ, crypt_path_t *layer_hint)
   return true;
 }
 
-/** An incoming relay cell has arrived on circuit <b>circ</b>. If
- * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
- * destined for <b>conn</b>.
+/** Process a SENDME cell that arrived on <b>circ</b>. If it is a stream level
+ * cell, it is destined for the given <b>conn</b>. If it is a circuit level
+ * cell, it is destined for the <b>layer_hint</b>. The <b>domain</b> is the
+ * logging domain that should be used.
  *
- * If <b>layer_hint</b> is defined, then we're the origin of the
- * circuit, and it specifies the hop that packaged <b>cell</b>.
- *
- * Return -reason if you want to warn and tear down the circuit, else 0.
- */
-STATIC int
-connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
-                                   edge_connection_t *conn,
-                                   crypt_path_t *layer_hint)
+ * Return 0 if everything went well or a negative value representing a circuit
+ * end reason on error for which the caller is responsible for closing it. */
+static int
+process_sendme_cell(const relay_header_t *rh, const cell_t *cell,
+                    circuit_t *circ, edge_connection_t *conn,
+                    crypt_path_t *layer_hint, int domain)
 {
-  static int num_seen=0;
-  relay_header_t rh;
-  unsigned domain = layer_hint?LD_APP:LD_EXIT;
-  int reason;
-  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
-                            * that's in the EXIT_CONN_STATE_RESOLVING
-                            * or EXIT_CONN_STATE_CONNECTING states. */
-
-  tor_assert(cell);
-  tor_assert(circ);
+  int ret;
 
-  relay_header_unpack(&rh, cell->payload);
-//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
-  num_seen++;
-  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
-            num_seen, rh.command, rh.stream_id);
+  tor_assert(rh);
 
-  if (rh.length > RELAY_PAYLOAD_SIZE) {
-    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
-           "Relay cell length field too long. Closing circuit.");
-    return - END_CIRC_REASON_TORPROTOCOL;
+  if (!rh->stream_id) {
+    /* Circuit level SENDME cell. */
+    ret = sendme_process_circuit_level(layer_hint, circ,
+                                       cell->payload + RELAY_HEADER_SIZE,
+                                       rh->length);
+    if (ret < 0) {
+      return ret;
+    }
+    /* Resume reading on any streams now that we've processed a valid
+     * SENDME cell that updated our package window. */
+    circuit_resume_edge_reading(circ, layer_hint);
+    /* We are done, the rest of the code is for the stream level. */
+    return 0;
   }
 
-  if (rh.stream_id == 0) {
-    switch (rh.command) {
-      case RELAY_COMMAND_BEGIN:
-      case RELAY_COMMAND_CONNECTED:
-      case RELAY_COMMAND_END:
-      case RELAY_COMMAND_RESOLVE:
-      case RELAY_COMMAND_RESOLVED:
-      case RELAY_COMMAND_BEGIN_DIR:
-        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
-               "stream_id. Dropping.", (int)rh.command);
-        return 0;
-      default:
-        ;
+  /* No connection, might be half edge state. We are done if so. */
+  if (!conn) {
+    if (CIRCUIT_IS_ORIGIN(circ)) {
+      origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+      if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
+                                               rh->stream_id)) {
+        circuit_read_valid_data(ocirc, rh->length);
+        log_info(domain, "Sendme cell on circ %u valid on half-closed "
+                         "stream id %d",
+                 ocirc->global_identifier, rh->stream_id);
+      }
     }
+
+    log_info(domain, "SENDME cell dropped, unknown stream (streamid %d).",
+             rh->stream_id);
+    return 0;
   }
 
-  /* either conn is NULL, in which case we've got a control cell, or else
-   * conn points to the recognized stream. */
+  /* Stream level SENDME cell. */
+  ret = sendme_process_stream_level(conn, circ, rh->length);
+  if (ret < 0) {
+    /* Means we need to close the circuit with reason ret. */
+    return ret;
+  }
 
-  if (conn && !connection_state_is_open(TO_CONN(conn))) {
-    if (conn->base_.type == CONN_TYPE_EXIT &&
-        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
-         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
-        rh.command == RELAY_COMMAND_DATA) {
-      /* Allow DATA cells to be delivered to an exit node in state
-       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
-       * This speeds up HTTP, for example. */
-      optimistic_data = 1;
-    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
-      log_warn(LD_BUG, "Somehow I had a connection that matched a "
-               "data cell with stream ID 0.");
-    } else {
-      return connection_edge_process_relay_cell_not_open(
-               &rh, cell, circ, conn, layer_hint);
-    }
+  /* We've now processed properly a SENDME cell, all windows have been
+   * properly updated, we'll read on the edge connection to see if we can
+   * get data out towards the end point (Exit or client) since we are now
+   * allowed to deliver more cells. */
+
+  if (circuit_queue_streams_are_blocked(circ)) {
+    /* Still waiting for queue to flush; don't touch conn */
+    return 0;
+  }
+  connection_start_reading(TO_CONN(conn));
+  /* handle whatever might still be on the inbuf */
+  if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
+    /* (We already sent an end cell if possible) */
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
   }
+  return 0;
+}
 
-  switch (rh.command) {
-    case RELAY_COMMAND_DROP:
-      rep_hist_padding_count_read(PADDING_TYPE_DROP);
-//      log_info(domain,"Got a relay-level padding cell. Dropping.");
-      return 0;
+/** A helper for connection_edge_process_relay_cell(): Actually handles the
+ *  cell that we received on the connection.
+ *
+ *  The arguments are the same as in the parent function
+ *  connection_edge_process_relay_cell(), plus the relay header <b>rh</b> as
+ *  unpacked by the parent function, and <b>optimistic_data</b> as set by the
+ *  parent function.
+ */
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data)
+{
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int reason;
+
+  tor_assert(rh);
+
+  /* First pass the cell to the circuit padding subsystem, in case it's a
+   * padding cell or circuit that should be handled there. */
+  if (circpad_check_received_cell(cell, circ, layer_hint, rh) == 0) {
+    log_debug(domain, "Cell handled as circuit padding");
+    return 0;
+  }
+
+  /* Now handle all the other commands */
+  switch (rh->command) {
     case RELAY_COMMAND_BEGIN:
     case RELAY_COMMAND_BEGIN_DIR:
       if (layer_hint &&
@@ -1546,7 +1644,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                "Begin cell for known stream. Dropping.");
         return 0;
       }
-      if (rh.command == RELAY_COMMAND_BEGIN_DIR &&
+      if (rh->command == RELAY_COMMAND_BEGIN_DIR &&
           circ->purpose != CIRCUIT_PURPOSE_S_REND_JOINED) {
         /* Assign this circuit and its app-ward OR connection a unique ID,
          * so that we can measure download times. The local edge and dir
@@ -1559,24 +1657,21 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       return connection_exit_begin_conn(cell, circ);
     case RELAY_COMMAND_DATA:
       ++stats_n_data_cells_received;
-      if (( layer_hint && --layer_hint->deliver_window < 0) ||
-          (!layer_hint && --circ->deliver_window < 0)) {
+
+      /* Update our circuit-level deliver window that we received a DATA cell.
+       * If the deliver window goes below 0, we end the circuit and stream due
+       * to a protocol failure. */
+      if (sendme_circuit_data_received(circ, layer_hint) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) circ deliver_window below 0. Killing.");
-        if (conn) {
-          /* XXXX Do we actually need to do this?  Will killing the circuit
-           * not send an END and mark the stream for close as appropriate? */
-          connection_edge_end(conn, END_STREAM_REASON_TORPROTOCOL);
-          connection_mark_for_close(TO_CONN(conn));
-        }
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
-      log_debug(domain,"circ deliver_window now %d.", layer_hint ?
-                layer_hint->deliver_window : circ->deliver_window);
 
-      circuit_consider_sending_sendme(circ, layer_hint);
+      /* Consider sending a circuit-level SENDME cell. */
+      sendme_circuit_consider_sending(circ, layer_hint);
 
-      if (rh.stream_id == 0) {
+      if (rh->stream_id == 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay data cell with zero "
                "stream_id. Dropping.");
         return 0;
@@ -1584,32 +1679,37 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (connection_half_edge_is_valid_data(ocirc->half_streams,
-                                                 rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
+                                                 rh->stream_id)) {
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "data cell on circ %u valid on half-closed "
-                     "stream id %d", ocirc->global_identifier, rh.stream_id);
+                     "stream id %d", ocirc->global_identifier, rh->stream_id);
           }
         }
 
         log_info(domain,"data cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
+                 rh->stream_id);
         return 0;
       }
 
-      if (--conn->deliver_window < 0) { /* is it below 0 after decrement? */
+      /* Update our stream-level deliver window that we just received a DATA
+       * cell. Going below 0 means we have a protocol level error so the
+       * stream and circuit are closed. */
+
+      if (sendme_stream_data_received(conn) < 0) {
         log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
                "(relay data) conn deliver_window below 0. Killing.");
+        connection_edge_end_close(conn, END_STREAM_REASON_TORPROTOCOL);
         return -END_CIRC_REASON_TORPROTOCOL;
       }
       /* Total all valid application bytes delivered */
-      if (CIRCUIT_IS_ORIGIN(circ) && rh.length > 0) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+      if (CIRCUIT_IS_ORIGIN(circ) && rh->length > 0) {
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
 
-      stats_n_data_bytes_received += rh.length;
+      stats_n_data_bytes_received += rh->length;
       connection_buf_add((char*)(cell->payload + RELAY_HEADER_SIZE),
-                              rh.length, TO_CONN(conn));
+                              rh->length, TO_CONN(conn));
 
 #ifdef MEASUREMENTS_21206
       /* Count number of RELAY_DATA cells received on a linked directory
@@ -1625,26 +1725,26 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         /* Only send a SENDME if we're not getting optimistic data; otherwise
          * a SENDME could arrive before the CONNECTED.
          */
-        connection_edge_consider_sending_sendme(conn);
+        sendme_connection_edge_consider_sending(conn);
       }
 
       return 0;
     case RELAY_COMMAND_END:
-      reason = rh.length > 0 ?
+      reason = rh->length > 0 ?
         get_uint8(cell->payload+RELAY_HEADER_SIZE) : END_STREAM_REASON_MISC;
       if (!conn) {
         if (CIRCUIT_IS_ORIGIN(circ)) {
           origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
           if (relay_crypt_from_last_hop(ocirc, layer_hint) &&
               connection_half_edge_is_valid_end(ocirc->half_streams,
-                                                rh.stream_id)) {
+                                                rh->stream_id)) {
 
-            circuit_read_valid_data(ocirc, rh.length);
+            circuit_read_valid_data(ocirc, rh->length);
             log_info(domain,
                      "end cell (%s) on circ %u valid on half-closed "
                      "stream id %d",
                      stream_end_reason_to_string(reason),
-                     ocirc->global_identifier, rh.stream_id);
+                     ocirc->global_identifier, rh->stream_id);
             return 0;
           }
         }
@@ -1676,7 +1776,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
 
         /* Total all valid application bytes delivered */
         if (CIRCUIT_IS_ORIGIN(circ)) {
-          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+          circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
         }
       }
       return 0;
@@ -1684,7 +1784,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_EXTEND2: {
       static uint64_t total_n_extend=0, total_nonearly=0;
       total_n_extend++;
-      if (rh.stream_id) {
+      if (rh->stream_id) {
         log_fn(LOG_PROTOCOL_WARN, domain,
                "'extend' cell received for non-zero stream. Dropping.");
         return 0;
@@ -1725,9 +1825,9 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_debug(domain,"Got an extended cell! Yay.");
       {
         extended_cell_t extended_cell;
-        if (extended_cell_parse(&extended_cell, rh.command,
+        if (extended_cell_parse(&extended_cell, rh->command,
                         (const uint8_t*)cell->payload+RELAY_HEADER_SIZE,
-                        rh.length)<0) {
+                        rh->length)<0) {
           log_warn(LD_PROTOCOL,
                    "Can't parse EXTENDED cell; killing circuit.");
           return -END_CIRC_REASON_TORPROTOCOL;
@@ -1745,7 +1845,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       }
       /* Total all valid bytes delivered. */
       if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh.length);
+        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), rh->length);
       }
       return 0;
     case RELAY_COMMAND_TRUNCATE:
@@ -1789,7 +1889,7 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
        * circuit is being torn down anyway, though.  */
       if (CIRCUIT_IS_ORIGIN(circ)) {
         circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
+                                rh->length);
       }
       circuit_truncated(TO_ORIGIN_CIRCUIT(circ),
                         get_uint8(cell->payload + RELAY_HEADER_SIZE));
@@ -1804,11 +1904,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       if (CIRCUIT_IS_ORIGIN(circ)) {
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (connection_half_edge_is_valid_connected(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "connected cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1816,102 +1916,10 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
       log_info(domain,
                "'connected' received on circid %u for streamid %d, "
                "no conn attached anymore. Ignoring.",
-               (unsigned)circ->n_circ_id, rh.stream_id);
+               (unsigned)circ->n_circ_id, rh->stream_id);
       return 0;
     case RELAY_COMMAND_SENDME:
-      if (!rh.stream_id) {
-        if (layer_hint) {
-          if (layer_hint->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from exit relay. "
-                   "Closing circ.");
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          layer_hint->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,"circ-level sendme at origin, packagewindow %d.",
-                    layer_hint->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-
-          /* We count circuit-level sendme's as valid delivered data because
-           * they are rate limited.
-           */
-          if (CIRCUIT_IS_ORIGIN(circ)) {
-            circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                    rh.length);
-          }
-
-        } else {
-          if (circ->package_window + CIRCWINDOW_INCREMENT >
-                CIRCWINDOW_START_MAX) {
-            static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
-            log_fn_ratelim(&client_warn_ratelim,LOG_PROTOCOL_WARN, LD_PROTOCOL,
-                   "Unexpected sendme cell from client. "
-                   "Closing circ (window %d).",
-                   circ->package_window);
-            return -END_CIRC_REASON_TORPROTOCOL;
-          }
-          circ->package_window += CIRCWINDOW_INCREMENT;
-          log_debug(LD_APP,
-                    "circ-level sendme at non-origin, packagewindow %d.",
-                    circ->package_window);
-          circuit_resume_edge_reading(circ, layer_hint);
-        }
-        return 0;
-      }
-      if (!conn) {
-        if (CIRCUIT_IS_ORIGIN(circ)) {
-          origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-          if (connection_half_edge_is_valid_sendme(ocirc->half_streams,
-                                                   rh.stream_id)) {
-            circuit_read_valid_data(ocirc, rh.length);
-            log_info(domain,
-                    "sendme cell on circ %u valid on half-closed "
-                    "stream id %d", ocirc->global_identifier, rh.stream_id);
-          }
-        }
-
-        log_info(domain,"sendme cell dropped, unknown stream (streamid %d).",
-                 rh.stream_id);
-        return 0;
-      }
-
-      /* Don't allow the other endpoint to request more than our maximum
-       * (i.e. initial) stream SENDME window worth of data. Well-behaved
-       * stock clients will not request more than this max (as per the check
-       * in the while loop of connection_edge_consider_sending_sendme()).
-       */
-      if (conn->package_window + STREAMWINDOW_INCREMENT >
-          STREAMWINDOW_START_MAX) {
-        static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
-        log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
-               "Unexpected stream sendme cell. Closing circ (window %d).",
-               conn->package_window);
-        return -END_CIRC_REASON_TORPROTOCOL;
-      }
-
-      /* At this point, the stream sendme is valid */
-      if (CIRCUIT_IS_ORIGIN(circ)) {
-        circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ),
-                                rh.length);
-      }
-
-      conn->package_window += STREAMWINDOW_INCREMENT;
-      log_debug(domain,"stream-level sendme, packagewindow now %d.",
-                conn->package_window);
-      if (circuit_queue_streams_are_blocked(circ)) {
-        /* Still waiting for queue to flush; don't touch conn */
-        return 0;
-      }
-      connection_start_reading(TO_CONN(conn));
-      /* handle whatever might still be on the inbuf */
-      if (connection_edge_package_raw_inbuf(conn, 1, NULL) < 0) {
-        /* (We already sent an end cell if possible) */
-        connection_mark_for_close(TO_CONN(conn));
-        return 0;
-      }
-      return 0;
+      return process_sendme_cell(rh, cell, circ, conn, layer_hint, domain);
     case RELAY_COMMAND_RESOLVE:
       if (layer_hint) {
         log_fn(LOG_PROTOCOL_WARN, LD_APP,
@@ -1940,11 +1948,11 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
         origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
         if (relay_crypt_from_last_hop(ocirc, layer_hint) &&
             connection_half_edge_is_valid_resolved(ocirc->half_streams,
-                                                    rh.stream_id)) {
-          circuit_read_valid_data(ocirc, rh.length);
+                                                    rh->stream_id)) {
+          circuit_read_valid_data(ocirc, rh->length);
           log_info(domain,
                    "resolved cell on circ %u valid on half-closed "
-                   "stream id %d", ocirc->global_identifier, rh.stream_id);
+                   "stream id %d", ocirc->global_identifier, rh->stream_id);
           return 0;
         }
       }
@@ -1962,17 +1970,96 @@ connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
     case RELAY_COMMAND_INTRO_ESTABLISHED:
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       rend_process_relay_cell(circ, layer_hint,
-                              rh.command, rh.length,
+                              rh->command, rh->length,
                               cell->payload+RELAY_HEADER_SIZE);
       return 0;
   }
   log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
          "Received unknown relay command %d. Perhaps the other side is using "
          "a newer version of Tor? Dropping.",
-         rh.command);
+         rh->command);
   return 0; /* for forward compatibility, don't kill the circuit */
 }
 
+/** An incoming relay cell has arrived on circuit <b>circ</b>. If
+ * <b>conn</b> is NULL this is a control cell, else <b>cell</b> is
+ * destined for <b>conn</b>.
+ *
+ * If <b>layer_hint</b> is defined, then we're the origin of the
+ * circuit, and it specifies the hop that packaged <b>cell</b>.
+ *
+ * Return -reason if you want to warn and tear down the circuit, else 0.
+ */
+STATIC int
+connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
+                                   edge_connection_t *conn,
+                                   crypt_path_t *layer_hint)
+{
+  static int num_seen=0;
+  relay_header_t rh;
+  unsigned domain = layer_hint?LD_APP:LD_EXIT;
+  int optimistic_data = 0; /* Set to 1 if we receive data on a stream
+                            * that's in the EXIT_CONN_STATE_RESOLVING
+                            * or EXIT_CONN_STATE_CONNECTING states. */
+
+  tor_assert(cell);
+  tor_assert(circ);
+
+  relay_header_unpack(&rh, cell->payload);
+//  log_fn(LOG_DEBUG,"command %d stream %d", rh.command, rh.stream_id);
+  num_seen++;
+  log_debug(domain, "Now seen %d relay cells here (command %d, stream %d).",
+            num_seen, rh.command, rh.stream_id);
+
+  if (rh.length > RELAY_PAYLOAD_SIZE) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Relay cell length field too long. Closing circuit.");
+    return - END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  if (rh.stream_id == 0) {
+    switch (rh.command) {
+      case RELAY_COMMAND_BEGIN:
+      case RELAY_COMMAND_CONNECTED:
+      case RELAY_COMMAND_END:
+      case RELAY_COMMAND_RESOLVE:
+      case RELAY_COMMAND_RESOLVED:
+      case RELAY_COMMAND_BEGIN_DIR:
+        log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Relay command %d with zero "
+               "stream_id. Dropping.", (int)rh.command);
+        return 0;
+      default:
+        ;
+    }
+  }
+
+  /* Tell circpad that we've received a recognized cell */
+  circpad_deliver_recognized_relay_cell_events(circ, rh.command, layer_hint);
+
+  /* either conn is NULL, in which case we've got a control cell, or else
+   * conn points to the recognized stream. */
+  if (conn && !connection_state_is_open(TO_CONN(conn))) {
+    if (conn->base_.type == CONN_TYPE_EXIT &&
+        (conn->base_.state == EXIT_CONN_STATE_CONNECTING ||
+         conn->base_.state == EXIT_CONN_STATE_RESOLVING) &&
+        rh.command == RELAY_COMMAND_DATA) {
+      /* Allow DATA cells to be delivered to an exit node in state
+       * EXIT_CONN_STATE_CONNECTING or EXIT_CONN_STATE_RESOLVING.
+       * This speeds up HTTP, for example. */
+      optimistic_data = 1;
+    } else if (rh.stream_id == 0 && rh.command == RELAY_COMMAND_DATA) {
+      log_warn(LD_BUG, "Somehow I had a connection that matched a "
+               "data cell with stream ID 0.");
+    } else {
+      return connection_edge_process_relay_cell_not_open(
+               &rh, cell, circ, conn, layer_hint);
+    }
+  }
+
+  return handle_relay_cell_command(cell, circ, conn, layer_hint,
+                              &rh, optimistic_data);
+}
+
 /** How many relay_data cells have we built, ever? */
 uint64_t stats_n_data_cells_packaged = 0;
 /** How many bytes of data have we put in relay_data cells have we built,
@@ -1986,6 +2073,84 @@ uint64_t stats_n_data_cells_received = 0;
  * ever received were completely full of data. */
 uint64_t stats_n_data_bytes_received = 0;
 
+/**
+ * Called when initializing a circuit, or when we have reached the end of the
+ * window in which we need to send some randomness so that incoming sendme
+ * cells will be unpredictable.  Resets the flags and picks a new window.
+ */
+void
+circuit_reset_sendme_randomness(circuit_t *circ)
+{
+  circ->have_sent_sufficiently_random_cell = 0;
+  circ->send_randomness_after_n_cells = CIRCWINDOW_INCREMENT / 2 +
+    crypto_fast_rng_get_uint(get_thread_fast_rng(), CIRCWINDOW_INCREMENT / 2);
+}
+
+/**
+ * Any relay data payload containing fewer than this many real bytes is
+ * considered to have enough randomness to.
+ **/
+#define RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES \
+  (RELAY_PAYLOAD_SIZE - CELL_PADDING_GAP - 16)
+
+/**
+ * Helper. Return the number of bytes that should be put into a cell from a
+ * given edge connection on which <b>n_available</b> bytes are available.
+ */
+STATIC size_t
+connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                           int package_partial,
+                                           circuit_t *on_circuit)
+{
+  if (!n_available)
+    return 0;
+
+  /* Do we need to force this payload to have space for randomness? */
+  const bool force_random_bytes =
+    (on_circuit->send_randomness_after_n_cells == 0) &&
+    (! on_circuit->have_sent_sufficiently_random_cell);
+
+  /* At most how much would we like to send in this cell? */
+  size_t target_length;
+  if (force_random_bytes) {
+    target_length = RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES;
+  } else {
+    target_length = RELAY_PAYLOAD_SIZE;
+  }
+
+  /* Decide how many bytes we will actually put into this cell. */
+  size_t package_length;
+  if (n_available >= target_length) { /* A full payload is available. */
+    package_length = target_length;
+  } else { /* not a full payload available */
+    if (package_partial)
+      package_length = n_available; /* just take whatever's available now */
+    else
+      return 0; /* nothing to do until we have a full payload */
+  }
+
+  /* If we reach this point, we will be definitely sending the cell. */
+  tor_assert_nonfatal(package_length > 0);
+
+  if (package_length <= RELAY_PAYLOAD_LENGTH_FOR_RANDOM_SENDMES) {
+    /* This cell will have enough randomness in the padding to make a future
+     * sendme cell unpredictable. */
+    on_circuit->have_sent_sufficiently_random_cell = 1;
+  }
+
+  if (on_circuit->send_randomness_after_n_cells == 0) {
+    /* Either this cell, or some previous cell, had enough padding to
+     * ensure sendme unpredictability. */
+    tor_assert_nonfatal(on_circuit->have_sent_sufficiently_random_cell);
+    /* Pick a new interval in which we need to send randomness. */
+    circuit_reset_sendme_randomness(on_circuit);
+  }
+
+  --on_circuit->send_randomness_after_n_cells;
+
+  return package_length;
+}
+
 /** If <b>conn</b> has an entire relay payload of bytes on its inbuf (or
  * <b>package_partial</b> is true), and the appropriate package windows aren't
  * empty, grab a cell and send it down the circuit.
@@ -2058,17 +2223,14 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     bytes_to_process = connection_get_inbuf_len(TO_CONN(conn));
   }
 
-  if (!bytes_to_process)
+  length = connection_edge_get_inbuf_bytes_to_package(bytes_to_process,
+                                                      package_partial, circ);
+  if (!length)
     return 0;
 
-  if (!package_partial && bytes_to_process < RELAY_PAYLOAD_SIZE)
-    return 0;
+  /* If we reach this point, we will definitely be packaging bytes into
+   * a cell. */
 
-  if (bytes_to_process > RELAY_PAYLOAD_SIZE) {
-    length = RELAY_PAYLOAD_SIZE;
-  } else {
-    length = bytes_to_process;
-  }
   stats_n_data_bytes_packaged += length;
   stats_n_data_cells_packaged += 1;
 
@@ -2103,15 +2265,17 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
     return 0;
   }
 
-  if (!cpath_layer) { /* non-rendezvous exit */
-    tor_assert(circ->package_window > 0);
-    circ->package_window--;
-  } else { /* we're an AP, or an exit on a rendezvous circ */
-    tor_assert(cpath_layer->package_window > 0);
-    cpath_layer->package_window--;
+  /* Handle the circuit-level SENDME package window. */
+  if (sendme_note_circuit_data_packaged(circ, cpath_layer) < 0) {
+    /* Package window has gone under 0. Protocol issue. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Circuit package window is below 0. Closing circuit.");
+    conn->end_reason = END_STREAM_REASON_TORPROTOCOL;
+    return -1;
   }
 
-  if (--conn->package_window <= 0) { /* is it 0 after decrement? */
+  /* Handle the stream-level SENDME package window. */
+  if (sendme_note_stream_data_packaged(conn) < 0) {
     connection_stop_reading(TO_CONN(conn));
     log_debug(domain,"conn->package_window reached 0.");
     circuit_consider_stop_edge_reading(circ, cpath_layer);
@@ -2129,42 +2293,6 @@ connection_edge_package_raw_inbuf(edge_connection_t *conn, int package_partial,
   goto repeat_connection_edge_package_raw_inbuf;
 }
 
-/** Called when we've just received a relay data cell, when
- * we've just finished flushing all bytes to stream <b>conn</b>,
- * or when we've flushed *some* bytes to the stream <b>conn</b>.
- *
- * If conn->outbuf is not too full, and our deliver window is
- * low, send back a suitable number of stream-level sendme cells.
- */
-void
-connection_edge_consider_sending_sendme(edge_connection_t *conn)
-{
-  circuit_t *circ;
-
-  if (connection_outbuf_too_full(TO_CONN(conn)))
-    return;
-
-  circ = circuit_get_by_edge_conn(conn);
-  if (!circ) {
-    /* this can legitimately happen if the destroy has already
-     * arrived and torn down the circuit */
-    log_info(LD_APP,"No circuit associated with conn. Skipping.");
-    return;
-  }
-
-  while (conn->deliver_window <= STREAMWINDOW_START - STREAMWINDOW_INCREMENT) {
-    log_debug(conn->base_.type == CONN_TYPE_AP ?LD_APP:LD_EXIT,
-              "Outbuf %d, Queuing stream sendme.",
-              (int)conn->base_.outbuf_flushlen);
-    conn->deliver_window += STREAMWINDOW_INCREMENT;
-    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
-                                     NULL, 0) < 0) {
-      log_warn(LD_APP,"connection_edge_send_command failed. Skipping.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The circuit <b>circ</b> has received a circuit-level sendme
  * (on hop <b>layer_hint</b>, if we're the OP). Go through all the
  * attached streams and let them resume reading and packaging, if
@@ -2187,12 +2315,6 @@ circuit_resume_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
                                        circ, layer_hint);
 }
 
-void
-stream_choice_seed_weak_rng(void)
-{
-  crypto_seed_weak_rng(&stream_choice_rng);
-}
-
 /** A helper function for circuit_resume_edge_reading() above.
  * The arguments are the same, except that <b>conn</b> is the head
  * of a linked list of edge streams that should each be considered.
@@ -2244,7 +2366,8 @@ circuit_resume_edge_reading_helper(edge_connection_t *first_conn,
     int num_streams = 0;
     for (conn = first_conn; conn; conn = conn->next_stream) {
       num_streams++;
-      if (tor_weak_random_one_in_n(&stream_choice_rng, num_streams)) {
+
+      if (crypto_fast_rng_one_in_n(get_thread_fast_rng(), num_streams)) {
         chosen_stream = conn;
       }
       /* Invariant: chosen_stream has been chosen uniformly at random from
@@ -2386,33 +2509,6 @@ circuit_consider_stop_edge_reading(circuit_t *circ, crypt_path_t *layer_hint)
   return 0;
 }
 
-/** Check if the deliver_window for circuit <b>circ</b> (at hop
- * <b>layer_hint</b> if it's defined) is low enough that we should
- * send a circuit-level sendme back down the circuit. If so, send
- * enough sendmes that the window would be overfull if we sent any
- * more.
- */
-static void
-circuit_consider_sending_sendme(circuit_t *circ, crypt_path_t *layer_hint)
-{
-//  log_fn(LOG_INFO,"Considering: layer_hint is %s",
-//         layer_hint ? "defined" : "null");
-  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
-          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
-    log_debug(LD_CIRC,"Queuing circuit sendme.");
-    if (layer_hint)
-      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
-    else
-      circ->deliver_window += CIRCWINDOW_INCREMENT;
-    if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
-                                     NULL, 0, layer_hint) < 0) {
-      log_warn(LD_CIRC,
-               "relay_send_command_from_edge failed. Circuit's closed.");
-      return; /* the circuit's closed, don't continue */
-    }
-  }
-}
-
 /** The total number of cells we have allocated. */
 static size_t total_cells_allocated = 0;
 
@@ -2787,7 +2883,7 @@ set_streams_blocked_on_circ(circuit_t *circ, channel_t *chan,
 }
 
 /** Extract the command from a packed cell. */
-static uint8_t
+uint8_t
 packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids)
 {
   if (wide_circ_ids) {
@@ -3142,7 +3238,7 @@ decode_address_from_payload(tor_addr_t *addr_out, const uint8_t *payload,
   case RESOLVED_TYPE_IPV6:
     if (payload[1] != 16)
       return NULL;
-    tor_addr_from_ipv6_bytes(addr_out, (char*)(payload+2));
+    tor_addr_from_ipv6_bytes(addr_out, (payload+2));
     break;
   default:
     tor_addr_make_unspec(addr_out);
diff --git a/src/core/or/relay.h b/src/core/or/relay.h
index 7cc3c43e43..31bed0e01b 100644
--- a/src/core/or/relay.h
+++ b/src/core/or/relay.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,6 +42,7 @@ int connection_edge_package_raw_inbuf(edge_connection_t *conn,
                                       int package_partial,
                                       int *max_cells);
 void connection_edge_consider_sending_sendme(edge_connection_t *conn);
+void circuit_reset_sendme_randomness(circuit_t *circ);
 
 extern uint64_t stats_n_data_cells_packaged;
 extern uint64_t stats_n_data_bytes_packaged;
@@ -78,6 +79,11 @@ void destroy_cell_queue_append(destroy_cell_queue_t *queue,
 void channel_unlink_all_circuits(channel_t *chan, smartlist_t *detached_out);
 MOCK_DECL(int, channel_flush_from_first_active_circuit,
           (channel_t *chan, int max));
+MOCK_DECL(int, circuit_package_relay_cell, (cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno));
+
 void update_circuit_on_cmux_(circuit_t *circ, cell_direction_t direction,
                              const char *file, int lineno);
 #define update_circuit_on_cmux(circ, direction) \
@@ -89,15 +95,19 @@ const uint8_t *decode_address_from_payload(tor_addr_t *addr_out,
                                         int payload_len);
 void circuit_clear_cell_queue(circuit_t *circ, channel_t *chan);
 
-void stream_choice_seed_weak_rng(void);
-
 circid_t packed_cell_get_circid(const packed_cell_t *cell, int wide_circ_ids);
+uint8_t packed_cell_get_command(const packed_cell_t *cell, int wide_circ_ids);
 
 #ifdef RELAY_PRIVATE
+STATIC int
+handle_relay_cell_command(cell_t *cell, circuit_t *circ,
+                     edge_connection_t *conn, crypt_path_t *layer_hint,
+                     relay_header_t *rh, int optimistic_data);
+
 STATIC int connected_cell_parse(const relay_header_t *rh, const cell_t *cell,
                          tor_addr_t *addr_out, int *ttl_out);
 /** An address-and-ttl tuple as yielded by resolved_cell_parse */
-typedef struct address_ttl_s {
+typedef struct address_ttl_t {
   tor_addr_t addr;
   char *hostname;
   int ttl;
@@ -117,8 +127,11 @@ STATIC int cell_queues_check_size(void);
 STATIC int connection_edge_process_relay_cell(cell_t *cell, circuit_t *circ,
                                    edge_connection_t *conn,
                                    crypt_path_t *layer_hint);
+STATIC size_t get_pad_cell_offset(size_t payload_len);
+STATIC size_t connection_edge_get_inbuf_bytes_to_package(size_t n_available,
+                                                      int package_partial,
+                                                      circuit_t *on_circuit);
 
 #endif /* defined(RELAY_PRIVATE) */
 
 #endif /* !defined(TOR_RELAY_H) */
-
diff --git a/src/core/or/relay_crypto_st.h b/src/core/or/relay_crypto_st.h
index dafce257c7..d92f486a90 100644
--- a/src/core/or/relay_crypto_st.h
+++ b/src/core/or/relay_crypto_st.h
@@ -1,13 +1,18 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file relay_crypto_st.h
+ * @brief Relay-cell encryption state structure.
+ **/
+
 #ifndef RELAY_CRYPTO_ST_H
 #define RELAY_CRYPTO_ST_H
 
-#define crypto_cipher_t aes_cnt_cipher
+#define crypto_cipher_t aes_cnt_cipher_t
 struct crypto_cipher_t;
 struct crypto_digest_t;
 
@@ -25,7 +30,9 @@ struct relay_crypto_t {
   /** Digest state for cells heading away from the OR at this step. */
   struct crypto_digest_t *b_digest;
 
+  /** Digest used for the next SENDME cell if any. */
+  uint8_t sendme_digest[DIGEST_LEN];
 };
 #undef crypto_cipher_t
 
-#endif
+#endif /* !defined(RELAY_CRYPTO_ST_H) */
diff --git a/src/core/or/scheduler.c b/src/core/or/scheduler.c
index 9f1a27d501..072d78128b 100644
--- a/src/core/or/scheduler.c
+++ b/src/core/or/scheduler.c
@@ -1,18 +1,17 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
 #include "app/config/config.h"
 
 #include "lib/evloop/compat_libevent.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #define SCHEDULER_KIST_PRIVATE
 #include "core/or/scheduler.h"
 #include "core/mainloop/mainloop.h"
-#include "lib/container/buffers.h"
-#define TOR_CHANNEL_INTERNAL_
+#include "lib/buf/buffers.h"
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/channeltls.h"
-#include "lib/evloop/compat_libevent.h"
 
 #include "core/or/or_connection_st.h"
 
@@ -43,7 +42,7 @@
  * circuit scheduler. It was supposed to prioritize circuits across many
  * channels, but wasn't effective. It is preserved in scheduler_vanilla.c.
  *
- * [0]: http://www.robgjansen.com/publications/kist-sec2014.pdf
+ * [0]: https://www.robgjansen.com/publications/kist-sec2014.pdf
  *
  * Then we actually got around to implementing KIST for real. We decided to
  * modularize the scheduler so new ones can be implemented. You can find KIST
@@ -267,7 +266,7 @@ select_scheduler(void)
           log_notice(LD_SCHED, "Scheduler type KIST has been disabled by "
                                "the consensus or no kernel support.");
         }
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
         log_info(LD_SCHED, "Scheduler type KIST not built in");
 #endif /* defined(HAVE_KIST_SUPPORT) */
         continue;
@@ -503,7 +502,12 @@ scheduler_free_all(void)
   the_scheduler = NULL;
 }
 
-/** Mark a channel as no longer ready to accept writes. */
+/** Mark a channel as no longer ready to accept writes.
+  *
+  * Possible state changes:
+  *  - SCHED_CHAN_PENDING -> SCHED_CHAN_WAITING_TO_WRITE
+  *  - SCHED_CHAN_WAITING_FOR_CELLS -> SCHED_CHAN_IDLE
+  */
 MOCK_IMPL(void,
 scheduler_channel_doesnt_want_writes,(channel_t *chan))
 {
@@ -514,31 +518,32 @@ scheduler_channel_doesnt_want_writes,(channel_t *chan))
     return;
   }
 
-  /* If it's already in pending, we can put it in waiting_to_write */
   if (chan->scheduler_state == SCHED_CHAN_PENDING) {
     /*
-     * It's in channels_pending, so it shouldn't be in any of
-     * the other lists.  It can't write any more, so it goes to
-     * channels_waiting_to_write.
+     * It has cells but no longer can write, so it becomes
+     * SCHED_CHAN_WAITING_TO_WRITE. It's in channels_pending, so we
+     * should remove it from the list.
      */
     smartlist_pqueue_remove(channels_pending,
                             scheduler_compare_channels,
                             offsetof(channel_t, sched_heap_idx),
                             chan);
     scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
-  } else {
+  } else if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
     /*
-     * It's not in pending, so it can't become waiting_to_write; it's
-     * either not in any of the lists (nothing to do) or it's already in
-     * waiting_for_cells (remove it, can't write any more).
+     * It does not have cells and no longer can write, so it becomes
+     * SCHED_CHAN_IDLE.
      */
-    if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
-      scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
-    }
+    scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
   }
 }
 
-/** Mark a channel as having waiting cells. */
+/** Mark a channel as having waiting cells.
+  *
+  * Possible state changes:
+  *  - SCHED_CHAN_WAITING_FOR_CELLS -> SCHED_CHAN_PENDING
+  *  - SCHED_CHAN_IDLE -> SCHED_CHAN_WAITING_TO_WRITE
+  */
 MOCK_IMPL(void,
 scheduler_channel_has_waiting_cells,(channel_t *chan))
 {
@@ -549,12 +554,11 @@ scheduler_channel_has_waiting_cells,(channel_t *chan))
     return;
   }
 
-  /* First, check if it's also writeable */
   if (chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS) {
     /*
-     * It's in channels_waiting_for_cells, so it shouldn't be in any of
-     * the other lists.  It has waiting cells now, so it goes to
-     * channels_pending.
+     * It is able to write and now has cells, so it becomes
+     * SCHED_CHAN_PENDING. It must be added to the channels_pending
+     * list.
      */
     scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
     if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
@@ -566,16 +570,12 @@ scheduler_channel_has_waiting_cells,(channel_t *chan))
     /* If we made a channel pending, we potentially have scheduling work to
      * do. */
     the_scheduler->schedule();
-  } else {
+  } else if (chan->scheduler_state == SCHED_CHAN_IDLE) {
     /*
-     * It's not in waiting_for_cells, so it can't become pending; it's
-     * either not in any of the lists (we add it to waiting_to_write)
-     * or it's already in waiting_to_write or pending (we do nothing)
+     * It is not able to write but now has cells, so it becomes
+     * SCHED_CHAN_WAITING_TO_WRITE.
      */
-    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE ||
-          chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
-    }
+    scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_TO_WRITE);
   }
 }
 
@@ -663,8 +663,12 @@ scheduler_release_channel,(channel_t *chan))
   scheduler_set_channel_state(chan, SCHED_CHAN_IDLE);
 }
 
-/** Mark a channel as ready to accept writes */
-
+/** Mark a channel as ready to accept writes.
+  * Possible state changes:
+  *
+  *  - SCHED_CHAN_WAITING_TO_WRITE -> SCHED_CHAN_PENDING
+  *  - SCHED_CHAN_IDLE -> SCHED_CHAN_WAITING_FOR_CELLS
+  */
 void
 scheduler_channel_wants_writes(channel_t *chan)
 {
@@ -675,10 +679,11 @@ scheduler_channel_wants_writes(channel_t *chan)
     return;
   }
 
-  /* If it's already in waiting_to_write, we can put it in pending */
   if (chan->scheduler_state == SCHED_CHAN_WAITING_TO_WRITE) {
     /*
-     * It can write now, so it goes to channels_pending.
+     * It has cells and can now write, so it becomes
+     * SCHED_CHAN_PENDING. It must be added to the channels_pending
+     * list.
      */
     scheduler_set_channel_state(chan, SCHED_CHAN_PENDING);
     if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
@@ -689,15 +694,12 @@ scheduler_channel_wants_writes(channel_t *chan)
     }
     /* We just made a channel pending, we have scheduling work to do. */
     the_scheduler->schedule();
-  } else {
+  } else if (chan->scheduler_state == SCHED_CHAN_IDLE) {
     /*
-     * It's not in SCHED_CHAN_WAITING_TO_WRITE, so it can't become pending;
-     * it's either idle and goes to WAITING_FOR_CELLS, or it's a no-op.
+     * It does not have cells but can now write, so it becomes
+     * SCHED_CHAN_WAITING_FOR_CELLS.
      */
-    if (!(chan->scheduler_state == SCHED_CHAN_WAITING_FOR_CELLS ||
-          chan->scheduler_state == SCHED_CHAN_PENDING)) {
-      scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
-    }
+    scheduler_set_channel_state(chan, SCHED_CHAN_WAITING_FOR_CELLS);
   }
 }
 
diff --git a/src/core/or/scheduler.h b/src/core/or/scheduler.h
index 843be2603c..82df2b0b0f 100644
--- a/src/core/or/scheduler.h
+++ b/src/core/or/scheduler.h
@@ -1,4 +1,4 @@
-/* * Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* * Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,7 +40,7 @@ typedef enum {
  * doesn't create any state for itself, thus it has nothing to free when Tor
  * is shutting down), then set that function pointer to NULL.
  */
-typedef struct scheduler_s {
+typedef struct scheduler_t {
   /* Scheduler type. This is used for logging when the scheduler is switched
    * during runtime. */
   scheduler_types_t type;
@@ -136,7 +136,9 @@ MOCK_DECL(void, scheduler_channel_has_waiting_cells, (channel_t *chan));
  * These functions are only visible to the scheduling system, the current
  * scheduler implementation, and tests.
  *****************************************************************************/
-#ifdef SCHEDULER_PRIVATE_
+#ifdef SCHEDULER_PRIVATE
+
+#include "ext/ht.h"
 
 /*********************************
  * Defined in scheduler.c
@@ -173,8 +175,8 @@ void scheduler_touch_channel(channel_t *chan);
 
 /* Socket table entry which holds information of a channel's socket and kernel
  * TCP information. Only used by KIST. */
-typedef struct socket_table_ent_s {
-  HT_ENTRY(socket_table_ent_s) node;
+typedef struct socket_table_ent_t {
+  HT_ENTRY(socket_table_ent_t) node;
   const channel_t *chan;
   /* Amount written this scheduling run */
   uint64_t written;
@@ -187,7 +189,7 @@ typedef struct socket_table_ent_s {
   uint32_t notsent;
 } socket_table_ent_t;
 
-typedef HT_HEAD(outbuf_table_s, outbuf_table_ent_s) outbuf_table_t;
+typedef HT_HEAD(outbuf_table_s, outbuf_table_ent_t) outbuf_table_t;
 
 MOCK_DECL(int, channel_should_write_to_kernel,
           (outbuf_table_t *table, channel_t *chan));
@@ -212,7 +214,6 @@ extern int32_t sched_run_interval;
 
 scheduler_t *get_vanilla_scheduler(void);
 
-#endif /* defined(SCHEDULER_PRIVATE_) */
+#endif /* defined(SCHEDULER_PRIVATE) */
 
 #endif /* !defined(TOR_SCHEDULER_H) */
-
diff --git a/src/core/or/scheduler_kist.c b/src/core/or/scheduler_kist.c
index 79ecb0bc7e..c73d768f88 100644
--- a/src/core/or/scheduler_kist.c
+++ b/src/core/or/scheduler_kist.c
@@ -1,17 +1,22 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file scheduler_kist.c
+ * @brief Implements the KIST cell scheduler.
+ **/
+
 #define SCHEDULER_KIST_PRIVATE
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #include "core/or/scheduler.h"
 #include "lib/math/fp.h"
 
@@ -46,22 +51,22 @@ socket_table_ent_eq(const socket_table_ent_t *a, const socket_table_ent_t *b)
   return a->chan == b->chan;
 }
 
-typedef HT_HEAD(socket_table_s, socket_table_ent_s) socket_table_t;
+typedef HT_HEAD(socket_table_s, socket_table_ent_t) socket_table_t;
 
 static socket_table_t socket_table = HT_INITIALIZER();
 
-HT_PROTOTYPE(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
-             socket_table_ent_eq)
-HT_GENERATE2(socket_table_s, socket_table_ent_s, node, socket_table_ent_hash,
-             socket_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+HT_PROTOTYPE(socket_table_s, socket_table_ent_t, node, socket_table_ent_hash,
+             socket_table_ent_eq);
+HT_GENERATE2(socket_table_s, socket_table_ent_t, node, socket_table_ent_hash,
+             socket_table_ent_eq, 0.6, tor_reallocarray, tor_free_);
 
 /* outbuf_table hash table stuff. The outbuf_table keeps track of which
  * channels have data sitting in their outbuf so the kist scheduler can force
  * a write from outbuf to kernel periodically during a run and at the end of a
  * run. */
 
-typedef struct outbuf_table_ent_s {
-  HT_ENTRY(outbuf_table_ent_s) node;
+typedef struct outbuf_table_ent_t {
+  HT_ENTRY(outbuf_table_ent_t) node;
   channel_t *chan;
 } outbuf_table_ent_t;
 
@@ -77,10 +82,10 @@ outbuf_table_ent_eq(const outbuf_table_ent_t *a, const outbuf_table_ent_t *b)
   return a->chan->global_identifier == b->chan->global_identifier;
 }
 
-HT_PROTOTYPE(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
-             outbuf_table_ent_eq)
-HT_GENERATE2(outbuf_table_s, outbuf_table_ent_s, node, outbuf_table_ent_hash,
-             outbuf_table_ent_eq, 0.6, tor_reallocarray, tor_free_)
+HT_PROTOTYPE(outbuf_table_s, outbuf_table_ent_t, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq);
+HT_GENERATE2(outbuf_table_s, outbuf_table_ent_t, node, outbuf_table_ent_hash,
+             outbuf_table_ent_eq, 0.6, tor_reallocarray, tor_free_);
 
 /*****************************************************************************
  * Other internal data
@@ -104,7 +109,7 @@ static unsigned int kist_lite_mode = 0;
  * changed and it doesn't recognized the values passed to the syscalls needed
  * by KIST. In that case, fallback to the naive approach. */
 static unsigned int kist_no_kernel_support = 0;
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 static unsigned int kist_lite_mode = 1;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -298,7 +303,7 @@ update_socket_info_impl, (socket_table_ent_t *ent))
   }
   return;
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
   goto fallback;
 #endif /* defined(HAVE_KIST_SUPPORT) */
 
@@ -458,6 +463,13 @@ MOCK_IMPL(void, channel_write_to_kernel, (channel_t *chan))
   log_debug(LD_SCHED, "Writing %lu bytes to kernel for chan %" PRIu64,
             (unsigned long)channel_outbuf_length(chan),
             chan->global_identifier);
+  /* Note that 'connection_handle_write()' may change the scheduler state of
+   * the channel during the scheduling loop with
+   * 'connection_or_flushed_some()' -> 'scheduler_channel_wants_writes()'.
+   * This side-effect will only occur if the channel is currently in the
+   * 'SCHED_CHAN_WAITING_TO_WRITE' or 'SCHED_CHAN_IDLE' states, which KIST
+   * rarely uses, so it should be fine unless KIST begins using these states
+   * in the future. */
   connection_handle_write(TO_CONN(BASE_CHAN_TO_TLS(chan)->conn), 0);
 }
 
@@ -724,7 +736,7 @@ kist_scheduler_run(void)
     SMARTLIST_FOREACH_BEGIN(to_readd, channel_t *, readd_chan) {
       scheduler_set_channel_state(readd_chan, SCHED_CHAN_PENDING);
       if (!smartlist_contains(cp, readd_chan)) {
-        if (!SCHED_BUG(chan->sched_heap_idx != -1, chan)) {
+        if (!SCHED_BUG(readd_chan->sched_heap_idx != -1, readd_chan)) {
           /* XXXX Note that the check above is in theory redundant with
            * the smartlist_contains check.  But let's make sure we're
            * not messing anything up, and leave them both for now. */
@@ -833,7 +845,7 @@ scheduler_can_use_kist(void)
   return run_interval > 0;
 }
 
-#else /* !(defined(HAVE_KIST_SUPPORT)) */
+#else /* !defined(HAVE_KIST_SUPPORT) */
 
 int
 scheduler_can_use_kist(void)
diff --git a/src/core/or/scheduler_vanilla.c b/src/core/or/scheduler_vanilla.c
index 33536ae04b..d862ff8710 100644
--- a/src/core/or/scheduler_vanilla.c
+++ b/src/core/or/scheduler_vanilla.c
@@ -1,11 +1,16 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file scheduler_vanilla.c
+ * @brief "Vanilla" (pre-KIST) cell scheduler code.
+ **/
+
 #include "core/or/or.h"
 #include "app/config/config.h"
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/channel.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #include "core/or/scheduler.h"
 
 /*****************************************************************************
@@ -172,4 +177,3 @@ get_vanilla_scheduler(void)
 {
   return &vanilla_scheduler;
 }
-
diff --git a/src/core/or/sendme.c b/src/core/or/sendme.c
new file mode 100644
index 0000000000..788f56088c
--- /dev/null
+++ b/src/core/or/sendme.c
@@ -0,0 +1,710 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.c
+ * \brief Code that is related to SENDME cells both in terms of
+ *        creating/parsing cells and handling the content.
+ */
+
+#define SENDME_PRIVATE
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/mainloop/connection.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+#include "feature/nodelist/networkstatus.h"
+#include "lib/ctime/di_ops.h"
+#include "trunnel/sendme_cell.h"
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * used when emitting a SENDME cell. */
+STATIC int
+get_emit_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_emit_min_version",
+                                 SENDME_EMIT_MIN_VERSION_DEFAULT,
+                                 SENDME_EMIT_MIN_VERSION_MIN,
+                                 SENDME_EMIT_MIN_VERSION_MAX);
+}
+
+/* Return the minimum version given by the consensus (if any) that should be
+ * accepted when receiving a SENDME cell. */
+STATIC int
+get_accept_min_version(void)
+{
+  return networkstatus_get_param(NULL, "sendme_accept_min_version",
+                                 SENDME_ACCEPT_MIN_VERSION_DEFAULT,
+                                 SENDME_ACCEPT_MIN_VERSION_MIN,
+                                 SENDME_ACCEPT_MIN_VERSION_MAX);
+}
+
+/* Pop the first cell digset on the given circuit from the SENDME last digests
+ * list. NULL is returned if the list is uninitialized or empty.
+ *
+ * The caller gets ownership of the returned digest thus is responsible for
+ * freeing the memory. */
+static uint8_t *
+pop_first_cell_digest(const circuit_t *circ)
+{
+  uint8_t *circ_digest;
+
+  tor_assert(circ);
+
+  if (circ->sendme_last_digests == NULL ||
+      smartlist_len(circ->sendme_last_digests) == 0) {
+    return NULL;
+  }
+
+  /* More cell digest than the SENDME window is never suppose to happen. The
+   * cell should have been rejected before reaching this point due to its
+   * package_window down to 0 leading to a circuit close. Scream loudly but
+   * still pop the element so we don't memory leak. */
+  tor_assert_nonfatal(smartlist_len(circ->sendme_last_digests) <=
+                      CIRCWINDOW_START_MAX / CIRCWINDOW_INCREMENT);
+
+  circ_digest = smartlist_get(circ->sendme_last_digests, 0);
+  smartlist_del_keeporder(circ->sendme_last_digests, 0);
+  return circ_digest;
+}
+
+/* Return true iff the given cell digest matches the first digest in the
+ * circuit sendme list. */
+static bool
+v1_digest_matches(const uint8_t *circ_digest, const uint8_t *cell_digest)
+{
+  tor_assert(circ_digest);
+  tor_assert(cell_digest);
+
+  /* Compare the digest with the one in the SENDME. This cell is invalid
+   * without a perfect match. */
+  if (tor_memneq(circ_digest, cell_digest, TRUNNEL_SENDME_V1_DIGEST_LEN)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "SENDME v1 cell digest do not match.");
+    return false;
+  }
+
+  /* Digests matches! */
+  return true;
+}
+
+/* Return true iff the given decoded SENDME version 1 cell is valid and
+ * matches the expected digest on the circuit.
+ *
+ * Validation is done by comparing the digest in the cell from the previous
+ * cell we saw which tells us that the other side has in fact seen that cell.
+ * See proposal 289 for more details. */
+static bool
+cell_v1_is_valid(const sendme_cell_t *cell, const uint8_t *circ_digest)
+{
+  tor_assert(cell);
+  tor_assert(circ_digest);
+
+  const uint8_t *cell_digest = sendme_cell_getconstarray_data_v1_digest(cell);
+  return v1_digest_matches(circ_digest, cell_digest);
+}
+
+/* Return true iff the given cell version can be handled or if the minimum
+ * accepted version from the consensus is known to us. */
+STATIC bool
+cell_version_can_be_handled(uint8_t cell_version)
+{
+  int accept_version = get_accept_min_version();
+
+  /* We will first check if the consensus minimum accepted version can be
+   * handled by us and if not, regardless of the cell version we got, we can't
+   * continue. */
+  if (accept_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Unable to accept SENDME version %u (from consensus). "
+           "We only support <= %u. Probably your tor is too old?",
+           accept_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  /* Then, is this version below the accepted version from the consensus? If
+   * yes, we must not handle it. */
+  if (cell_version < accept_version) {
+    log_info(LD_PROTOCOL, "Unacceptable SENDME version %u. Only "
+                          "accepting %u (from consensus). Closing circuit.",
+             cell_version, accept_version);
+    goto invalid;
+  }
+
+  /* Is this cell version supported by us? */
+  if (cell_version > SENDME_MAX_SUPPORTED_VERSION) {
+    log_info(LD_PROTOCOL, "SENDME cell version %u is not supported by us. "
+                          "We only support <= %u",
+             cell_version, SENDME_MAX_SUPPORTED_VERSION);
+    goto invalid;
+  }
+
+  return true;
+ invalid:
+  return false;
+}
+
+/* Return true iff the encoded SENDME cell in cell_payload of length
+ * cell_payload_len is valid. For each version:
+ *
+ *  0: No validation
+ *  1: Authenticated with last cell digest.
+ *
+ * This is the main critical function to make sure we can continue to
+ * send/recv cells on a circuit. If the SENDME is invalid, the circuit should
+ * be marked for close by the caller. */
+STATIC bool
+sendme_is_valid(const circuit_t *circ, const uint8_t *cell_payload,
+                size_t cell_payload_len)
+{
+  uint8_t cell_version;
+  uint8_t *circ_digest = NULL;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* An empty payload means version 0 so skip trunnel parsing. We won't be
+   * able to parse a 0 length buffer into a valid SENDME cell. */
+  if (cell_payload_len == 0) {
+    cell_version = 0;
+  } else {
+    /* First we'll decode the cell so we can get the version. */
+    if (sendme_cell_parse(&cell, cell_payload, cell_payload_len) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+             "Unparseable SENDME cell received. Closing circuit.");
+      goto invalid;
+    }
+    cell_version = sendme_cell_get_version(cell);
+  }
+
+  /* Validate that we can handle this cell version. */
+  if (!cell_version_can_be_handled(cell_version)) {
+    goto invalid;
+  }
+
+  /* Pop the first element that was added (FIFO). We do that regardless of the
+   * version so we don't accumulate on the circuit if v0 is used by the other
+   * end point. */
+  circ_digest = pop_first_cell_digest(circ);
+  if (circ_digest == NULL) {
+    /* We shouldn't have received a SENDME if we have no digests. Log at
+     * protocol warning because it can be tricked by sending many SENDMEs
+     * without prior data cell. */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "We received a SENDME but we have no cell digests to match. "
+           "Closing circuit.");
+    goto invalid;
+  }
+
+  /* Validate depending on the version now. */
+  switch (cell_version) {
+  case 0x01:
+    if (!cell_v1_is_valid(cell, circ_digest)) {
+      goto invalid;
+    }
+    break;
+  case 0x00:
+    /* Version 0, there is no work to be done on the payload so it is
+     * necessarily valid if we pass the version validation. */
+    break;
+  default:
+    log_warn(LD_PROTOCOL, "Unknown SENDME cell version %d received.",
+             cell_version);
+    tor_assert_nonfatal_unreached();
+    break;
+  }
+
+  /* Valid cell. */
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return true;
+ invalid:
+  sendme_cell_free(cell);
+  tor_free(circ_digest);
+  return false;
+}
+
+/* Build and encode a version 1 SENDME cell into payload, which must be at
+ * least of RELAY_PAYLOAD_SIZE bytes, using the digest for the cell data.
+ *
+ * Return the size in bytes of the encoded cell in payload. A negative value
+ * is returned on encoding failure. */
+STATIC ssize_t
+build_cell_payload_v1(const uint8_t *cell_digest, uint8_t *payload)
+{
+  ssize_t len = -1;
+  sendme_cell_t *cell = NULL;
+
+  tor_assert(cell_digest);
+  tor_assert(payload);
+
+  cell = sendme_cell_new();
+
+  /* Building a payload for version 1. */
+  sendme_cell_set_version(cell, 0x01);
+  /* Set the data length field for v1. */
+  sendme_cell_set_data_len(cell, TRUNNEL_SENDME_V1_DIGEST_LEN);
+
+  /* Copy the digest into the data payload. */
+  memcpy(sendme_cell_getarray_data_v1_digest(cell), cell_digest,
+         sendme_cell_get_data_len(cell));
+
+  /* Finally, encode the cell into the payload. */
+  len = sendme_cell_encode(payload, RELAY_PAYLOAD_SIZE, cell);
+
+  sendme_cell_free(cell);
+  return len;
+}
+
+/* Send a circuit-level SENDME on the given circuit using the layer_hint if
+ * not NULL. The digest is only used for version 1.
+ *
+ * Return 0 on success else a negative value and the circuit will be closed
+ * because we failed to send the cell on it. */
+static int
+send_circuit_level_sendme(circuit_t *circ, crypt_path_t *layer_hint,
+                          const uint8_t *cell_digest)
+{
+  uint8_t emit_version;
+  uint8_t payload[RELAY_PAYLOAD_SIZE];
+  ssize_t payload_len;
+
+  tor_assert(circ);
+  tor_assert(cell_digest);
+
+  emit_version = get_emit_min_version();
+  switch (emit_version) {
+  case 0x01:
+    payload_len = build_cell_payload_v1(cell_digest, payload);
+    if (BUG(payload_len < 0)) {
+      /* Unable to encode the cell, abort. We can recover from this by closing
+       * the circuit but in theory it should never happen. */
+      return -1;
+    }
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 1 cell.");
+    break;
+  case 0x00:
+    FALLTHROUGH;
+  default:
+    /* Unknown version, fallback to version 0 meaning no payload. */
+    payload_len = 0;
+    log_debug(LD_PROTOCOL, "Emitting SENDME version 0 cell. "
+                           "Consensus emit version is %d", emit_version);
+    break;
+  }
+
+  if (relay_send_command_from_edge(0, circ, RELAY_COMMAND_SENDME,
+                                   (char *) payload, payload_len,
+                                   layer_hint) < 0) {
+    log_warn(LD_CIRC,
+             "SENDME relay_send_command_from_edge failed. Circuit's closed.");
+    return -1; /* the circuit's closed, don't continue */
+  }
+  return 0;
+}
+
+/* Record the cell digest only if the next cell is expected to be a SENDME. */
+static void
+record_cell_digest_on_circ(circuit_t *circ, const uint8_t *sendme_digest)
+{
+  tor_assert(circ);
+  tor_assert(sendme_digest);
+
+  /* Add the digest to the last seen list in the circuit. */
+  if (circ->sendme_last_digests == NULL) {
+    circ->sendme_last_digests = smartlist_new();
+  }
+  smartlist_add(circ->sendme_last_digests,
+                tor_memdup(sendme_digest, DIGEST_LEN));
+}
+
+/*
+ * Public API
+ */
+
+/** Return true iff the next cell for the given cell window is expected to be
+ * a SENDME.
+ *
+ * We are able to know that because the package or deliver window value minus
+ * one cell (the possible SENDME cell) should be a multiple of the increment
+ * window value. */
+static bool
+circuit_sendme_cell_is_next(int window)
+{
+  /* At the start of the window, no SENDME will be expected. */
+  if (window == CIRCWINDOW_START) {
+    return false;
+  }
+
+  /* Are we at the limit of the increment and if not, we don't expect next
+   * cell is a SENDME.
+   *
+   * We test against the window minus 1 because when we are looking if the
+   * next cell is a SENDME, the window (either package or deliver) hasn't been
+   * decremented just yet so when this is called, we are currently processing
+   * the "window - 1" cell.
+   *
+   * This function is used when recording a cell digest and this is done quite
+   * low in the stack when decrypting or encrypting a cell. The window is only
+   * updated once the cell is actually put in the outbuf. */
+  if (((window - 1) % CIRCWINDOW_INCREMENT) != 0) {
+    return false;
+  }
+
+  /* Next cell is expected to be a SENDME. */
+  return true;
+}
+
+/** Called when we've just received a relay data cell, when we've just
+ * finished flushing all bytes to stream <b>conn</b>, or when we've flushed
+ * *some* bytes to the stream <b>conn</b>.
+ *
+ * If conn->outbuf is not too full, and our deliver window is low, send back a
+ * suitable number of stream-level sendme cells.
+ */
+void
+sendme_connection_edge_consider_sending(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  int log_domain = TO_CONN(conn)->type == CONN_TYPE_AP ? LD_APP : LD_EXIT;
+
+  /* Don't send it if we still have data to deliver. */
+  if (connection_outbuf_too_full(TO_CONN(conn))) {
+    goto end;
+  }
+
+  if (circuit_get_by_edge_conn(conn) == NULL) {
+    /* This can legitimately happen if the destroy has already arrived and
+     * torn down the circuit. */
+    log_info(log_domain, "No circuit associated with edge connection. "
+                         "Skipping sending SENDME.");
+    goto end;
+  }
+
+  while (conn->deliver_window <=
+         (STREAMWINDOW_START - STREAMWINDOW_INCREMENT)) {
+    log_debug(log_domain, "Outbuf %" TOR_PRIuSZ ", queuing stream SENDME.",
+              TO_CONN(conn)->outbuf_flushlen);
+    conn->deliver_window += STREAMWINDOW_INCREMENT;
+    if (connection_edge_send_command(conn, RELAY_COMMAND_SENDME,
+                                     NULL, 0) < 0) {
+      log_warn(LD_BUG, "connection_edge_send_command failed while sending "
+                       "a SENDME. Circuit probably closed, skipping.");
+      goto end; /* The circuit's closed, don't continue */
+    }
+  }
+
+ end:
+  return;
+}
+
+/** Check if the deliver_window for circuit <b>circ</b> (at hop
+ * <b>layer_hint</b> if it's defined) is low enough that we should
+ * send a circuit-level sendme back down the circuit. If so, send
+ * enough sendmes that the window would be overfull if we sent any
+ * more.
+ */
+void
+sendme_circuit_consider_sending(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  bool sent_one_sendme = false;
+  const uint8_t *digest;
+
+  while ((layer_hint ? layer_hint->deliver_window : circ->deliver_window) <=
+          CIRCWINDOW_START - CIRCWINDOW_INCREMENT) {
+    log_debug(LD_CIRC,"Queuing circuit sendme.");
+    if (layer_hint) {
+      layer_hint->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = cpath_get_sendme_digest(layer_hint);
+    } else {
+      circ->deliver_window += CIRCWINDOW_INCREMENT;
+      digest = relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+    }
+    if (send_circuit_level_sendme(circ, layer_hint, digest) < 0) {
+      return; /* The circuit's closed, don't continue */
+    }
+    /* Current implementation is not suppose to send multiple SENDME at once
+     * because this means we would use the same relay crypto digest for each
+     * SENDME leading to a mismatch on the other side and the circuit to
+     * collapse. Scream loudly if it ever happens so we can address it. */
+    tor_assert_nonfatal(!sent_one_sendme);
+    sent_one_sendme = true;
+  }
+}
+
+/* Process a circuit-level SENDME cell that we just received. The layer_hint,
+ * if not NULL, is the Exit hop of the connection which means that we are a
+ * client. In that case, circ must be an origin circuit. The cell_body_len is
+ * the length of the SENDME cell payload (excluding the header). The
+ * cell_payload is the payload.
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_circuit_level(crypt_path_t *layer_hint,
+                             circuit_t *circ, const uint8_t *cell_payload,
+                             uint16_t cell_payload_len)
+{
+  tor_assert(circ);
+  tor_assert(cell_payload);
+
+  /* Validate the SENDME cell. Depending on the version, different validation
+   * can be done. An invalid SENDME requires us to close the circuit. */
+  if (!sendme_is_valid(circ, cell_payload, cell_payload_len)) {
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+
+  /* If we are the origin of the circuit, we are the Client so we use the
+   * layer hint (the Exit hop) for the package window tracking. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* If we are the origin of the circuit, it is impossible to not have a
+     * cpath. Just in case, bug on it and close the circuit. */
+    if (BUG(layer_hint == NULL)) {
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    if ((layer_hint->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t exit_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&exit_warn_ratelim, LOG_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from exit relay. "
+                     "Closing circ.");
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    layer_hint->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_APP, "circ-level sendme at origin, packagewindow %d.",
+              layer_hint->package_window);
+
+    /* We count circuit-level sendme's as valid delivered data because they
+     * are rate limited. */
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_payload_len);
+  } else {
+    /* We aren't the origin of this circuit so we are the Exit and thus we
+     * track the package window with the circuit object. */
+    if ((circ->package_window + CIRCWINDOW_INCREMENT) >
+        CIRCWINDOW_START_MAX) {
+      static struct ratelim_t client_warn_ratelim = RATELIM_INIT(600);
+      log_fn_ratelim(&client_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                     "Unexpected sendme cell from client. "
+                     "Closing circ (window %d).", circ->package_window);
+      return -END_CIRC_REASON_TORPROTOCOL;
+    }
+    circ->package_window += CIRCWINDOW_INCREMENT;
+    log_debug(LD_EXIT, "circ-level sendme at non-origin, packagewindow %d.",
+              circ->package_window);
+  }
+
+  return 0;
+}
+
+/* Process a stream-level SENDME cell that we just received. The conn is the
+ * edge connection (stream) that the circuit circ is associated with. The
+ * cell_body_len is the length of the payload (excluding the header).
+ *
+ * Return 0 on success (the SENDME is valid and the package window has
+ * been updated properly).
+ *
+ * On error, a negative value is returned, which indicates that the
+ * circuit must be closed using the value as the reason for it. */
+int
+sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                            uint16_t cell_body_len)
+{
+  tor_assert(conn);
+  tor_assert(circ);
+
+  /* Don't allow the other endpoint to request more than our maximum (i.e.
+   * initial) stream SENDME window worth of data. Well-behaved stock clients
+   * will not request more than this max (as per the check in the while loop
+   * of sendme_connection_edge_consider_sending()). */
+  if ((conn->package_window + STREAMWINDOW_INCREMENT) >
+      STREAMWINDOW_START_MAX) {
+    static struct ratelim_t stream_warn_ratelim = RATELIM_INIT(600);
+    log_fn_ratelim(&stream_warn_ratelim, LOG_PROTOCOL_WARN, LD_PROTOCOL,
+                   "Unexpected stream sendme cell. Closing circ (window %d).",
+                   conn->package_window);
+    return -END_CIRC_REASON_TORPROTOCOL;
+  }
+  /* At this point, the stream sendme is valid */
+  conn->package_window += STREAMWINDOW_INCREMENT;
+
+  /* We count circuit-level sendme's as valid delivered data because they are
+   * rate limited. */
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    circuit_read_valid_data(TO_ORIGIN_CIRCUIT(circ), cell_body_len);
+  }
+
+  log_debug(CIRCUIT_IS_ORIGIN(circ) ? LD_APP : LD_EXIT,
+            "stream-level sendme, package_window now %d.",
+            conn->package_window);
+  return 0;
+}
+
+/* Called when a relay DATA cell is received on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the deliver window and return its new value. */
+int
+sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int deliver_window, domain;
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    tor_assert(layer_hint);
+    --layer_hint->deliver_window;
+    deliver_window = layer_hint->deliver_window;
+    domain = LD_APP;
+  } else {
+    tor_assert(!layer_hint);
+    --circ->deliver_window;
+    deliver_window = circ->deliver_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit deliver_window now %d.", deliver_window);
+  return deliver_window;
+}
+
+/* Called when a relay DATA cell is received for the given edge connection
+ * conn. Update the deliver window and return its new value. */
+int
+sendme_stream_data_received(edge_connection_t *conn)
+{
+  tor_assert(conn);
+  return --conn->deliver_window;
+}
+
+/* Called when a relay DATA cell is packaged on the given circuit. If
+ * layer_hint is NULL, this means we are the Exit end point else we are the
+ * Client. Update the package window and return its new value. */
+int
+sendme_note_circuit_data_packaged(circuit_t *circ, crypt_path_t *layer_hint)
+{
+  int package_window, domain;
+
+  tor_assert(circ);
+
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    /* Client side. */
+    tor_assert(layer_hint);
+    --layer_hint->package_window;
+    package_window = layer_hint->package_window;
+    domain = LD_APP;
+  } else {
+    /* Exit side. */
+    tor_assert(!layer_hint);
+    --circ->package_window;
+    package_window = circ->package_window;
+    domain = LD_EXIT;
+  }
+
+  log_debug(domain, "Circuit package_window now %d.", package_window);
+  return package_window;
+}
+
+/* Called when a relay DATA cell is packaged for the given edge connection
+ * conn. Update the package window and return its new value. */
+int
+sendme_note_stream_data_packaged(edge_connection_t *conn)
+{
+  tor_assert(conn);
+
+  --conn->package_window;
+  log_debug(LD_APP, "Stream package_window now %d.", conn->package_window);
+  return conn->package_window;
+}
+
+/* Record the cell digest into the circuit sendme digest list depending on
+ * which edge we are. The digest is recorded only if we expect the next cell
+ * that we will receive is a SENDME so we can match the digest. */
+void
+sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath)
+{
+  int package_window;
+  uint8_t *sendme_digest;
+
+  tor_assert(circ);
+
+  package_window = circ->package_window;
+  if (cpath) {
+    package_window = cpath->package_window;
+  }
+
+  /* Is this the last cell before a SENDME? The idea is that if the
+   * package_window reaches a multiple of the increment, after this cell, we
+   * should expect a SENDME. */
+  if (!circuit_sendme_cell_is_next(package_window)) {
+    return;
+  }
+
+  /* Getting the digest is expensive so we only do it once we are certain to
+   * record it on the circuit. */
+  if (cpath) {
+    sendme_digest = cpath_get_sendme_digest(cpath);
+  } else {
+    sendme_digest =
+      relay_crypto_get_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto);
+  }
+
+  record_cell_digest_on_circ(circ, sendme_digest);
+}
+
+/* Called once we decrypted a cell and recognized it. Record the cell digest
+ * as the next sendme digest only if the next cell we'll send on the circuit
+ * is expected to be a SENDME. */
+void
+sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->deliver_window :
+                                           circ->deliver_window)) {
+    return;
+  }
+
+  if (cpath) {
+    /* Record incoming digest. */
+    cpath_sendme_record_cell_digest(cpath, false);
+  } else {
+    /* Record foward digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, true);
+  }
+}
+
+/* Called once we encrypted a cell. Record the cell digest as the next sendme
+ * digest only if the next cell we expect to receive is a SENDME so we can
+ * match the digests. */
+void
+sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath)
+{
+  tor_assert(circ);
+
+  /* Only record if the next cell is expected to be a SENDME. */
+  if (!circuit_sendme_cell_is_next(cpath ? cpath->package_window :
+                                           circ->package_window)) {
+    goto end;
+  }
+
+  if (cpath) {
+    /* Record the forward digest. */
+    cpath_sendme_record_cell_digest(cpath, true);
+  } else {
+    /* Record the incoming digest. */
+    relay_crypto_record_sendme_digest(&TO_OR_CIRCUIT(circ)->crypto, false);
+  }
+
+ end:
+  return;
+}
diff --git a/src/core/or/sendme.h b/src/core/or/sendme.h
new file mode 100644
index 0000000000..05d37ec3bb
--- /dev/null
+++ b/src/core/or/sendme.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file sendme.h
+ * \brief Header file for sendme.c.
+ **/
+
+#ifndef TOR_SENDME_H
+#define TOR_SENDME_H
+
+#include "core/or/edge_connection_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/circuit_st.h"
+
+/* Sending SENDME cell. */
+void sendme_connection_edge_consider_sending(edge_connection_t *edge_conn);
+void sendme_circuit_consider_sending(circuit_t *circ,
+                                     crypt_path_t *layer_hint);
+
+/* Processing SENDME cell. */
+int sendme_process_circuit_level(crypt_path_t *layer_hint,
+                                 circuit_t *circ, const uint8_t *cell_payload,
+                                 uint16_t cell_payload_len);
+int sendme_process_stream_level(edge_connection_t *conn, circuit_t *circ,
+                                uint16_t cell_body_len);
+
+/* Update deliver window functions. */
+int sendme_stream_data_received(edge_connection_t *conn);
+int sendme_circuit_data_received(circuit_t *circ, crypt_path_t *layer_hint);
+
+/* Update package window functions. */
+int sendme_note_circuit_data_packaged(circuit_t *circ,
+                                      crypt_path_t *layer_hint);
+int sendme_note_stream_data_packaged(edge_connection_t *conn);
+
+/* Record cell digest on circuit. */
+void sendme_record_cell_digest_on_circ(circuit_t *circ, crypt_path_t *cpath);
+/* Record cell digest as the SENDME digest. */
+void sendme_record_received_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+void sendme_record_sending_cell_digest(circuit_t *circ, crypt_path_t *cpath);
+
+/* Private section starts. */
+#ifdef SENDME_PRIVATE
+
+/* The maximum supported version. Above that value, the cell can't be
+ * recognized as a valid SENDME. */
+#define SENDME_MAX_SUPPORTED_VERSION 1
+
+/* The cell version constants for when emitting a cell. */
+#define SENDME_EMIT_MIN_VERSION_DEFAULT 1
+#define SENDME_EMIT_MIN_VERSION_MIN 0
+#define SENDME_EMIT_MIN_VERSION_MAX UINT8_MAX
+
+/* The cell version constants for when accepting a cell. */
+#define SENDME_ACCEPT_MIN_VERSION_DEFAULT 0
+#define SENDME_ACCEPT_MIN_VERSION_MIN 0
+#define SENDME_ACCEPT_MIN_VERSION_MAX UINT8_MAX
+
+/*
+ * Unit tests declaractions.
+ */
+#ifdef TOR_UNIT_TESTS
+
+STATIC int get_emit_min_version(void);
+STATIC int get_accept_min_version(void);
+
+STATIC bool cell_version_can_be_handled(uint8_t cell_version);
+
+STATIC ssize_t build_cell_payload_v1(const uint8_t *cell_digest,
+                                     uint8_t *payload);
+STATIC bool sendme_is_valid(const circuit_t *circ,
+                            const uint8_t *cell_payload,
+                            size_t cell_payload_len);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(SENDME_PRIVATE) */
+
+#endif /* !defined(TOR_SENDME_H) */
diff --git a/src/core/or/server_port_cfg_st.h b/src/core/or/server_port_cfg_st.h
index bd026af7ee..9a005eccdf 100644
--- a/src/core/or/server_port_cfg_st.h
+++ b/src/core/or/server_port_cfg_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file server_port_cfg_st.h
+ * @brief Cnfiguration structure for server ports.
+ **/
+
 #ifndef SERVER_PORT_CFG_ST_H
 #define SERVER_PORT_CFG_ST_H
 
@@ -16,5 +21,4 @@ struct server_port_cfg_t {
   unsigned int bind_ipv6_only : 1;
 };
 
-#endif
-
+#endif /* !defined(SERVER_PORT_CFG_ST_H) */
diff --git a/src/core/or/socks_request_st.h b/src/core/or/socks_request_st.h
index 5922870c61..4bcdb48b92 100644
--- a/src/core/or/socks_request_st.h
+++ b/src/core/or/socks_request_st.h
@@ -1,12 +1,19 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file socks_request_st.h
+ * @brief Client request structure.
+ **/
+
 #ifndef SOCKS_REQUEST_ST_H
 #define SOCKS_REQUEST_ST_H
 
+#include "lib/net/socks5_status.h"
+
 #define MAX_SOCKS_REPLY_LEN 1024
 
 #define SOCKS_NO_AUTH 0x00
@@ -58,6 +65,11 @@ struct socks_request_t {
    * "username/password" authentication if both are offered. Used as input to
    * parse_socks. */
   unsigned int socks_prefer_no_auth : 1;
+  /** If set, we can send back the extended error code in the reply. */
+  unsigned int socks_use_extended_errors : 1;
+  /** If non zero, this contains the extended error code that should be used
+   * if the port was configured to use them. */
+  socks5_reply_status_t socks_extended_error_code;
 
   /** Number of bytes in username; 0 if username is NULL */
   size_t usernamelen;
@@ -74,4 +86,4 @@ struct socks_request_t {
   uint8_t socks5_atyp; /* SOCKS5 address type */
 };
 
-#endif
+#endif /* !defined(SOCKS_REQUEST_ST_H) */
diff --git a/src/core/or/status.c b/src/core/or/status.c
index 46494ca76c..ed8448883c 100644
--- a/src/core/or/status.c
+++ b/src/core/or/status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "core/or/or.h"
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
+#include "feature/dirclient/dirclient.h"
 #include "core/or/status.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/relay.h"
@@ -146,6 +147,8 @@ log_heartbeat(time_t now)
          uptime, count_circuits(), bw_sent, bw_rcvd,
          hibernating?" We are currently hibernating.":"");
 
+  dirclient_dump_total_dls();
+
   if (server_mode(options) && accounting_is_enabled(options) && !hibernating) {
     log_accounting(now, options);
   }
diff --git a/src/core/or/status.h b/src/core/or/status.h
index 3467501ebb..639f8cdf51 100644
--- a/src/core/or/status.h
+++ b/src/core/or/status.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file status.h
+ * @brief Header for status.c
+ **/
+
 #ifndef TOR_STATUS_H
 #define TOR_STATUS_H
 
@@ -15,4 +20,3 @@ STATIC char *bytes_to_usage(uint64_t bytes);
 #endif
 
 #endif /* !defined(TOR_STATUS_H) */
-
diff --git a/src/core/or/tor_version_st.h b/src/core/or/tor_version_st.h
index 716429bd32..46644c5eb8 100644
--- a/src/core/or/tor_version_st.h
+++ b/src/core/or/tor_version_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tor_version_st.h
+ * @brief Parsed Tor version structure.
+ **/
+
 #ifndef TOR_VERSION_ST_H
 #define TOR_VERSION_ST_H
 
@@ -28,5 +33,4 @@ struct tor_version_t {
   char git_tag[DIGEST_LEN];
 };
 
-#endif
-
+#endif /* !defined(TOR_VERSION_ST_H) */
diff --git a/src/core/or/var_cell_st.h b/src/core/or/var_cell_st.h
index 4287c83f6d..caf64c63a5 100644
--- a/src/core/or/var_cell_st.h
+++ b/src/core/or/var_cell_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file var_cell_st.h
+ * @brief Variable-length cell structure.
+ **/
+
 #ifndef VAR_CELL_ST_H
 #define VAR_CELL_ST_H
 
@@ -19,5 +24,4 @@ struct var_cell_t {
   uint8_t payload[FLEXIBLE_ARRAY_MEMBER];
 };
 
-#endif
-
+#endif /* !defined(VAR_CELL_ST_H) */
diff --git a/src/core/or/versions.c b/src/core/or/versions.c
index 33273a5294..31f1f5b997 100644
--- a/src/core/or/versions.c
+++ b/src/core/or/versions.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,25 @@
 
 #include "core/or/tor_version_st.h"
 
+/**
+ * Return the approximate date when this release came out, or was
+ * scheduled to come out, according to the APPROX_RELEASE_DATE set in
+ * configure.ac
+ **/
+time_t
+tor_get_approx_release_date(void)
+{
+  char tbuf[ISO_TIME_LEN+1];
+  tor_snprintf(tbuf, sizeof(tbuf),
+               "%s 00:00:00", APPROX_RELEASE_DATE);
+  time_t result = 0;
+  int r = parse_iso_time(tbuf, &result);
+  if (BUG(r < 0)) {
+    result = 0;
+  }
+  return result;
+}
+
 /** Return VS_RECOMMENDED if <b>myversion</b> is contained in
  * <b>versionlist</b>.  Else, return VS_EMPTY if versionlist has no
  * entries. Else, return VS_OLD if every member of
@@ -277,7 +296,7 @@ tor_version_parse(const char *s, tor_version_t *out)
       return -1;
     hexlen = (int)(close_paren-cp);
     memwipe(digest, 0, sizeof(digest));
-    if ( hexlen == 0 || (hexlen % 2) == 1)
+    if (hexlen == 0 || (hexlen % 2) == 1)
       return -1;
     if (base16_decode(digest, hexlen/2, cp, hexlen) != hexlen/2)
       return -1;
@@ -377,6 +396,69 @@ sort_version_list(smartlist_t *versions, int remove_duplicates)
     smartlist_uniq(versions, compare_tor_version_str_ptr_, tor_free_);
 }
 
+/** If there are more than this many entries, we're probably under
+ * some kind of weird DoS. */
+static const int MAX_PROTOVER_SUMMARY_MAP_LEN = 1024;
+
+/**
+ * Map from protover string to protover_summary_flags_t.
+ */
+static strmap_t *protover_summary_map = NULL;
+
+/**
+ * Helper.  Given a non-NULL protover string <b>protocols</b>, set <b>out</b>
+ * to its summary, and memoize the result in <b>protover_summary_map</b>.
+ */
+static void
+memoize_protover_summary(protover_summary_flags_t *out,
+                         const char *protocols)
+{
+  if (!protover_summary_map)
+    protover_summary_map = strmap_new();
+
+  if (strmap_size(protover_summary_map) >= MAX_PROTOVER_SUMMARY_MAP_LEN) {
+    protover_summary_cache_free_all();
+    tor_assert(protover_summary_map == NULL);
+    protover_summary_map = strmap_new();
+  }
+
+  const protover_summary_flags_t *cached =
+    strmap_get(protover_summary_map, protocols);
+
+  if (cached != NULL) {
+    /* We found a cached entry; no need to parse this one. */
+    memcpy(out, cached, sizeof(protover_summary_flags_t));
+    tor_assert(out->protocols_known);
+    return;
+  }
+
+  memset(out, 0, sizeof(*out));
+  out->protocols_known = 1;
+  out->supports_extend2_cells =
+    protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
+  out->supports_ed25519_link_handshake_compat =
+    protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_link_handshake_any =
+    protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
+  out->supports_ed25519_hs_intro =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
+  out->supports_v3_hsdir =
+    protocol_list_supports_protocol(protocols, PRT_HSDIR,
+                                    PROTOVER_HSDIR_V3);
+  out->supports_v3_rendezvous_point =
+    protocol_list_supports_protocol(protocols, PRT_HSREND,
+                                    PROTOVER_HS_RENDEZVOUS_POINT_V3);
+  out->supports_hs_setup_padding =
+    protocol_list_supports_protocol(protocols, PRT_PADDING,
+                                    PROTOVER_HS_SETUP_PADDING);
+  out->supports_establish_intro_dos_extension =
+    protocol_list_supports_protocol(protocols, PRT_HSINTRO, 5);
+
+  protover_summary_flags_t *new_cached = tor_memdup(out, sizeof(*out));
+  cached = strmap_set(protover_summary_map, protocols, new_cached);
+  tor_assert(!cached);
+}
+
 /** Summarize the protocols listed in <b>protocols</b> into <b>out</b>,
  * falling back or correcting them based on <b>version</b> as appropriate.
  */
@@ -388,21 +470,7 @@ summarize_protover_flags(protover_summary_flags_t *out,
   tor_assert(out);
   memset(out, 0, sizeof(*out));
   if (protocols) {
-    out->protocols_known = 1;
-    out->supports_extend2_cells =
-      protocol_list_supports_protocol(protocols, PRT_RELAY, 2);
-    out->supports_ed25519_link_handshake_compat =
-      protocol_list_supports_protocol(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_link_handshake_any =
-      protocol_list_supports_protocol_or_later(protocols, PRT_LINKAUTH, 3);
-    out->supports_ed25519_hs_intro =
-      protocol_list_supports_protocol(protocols, PRT_HSINTRO, 4);
-    out->supports_v3_hsdir =
-      protocol_list_supports_protocol(protocols, PRT_HSDIR,
-                                      PROTOVER_HSDIR_V3);
-    out->supports_v3_rendezvous_point =
-      protocol_list_supports_protocol(protocols, PRT_HSREND,
-                                      PROTOVER_HS_RENDEZVOUS_POINT_V3);
+    memoize_protover_summary(out, protocols);
   }
   if (version && !strcmpstart(version, "Tor ")) {
     if (!out->protocols_known) {
@@ -420,3 +488,13 @@ summarize_protover_flags(protover_summary_flags_t *out,
     }
   }
 }
+
+/**
+ * Free all space held in the protover_summary_map.
+ */
+void
+protover_summary_cache_free_all(void)
+{
+  strmap_free(protover_summary_map, tor_free_);
+  protover_summary_map = NULL;
+}
diff --git a/src/core/or/versions.h b/src/core/or/versions.h
index 22f3be176f..75dc17f9c7 100644
--- a/src/core/or/versions.h
+++ b/src/core/or/versions.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,6 +26,8 @@ typedef enum version_status_t {
   VS_UNKNOWN, /**< We have no idea. */
 } version_status_t;
 
+time_t tor_get_approx_release_date(void);
+
 version_status_t tor_version_is_obsolete(const char *myversion,
                                          const char *versionlist);
 int tor_version_parse_platform(const char *platform,
@@ -41,4 +43,6 @@ void summarize_protover_flags(protover_summary_flags_t *out,
                               const char *protocols,
                               const char *version);
 
+void protover_summary_cache_free_all(void);
+
 #endif /* !defined(TOR_VERSIONS_H) */
diff --git a/src/core/proto/.may_include b/src/core/proto/.may_include
new file mode 100644
index 0000000000..a66c3f83a6
--- /dev/null
+++ b/src/core/proto/.may_include
@@ -0,0 +1,14 @@
+!advisory
+
+orconfig.h
+
+lib/crypt_ops/*.h
+lib/buf/*.h
+lib/malloc/*.h
+lib/string/*.h
+
+lib/net/address.h
+
+trunnel/*.h
+
+core/proto/*.h
diff --git a/src/core/proto/core_proto.md b/src/core/proto/core_proto.md
new file mode 100644
index 0000000000..ad43bc5846
--- /dev/null
+++ b/src/core/proto/core_proto.md
@@ -0,0 +1,6 @@
+@dir /core/proto
+@brief core/proto: Protocol encoding/decoding
+
+These functions should (but do not always) exist at a lower level than most
+of the rest of core.
+
diff --git a/src/core/proto/include.am b/src/core/proto/include.am
new file mode 100644
index 0000000000..726ef924cf
--- /dev/null
+++ b/src/core/proto/include.am
@@ -0,0 +1,18 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/core/proto/proto_cell.c		\
+	src/core/proto/proto_control0.c		\
+	src/core/proto/proto_ext_or.c		\
+	src/core/proto/proto_haproxy.c		\
+	src/core/proto/proto_http.c		\
+	src/core/proto/proto_socks.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/core/proto/proto_cell.h			\
+	src/core/proto/proto_control0.h			\
+	src/core/proto/proto_ext_or.h			\
+	src/core/proto/proto_haproxy.h			\
+	src/core/proto/proto_http.h			\
+	src/core/proto/proto_socks.h
diff --git a/src/core/proto/proto_cell.c b/src/core/proto/proto_cell.c
index 0442e2c6ee..5c1a2c24d7 100644
--- a/src/core/proto/proto_cell.c
+++ b/src/core/proto/proto_cell.c
@@ -1,11 +1,18 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_cell.c
+ * @brief Decodes Tor cells from buffers.
+ **/
+/* Right now it only handles variable-length cells, but eventually
+ * we should refactor other cell-reading code into here. */
+
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_cell.h"
 
 #include "core/or/connection_or.h"
@@ -83,4 +90,3 @@ fetch_var_cell_from_buf(buf_t *buf, var_cell_t **out, int linkproto)
   *out = result;
   return 1;
 }
-
diff --git a/src/core/proto/proto_cell.h b/src/core/proto/proto_cell.h
index 4f3982ea43..0e9cfbfed3 100644
--- a/src/core/proto/proto_cell.h
+++ b/src/core/proto/proto_cell.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_cell.h
+ * @brief Header for proto_cell.c
+ **/
+
 #ifndef TOR_PROTO_CELL_H
 #define TOR_PROTO_CELL_H
 
@@ -14,4 +19,3 @@ int fetch_var_cell_from_buf(struct buf_t *buf, struct var_cell_t **out,
                             int linkproto);
 
 #endif /* !defined(TOR_PROTO_CELL_H) */
-
diff --git a/src/core/proto/proto_control0.c b/src/core/proto/proto_control0.c
index 21fa328f02..323b37c539 100644
--- a/src/core/proto/proto_control0.c
+++ b/src/core/proto/proto_control0.c
@@ -1,11 +1,16 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_control0.c
+ * @brief Code to detect the obsolete v0 control protocol.
+ **/
+
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_control0.h"
 
 /** Return 1 iff buf looks more like it has an (obsolete) v0 controller
@@ -23,4 +28,3 @@ peek_buf_has_control0_command(buf_t *buf)
   }
   return 0;
 }
-
diff --git a/src/core/proto/proto_control0.h b/src/core/proto/proto_control0.h
index 162e513a1b..561bf00d19 100644
--- a/src/core/proto/proto_control0.h
+++ b/src/core/proto/proto_control0.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_control0.h
+ * @brief Header for proto_control0.c
+ **/
+
 #ifndef TOR_PROTO_CONTROL0_H
 #define TOR_PROTO_CONTROL0_H
 
@@ -11,4 +16,3 @@ struct buf_t;
 int peek_buf_has_control0_command(struct buf_t *buf);
 
 #endif /* !defined(TOR_PROTO_CONTROL0_H) */
-
diff --git a/src/core/proto/proto_ext_or.c b/src/core/proto/proto_ext_or.c
index edbc51b10c..23fc2393b7 100644
--- a/src/core/proto/proto_ext_or.c
+++ b/src/core/proto/proto_ext_or.c
@@ -1,11 +1,16 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_ext_or.c
+ * @brief Parsing/encoding for the extended OR protocol.
+ **/
+
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_ext_or.h"
 
@@ -37,4 +42,3 @@ fetch_ext_or_command_from_buf(buf_t *buf, ext_or_cmd_t **out)
   buf_get_bytes(buf, (*out)->body, len);
   return 1;
 }
-
diff --git a/src/core/proto/proto_ext_or.h b/src/core/proto/proto_ext_or.h
index b2bc64af85..3408599fb7 100644
--- a/src/core/proto/proto_ext_or.h
+++ b/src/core/proto/proto_ext_or.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_ext_or.h
+ * @brief Header for proto_ext_or.c
+ **/
+
 #ifndef TOR_PROTO_EXT_OR_H
 #define TOR_PROTO_EXT_OR_H
 
@@ -19,4 +24,11 @@ struct ext_or_cmd_t {
 int fetch_ext_or_command_from_buf(struct buf_t *buf,
                                   struct ext_or_cmd_t **out);
 
+ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
+
+#define ext_or_cmd_free(cmd)                            \
+  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+
+void ext_or_cmd_free_(ext_or_cmd_t *cmd);
+
 #endif /* !defined(TOR_PROTO_EXT_OR_H) */
diff --git a/src/core/proto/proto_haproxy.c b/src/core/proto/proto_haproxy.c
new file mode 100644
index 0000000000..9129fc55bf
--- /dev/null
+++ b/src/core/proto/proto_haproxy.c
@@ -0,0 +1,45 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define PROTO_HAPROXY_PRIVATE
+#include "lib/malloc/malloc.h"
+#include "lib/net/address.h"
+#include "lib/string/printf.h"
+#include "core/proto/proto_haproxy.h"
+
+/** Return a newly allocated PROXY header null-terminated string. Returns NULL
+ * if addr_port->addr is incompatible with the proxy protocol.
+ */
+char *
+haproxy_format_proxy_header_line(const tor_addr_port_t *addr_port)
+{
+  tor_assert(addr_port);
+
+  sa_family_t family = tor_addr_family(&addr_port->addr);
+  const char *family_string = NULL;
+  const char *src_addr_string = NULL;
+
+  switch (family) {
+    case AF_INET:
+      family_string = "TCP4";
+      src_addr_string = "0.0.0.0";
+      break;
+    case AF_INET6:
+      family_string = "TCP6";
+      src_addr_string = "::";
+      break;
+    default:
+      /* Unknown family. */
+      return NULL;
+  }
+
+  char *buf;
+  char addrbuf[TOR_ADDR_BUF_LEN];
+
+  tor_addr_to_str(addrbuf, &addr_port->addr, sizeof(addrbuf), 0);
+
+  tor_asprintf(&buf, "PROXY %s %s %s 0 %d\r\n", family_string, src_addr_string,
+                                                addrbuf, addr_port->port);
+
+  return buf;
+}
diff --git a/src/core/proto/proto_haproxy.h b/src/core/proto/proto_haproxy.h
new file mode 100644
index 0000000000..63c164e1a1
--- /dev/null
+++ b/src/core/proto/proto_haproxy.h
@@ -0,0 +1,12 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PROTO_HAPROXY_H
+#define TOR_PROTO_HAPROXY_H
+
+struct tor_addr_port_t;
+
+char *haproxy_format_proxy_header_line(
+                                    const struct tor_addr_port_t *addr_port);
+
+#endif /* !defined(TOR_PROTO_HAPROXY_H) */
diff --git a/src/core/proto/proto_http.c b/src/core/proto/proto_http.c
index 5c86fc4979..ef4b897fcc 100644
--- a/src/core/proto/proto_http.c
+++ b/src/core/proto/proto_http.c
@@ -1,12 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_http.c
+ * @brief Parse a subset of the HTTP protocol.
+ **/
+
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 
 /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */
@@ -168,4 +173,3 @@ buf_http_find_content_length(const char *headers, size_t headerlen,
 
   return ok ? 1 : -1;
 }
-
diff --git a/src/core/proto/proto_http.h b/src/core/proto/proto_http.h
index cd70050205..e0c5135346 100644
--- a/src/core/proto/proto_http.h
+++ b/src/core/proto/proto_http.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_http.h
+ * @brief Header for proto_http.c
+ **/
+
 #ifndef TOR_PROTO_HTTP_H
 #define TOR_PROTO_HTTP_H
 
@@ -21,4 +26,3 @@ STATIC int buf_http_find_content_length(const char *headers, size_t headerlen,
 #endif
 
 #endif /* !defined(TOR_PROTO_HTTP_H) */
-
diff --git a/src/core/proto/proto_socks.c b/src/core/proto/proto_socks.c
index 5a7d7ac9be..bcb0c2b2f9 100644
--- a/src/core/proto/proto_socks.c
+++ b/src/core/proto/proto_socks.c
@@ -1,14 +1,19 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_socks.c
+ * @brief Implementations for SOCKS4 and SOCKS5 protocols.
+ **/
+
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/relay/ext_orport.h"
@@ -105,7 +110,7 @@ socks_request_free_(socks_request_t *req)
 /**
  * Parse a single SOCKS4 request from buffer <b>raw_data</b> of length
  * <b>datalen</b> and update relevant fields of <b>req</b>. If SOCKS4a
- * request is detected, set <b>*is_socks4a<b> to true. Set <b>*drain_out</b>
+ * request is detected, set <b>*is_socks4a</b> to true. Set <b>*drain_out</b>
  * to number of bytes we parsed so far.
  *
  * Return SOCKS_RESULT_DONE if parsing succeeded, SOCKS_RESULT_INVALID if
@@ -584,9 +589,8 @@ parse_socks5_client_request(const uint8_t *raw_data, socks_request_t *req,
       strlcpy(req->address, hostname, sizeof(req->address));
     } break;
     case 4: {
-      const char *ipv6 =
-        (const char *)socks5_client_request_getarray_dest_addr_ipv6(
-          trunnel_req);
+      const uint8_t *ipv6 =
+          socks5_client_request_getarray_dest_addr_ipv6(trunnel_req);
       tor_addr_from_ipv6_bytes(&destaddr, ipv6);
 
       tor_addr_to_str(req->address, &destaddr, sizeof(req->address), 1);
@@ -618,6 +622,7 @@ process_socks5_client_request(socks_request_t *req,
                               int safe_socks)
 {
   socks_result_t res = SOCKS_RESULT_DONE;
+  tor_addr_t tmpaddr;
 
   if (req->command != SOCKS_COMMAND_CONNECT &&
       req->command != SOCKS_COMMAND_RESOLVE &&
@@ -628,11 +633,10 @@ process_socks5_client_request(socks_request_t *req,
   }
 
   if (req->command == SOCKS_COMMAND_RESOLVE_PTR &&
-      !string_is_valid_ipv4_address(req->address) &&
-      !string_is_valid_ipv6_address(req->address)) {
+      tor_addr_parse(&tmpaddr, req->address) < 0) {
     socks_request_set_socks5_error(req, SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED);
     log_warn(LD_APP, "socks5 received RESOLVE_PTR command with "
-                     "hostname type. Rejecting.");
+                     "a malformed address. Rejecting.");
 
     res = SOCKS_RESULT_INVALID;
     goto end;
diff --git a/src/core/proto/proto_socks.h b/src/core/proto/proto_socks.h
index 2a387bf848..f3af0d988e 100644
--- a/src/core/proto/proto_socks.h
+++ b/src/core/proto/proto_socks.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file proto_socks.h
+ * @brief Header for proto_socks.c
+ **/
+
 #ifndef TOR_PROTO_SOCKS_H
 #define TOR_PROTO_SOCKS_H
 
diff --git a/src/core/stA1RajU b/src/core/stA1RajU
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/core/stA1RajU
diff --git a/src/core/stiysZND b/src/core/stiysZND
new file mode 100644
index 0000000000..faa365b769
--- /dev/null
+++ b/src/core/stiysZND
diff --git a/src/ext/.may_include b/src/ext/.may_include
new file mode 100644
index 0000000000..1eafff2eeb
--- /dev/null
+++ b/src/ext/.may_include
@@ -0,0 +1,10 @@
+
+orconfig.h
+
+lib/err/*.h
+lib/cc/*.h
+
+tinytest*.h
+ext/siphash.h
+ext/byteorder.h
+ext/tor_readpassphrase.h
\ No newline at end of file
diff --git a/src/ext/csiphash.c b/src/ext/csiphash.c
index a65b6fcbe6..e0f5b2e5c9 100644
--- a/src/ext/csiphash.c
+++ b/src/ext/csiphash.c
@@ -30,12 +30,12 @@
 */
 
 #include "lib/cc/torint.h"
-#include "lib/log/util_bug.h"
+#include "lib/err/torerr.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 #include <string.h>
 #include <stdlib.h>
-#include "byteorder.h"
+#include "ext/byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -87,6 +87,13 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		v0 ^= mi;
 	}
 
+#ifdef __COVERITY__
+	{
+		uint64_t mi = 0;
+		memcpy(&mi, m+i, (src_sz-blocks));
+		last7 = _le64toh(mi) | (uint64_t)(src_sz & 0xff) << 56;
+	}
+#else
 	switch (src_sz - blocks) {
 		case 7: last7 |= (uint64_t)m[i + 6] << 48; FALLTHROUGH;
 		case 6: last7 |= (uint64_t)m[i + 5] << 40; FALLTHROUGH;
@@ -98,6 +105,7 @@ uint64_t siphash24(const void *src, unsigned long src_sz, const struct sipkey *k
 		case 0:
 		default:;
 	}
+#endif
 	v3 ^= last7;
 	DOUBLE_ROUND(v0,v1,v2,v3);
 	v0 ^= last7;
@@ -112,13 +120,13 @@ static int the_siphash_key_is_set = 0;
 static struct sipkey the_siphash_key;
 
 uint64_t siphash24g(const void *src, unsigned long src_sz) {
-	tor_assert(the_siphash_key_is_set);
+	raw_assert(the_siphash_key_is_set);
 	return siphash24(src, src_sz, &the_siphash_key);
 }
 
 void siphash_set_global_key(const struct sipkey *key)
 {
-	tor_assert(! the_siphash_key_is_set);
+	raw_assert(! the_siphash_key_is_set);
 	the_siphash_key.k0 = key->k0;
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;
diff --git a/src/ext/ed25519/ref10/base.py b/src/ext/ed25519/ref10/base.py
index 84accc8580..3d477c5c39 100644
--- a/src/ext/ed25519/ref10/base.py
+++ b/src/ext/ed25519/ref10/base.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 b = 256
 q = 2**255 - 19
 l = 2**252 + 27742317777372353535851937790883648493
@@ -51,15 +56,15 @@ def radix255(x):
 
 Bi = B
 for i in range(32):
-  print "{"
+  print("{")
   Bij = Bi
   for j in range(8):
-    print " {"
-    print "  {",radix255(Bij[1]+Bij[0]),"},"
-    print "  {",radix255(Bij[1]-Bij[0]),"},"
-    print "  {",radix255(2*d*Bij[0]*Bij[1]),"},"
+    print(" {")
+    print("  {",radix255(Bij[1]+Bij[0]),"},")
+    print("  {",radix255(Bij[1]-Bij[0]),"},")
+    print("  {",radix255(2*d*Bij[0]*Bij[1]),"},")
     Bij = edwards(Bij,Bi)
-    print " },"
-  print "},"
+    print(" },")
+  print("},")
   for k in range(8):
     Bi = edwards(Bi,Bi)
diff --git a/src/ext/ed25519/ref10/base2.py b/src/ext/ed25519/ref10/base2.py
index 5e4e8739d0..3f8e3d25d2 100644
--- a/src/ext/ed25519/ref10/base2.py
+++ b/src/ext/ed25519/ref10/base2.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 b = 256
 q = 2**255 - 19
 l = 2**252 + 27742317777372353535851937790883648493
@@ -52,9 +57,9 @@ def radix255(x):
 Bi = B
 
 for i in range(8):
-  print " {"
-  print "  {",radix255(Bi[1]+Bi[0]),"},"
-  print "  {",radix255(Bi[1]-Bi[0]),"},"
-  print "  {",radix255(2*d*Bi[0]*Bi[1]),"},"
-  print " },"
+  print(" {")
+  print("  {",radix255(Bi[1]+Bi[0]),"},")
+  print("  {",radix255(Bi[1]-Bi[0]),"},")
+  print("  {",radix255(2*d*Bi[0]*Bi[1]),"},")
+  print(" },")
   Bi = edwards(B,edwards(B,Bi))
diff --git a/src/ext/ed25519/ref10/d.py b/src/ext/ed25519/ref10/d.py
index 8995bb86a3..5b875de666 100644
--- a/src/ext/ed25519/ref10/d.py
+++ b/src/ext/ed25519/ref10/d.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 q = 2**255 - 19
 
 def expmod(b,e,m):
@@ -25,4 +30,4 @@ def radix255(x):
   return result
 
 d = -121665 * inv(121666)
-print radix255(d)
+print(radix255(d))
diff --git a/src/ext/ed25519/ref10/d2.py b/src/ext/ed25519/ref10/d2.py
index 79841758be..f59a1bc62a 100644
--- a/src/ext/ed25519/ref10/d2.py
+++ b/src/ext/ed25519/ref10/d2.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 q = 2**255 - 19
 
 def expmod(b,e,m):
@@ -25,4 +30,4 @@ def radix255(x):
   return result
 
 d = -121665 * inv(121666)
-print radix255(d*2)
+print(radix255(d*2))
diff --git a/src/ext/ed25519/ref10/sqrtm1.py b/src/ext/ed25519/ref10/sqrtm1.py
index 9a47fbc12a..df9f26ee1d 100644
--- a/src/ext/ed25519/ref10/sqrtm1.py
+++ b/src/ext/ed25519/ref10/sqrtm1.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 q = 2**255 - 19
 
 def expmod(b,e,m):
@@ -25,4 +30,4 @@ def radix255(x):
   return result
 
 I = expmod(2,(q-1)/4,q)
-print radix255(I)
+print(radix255(I))
diff --git a/src/ext/ht.h b/src/ext/ht.h
index 54e5eb7cba..4bfce36903 100644
--- a/src/ext/ht.h
+++ b/src/ext/ht.h
@@ -226,11 +226,16 @@ ht_string_hash(const char *s)
        (x) = HT_NEXT(name, head, x))
 
 #ifndef HT_NDEBUG
-#define HT_ASSERT_(x) tor_assert(x)
+#include "lib/err/torerr.h"
+#define HT_ASSERT_(x) raw_assert(x)
 #else
 #define HT_ASSERT_(x) (void)0
 #endif
 
+/* Macro put at the end of the end of a macro definition so that it
+ * consumes the following semicolon at file scope. Used only inside ht.h. */
+#define HT_EAT_SEMICOLON__ struct ht_semicolon_eater
+
 #define HT_PROTOTYPE(name, type, field, hashfn, eqfn)                   \
   int name##_HT_GROW(struct name *ht, unsigned min_capacity);           \
   void name##_HT_CLEAR(struct name *ht);                                \
@@ -370,7 +375,8 @@ ht_string_hash(const char *s)
   /* Return the next element in 'head' after 'elm', under the arbitrary \
    * order used by HT_START.  If there are no more elements, return     \
    * NULL.  If 'elm' is to be removed from the table, you must call     \
-   * this function for the next value before you remove it.             \
+   * this function for the next value before you remove it, or use      \
+   * HT_NEXT_RMV instead.                                               \
    */                                                                   \
   ATTR_UNUSED static inline struct type **                              \
   name##_HT_NEXT(struct name *head, struct type **elm)                  \
@@ -392,6 +398,8 @@ ht_string_hash(const char *s)
       return NULL;                                                      \
     }                                                                   \
   }                                                                     \
+  /* As HT_NEXT, but also remove the current element 'elm' from the     \
+   * table. */                                                          \
   ATTR_UNUSED static inline struct type **                              \
   name##_HT_NEXT_RMV(struct name *head, struct type **elm)              \
   {                                                                     \
@@ -409,7 +417,8 @@ ht_string_hash(const char *s)
       }                                                                 \
       return NULL;                                                      \
     }                                                                   \
-  }
+  }                                                                     \
+  HT_EAT_SEMICOLON__
 
 #define HT_GENERATE2(name, type, field, hashfn, eqfn, load, reallocarrayfn, \
                      freefn)                                            \
@@ -534,7 +543,8 @@ ht_string_hash(const char *s)
     if (n != head->hth_n_entries)                                       \
       return 6;                                                         \
     return 0;                                                           \
-  }
+  }                                                                     \
+  HT_EAT_SEMICOLON__
 
 #define HT_GENERATE(name, type, field, hashfn, eqfn, load, mallocfn,    \
                     reallocfn, freefn)                                  \
@@ -617,4 +627,3 @@ ht_string_hash(const char *s)
 */
 
 #endif
-
diff --git a/src/ext/include.am b/src/ext/include.am
index 6bdce2d79e..317e25d78e 100644
--- a/src/ext/include.am
+++ b/src/ext/include.am
@@ -143,6 +143,7 @@ noinst_HEADERS += $(ED25519_DONNA_HDRS)
 LIBED25519_DONNA=src/ext/ed25519/donna/libed25519_donna.a
 noinst_LIBRARIES += $(LIBED25519_DONNA)
 
+if BUILD_KECCAK_TINY
 src_ext_keccak_tiny_libkeccak_tiny_a_CFLAGS=\
   @CFLAGS_CONSTTIME@
 
@@ -156,6 +157,7 @@ noinst_HEADERS += $(LIBKECCAK_TINY_HDRS)
 
 LIBKECCAK_TINY=src/ext/keccak-tiny/libkeccak-tiny.a
 noinst_LIBRARIES += $(LIBKECCAK_TINY)
+endif
 
 EXTRA_DIST += \
 	src/ext/timeouts/bench/bench-add.lua 		\
diff --git a/src/ext/readpassphrase.c b/src/ext/readpassphrase.c
index e0df05d7b7..16611af1e2 100644
--- a/src/ext/readpassphrase.c
+++ b/src/ext/readpassphrase.c
@@ -30,7 +30,7 @@
 #include <signal.h>
 #include <ctype.h>
 #include <fcntl.h>
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>
diff --git a/src/ext/siphash.h b/src/ext/siphash.h
index 730e49937d..0207a959ff 100644
--- a/src/ext/siphash.h
+++ b/src/ext/siphash.h
@@ -1,6 +1,8 @@
 #ifndef SIPHASH_H
 #define SIPHASH_H
 
+#include <stdint.h>
+
 struct sipkey {
   uint64_t k0;
   uint64_t k1;
diff --git a/src/ext/timeouts/.may_include b/src/ext/timeouts/.may_include
new file mode 100644
index 0000000000..92c7116555
--- /dev/null
+++ b/src/ext/timeouts/.may_include
@@ -0,0 +1,5 @@
+orconfig.h
+
+ext/tor_queue.h
+ext/timeouts/*.h
+ext/timeouts/timeout-bitops.c
diff --git a/src/ext/timeouts/test-timeout.c b/src/ext/timeouts/test-timeout.c
index 8077129376..52d2e31e0c 100644
--- a/src/ext/timeouts/test-timeout.c
+++ b/src/ext/timeouts/test-timeout.c
@@ -4,7 +4,7 @@
 #include <assert.h>
 #include <limits.h>
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #define THE_END_OF_TIME ((timeout_t)-1)
 
diff --git a/src/ext/timeouts/timeout.c b/src/ext/timeouts/timeout.c
index d4b514d2c5..79fcc168ed 100644
--- a/src/ext/timeouts/timeout.c
+++ b/src/ext/timeouts/timeout.c
@@ -38,16 +38,16 @@
 
 #include <errno.h>     /* errno */
 
-#include "tor_queue.h" /* TAILQ(3) */
+#include "ext/tor_queue.h" /* TAILQ(3) */
 
-#include "timeout.h"
+#include "ext/timeouts/timeout.h"
 
 #ifndef TIMEOUT_DEBUG
 #define TIMEOUT_DEBUG 0
 #endif
 
 #if TIMEOUT_DEBUG - 0
-#include "timeout-debug.h"
+#include "ext/timeouts/timeout-debug.h"
 #endif
 
 #ifdef TIMEOUT_DISABLE_RELATIVE_ACCESS
@@ -141,7 +141,7 @@
 #define WHEEL_MASK (WHEEL_LEN - 1)
 #define TIMEOUT_MAX ((TIMEOUT_C(1) << (WHEEL_BIT * WHEEL_NUM)) - 1)
 
-#include "timeout-bitops.c"
+#include "ext/timeouts/timeout-bitops.c"
 
 #if WHEEL_BIT == 6
 #define ctz(n) ctz64(n)
@@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) {
 			timeout = MIN(_timeout, timeout);
 		}
 
-		relmask <<= WHEEL_BIT; 
+		relmask <<= WHEEL_BIT;
 		relmask |= WHEEL_MASK;
 	}
 
@@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) {
 TIMEOUT_PUBLIC int timeout_v_api(void) {
 	return TIMEOUT_V_API;
 } /* timeout_version() */
-
diff --git a/src/ext/timeouts/timeout.h b/src/ext/timeouts/timeout.h
index b35874e153..f1028bfc80 100644
--- a/src/ext/timeouts/timeout.h
+++ b/src/ext/timeouts/timeout.h
@@ -31,7 +31,7 @@
 
 #include <inttypes.h>   /* PRIu64 PRIx64 PRIX64 uint64_t */
 
-#include "tor_queue.h"  /* TAILQ(3) */
+#include "ext/tor_queue.h"  /* TAILQ(3) */
 
 
 /*
@@ -89,10 +89,10 @@ typedef uint64_t timeout_t;
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
 
 #ifndef TIMEOUT_CB_OVERRIDE
-struct timeout_cb {
+struct timeout_cb_t {
 	void (*fn)(void);
 	void *arg;
-}; /* struct timeout_cb */
+}; /* struct timeout_cb_t */
 #endif
 
 /*
@@ -125,7 +125,7 @@ struct timeout {
 	/* entry member for struct timeout_list lists */
 
 #ifndef TIMEOUT_DISABLE_CALLBACKS
-	struct timeout_cb callback;
+	struct timeout_cb_t callback;
 	/* optional callback information */
 #endif
 
@@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int);
 #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS
 TIMEOUT_PUBLIC bool timeout_pending(struct timeout *);
 /* true if on timing wheel, false otherwise */
- 
+
 TIMEOUT_PUBLIC bool timeout_expired(struct timeout *);
 /* true if on expired queue, false otherwise */
 
diff --git a/src/ext/tinytest.c b/src/ext/tinytest.c
index 16f11e4639..239fdd0a38 100644
--- a/src/ext/tinytest.c
+++ b/src/ext/tinytest.c
@@ -492,6 +492,12 @@ tinytest_set_test_skipped_(void)
 		cur_test_outcome = SKIP;
 }
 
+int
+tinytest_cur_test_has_failed(void)
+{
+	return (cur_test_outcome == FAIL);
+}
+
 char *
 tinytest_format_hex_(const void *val_, unsigned long len)
 {
diff --git a/src/ext/tinytest.h b/src/ext/tinytest.h
index ed07b26bc0..05c2fda052 100644
--- a/src/ext/tinytest.h
+++ b/src/ext/tinytest.h
@@ -72,6 +72,9 @@ struct testlist_alias_t {
 };
 #define END_OF_ALIASES { NULL, NULL }
 
+/** Return true iff the current test has failed. */
+int tinytest_cur_test_has_failed(void);
+
 /** Implementation: called from a test to indicate failure, before logging. */
 void tinytest_set_test_failed_(void);
 /** Implementation: called from a test to indicate that we're skipping. */
diff --git a/src/ext/tinytest_macros.h b/src/ext/tinytest_macros.h
index c3728d1fdd..6fc2cea2da 100644
--- a/src/ext/tinytest_macros.h
+++ b/src/ext/tinytest_macros.h
@@ -99,11 +99,11 @@
 
 /* Assert b, but do not stop the test if b fails.  Log msg on failure. */
 #define tt_want_msg(b, msg)			\
-	tt_want_(b, msg, );
+	tt_want_(b, msg, )
 
 /* Assert b and stop the test if b fails.  Log msg on failure. */
 #define tt_assert_msg(b, msg)			\
-	tt_want_(b, msg, TT_EXIT_TEST_FUNCTION);
+	tt_want_(b, msg, TT_EXIT_TEST_FUNCTION)
 
 /* Assert b, but do not stop the test if b fails. */
 #define tt_want(b)   tt_want_msg( (b), "want("#b")")
diff --git a/src/ext/trunnel/trunnel-impl.h b/src/ext/trunnel/trunnel-impl.h
index 15d1c8633e..52afa9ccd4 100644
--- a/src/ext/trunnel/trunnel-impl.h
+++ b/src/ext/trunnel/trunnel-impl.h
@@ -1,4 +1,4 @@
-/* trunnel-impl.h -- copied from Trunnel v1.5.2
+/* trunnel-impl.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.c b/src/ext/trunnel/trunnel.c
index 3ae3fe02c8..01a55c5bec 100644
--- a/src/ext/trunnel/trunnel.c
+++ b/src/ext/trunnel/trunnel.c
@@ -1,4 +1,4 @@
-/* trunnel.c -- copied from Trunnel v1.5.2
+/* trunnel.c -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/ext/trunnel/trunnel.h b/src/ext/trunnel/trunnel.h
index 9b708437b8..87c75f4ec3 100644
--- a/src/ext/trunnel/trunnel.h
+++ b/src/ext/trunnel/trunnel.h
@@ -1,4 +1,4 @@
-/* trunnel.h -- copied from Trunnel v1.5.2
+/* trunnel.h -- copied from Trunnel v1.5.3
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/feature/api/.may_include b/src/feature/api/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/api/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/api/feature_api.md b/src/feature/api/feature_api.md
new file mode 100644
index 0000000000..3065c000aa
--- /dev/null
+++ b/src/feature/api/feature_api.md
@@ -0,0 +1,2 @@
+@dir /feature/api
+@brief feature/api: In-process interface to starting/stopping Tor.
diff --git a/src/feature/api/include.am b/src/feature/api/include.am
new file mode 100644
index 0000000000..8d490458d4
--- /dev/null
+++ b/src/feature/api/include.am
@@ -0,0 +1,11 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/api/tor_api.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/api/tor_api_internal.h
+
+# This may someday want to be an installed file?
+noinst_HEADERS += src/feature/api/tor_api.h
diff --git a/src/feature/api/tor_api.c b/src/feature/api/tor_api.c
index 697397d46b..531793301e 100644
--- a/src/feature/api/tor_api.c
+++ b/src/feature/api/tor_api.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,10 +40,10 @@
 #define raw_socketpair tor_ersatz_socketpair
 #define raw_closesocket closesocket
 #define snprintf _snprintf
-#else
+#else /* !defined(_WIN32) */
 #define raw_socketpair socketpair
 #define raw_closesocket close
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
diff --git a/src/feature/api/tor_api.h b/src/feature/api/tor_api.h
index 2bf130c376..e9993bb0d5 100644
--- a/src/feature/api/tor_api.h
+++ b/src/feature/api/tor_api.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -55,7 +55,7 @@ typedef SOCKET tor_control_socket_t;
 #else
 typedef int tor_control_socket_t;
 #define INVALID_TOR_CONTROL_SOCKET (-1)
-#endif
+#endif /* defined(_WIN32) */
 
 /** DOCDOC */
 tor_control_socket_t tor_main_configuration_setup_control_socket(
diff --git a/src/feature/api/tor_api_internal.h b/src/feature/api/tor_api_internal.h
index 60e0f3aa59..d52b2caf44 100644
--- a/src/feature/api/tor_api_internal.h
+++ b/src/feature/api/tor_api_internal.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tor_api_internal.h
+ * @brief Internal declarations for in-process Tor API.
+ **/
+
 #ifndef TOR_API_INTERNAL_H
 #define TOR_API_INTERNAL_H
 
diff --git a/src/feature/client/.may_include b/src/feature/client/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/client/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/client/addressmap.c b/src/feature/client/addressmap.c
index bbe786a6a2..9ad2d7f934 100644
--- a/src/feature/client/addressmap.c
+++ b/src/feature/client/addressmap.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,8 +22,7 @@
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
-#include "feature/relay/dns.h"
+#include "feature/control/control_events.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
 
@@ -689,7 +688,7 @@ client_dns_set_addressmap_impl(entry_connection_t *for_conn,
   if (ttl<0)
     ttl = DEFAULT_DNS_TTL;
   else
-    ttl = dns_clip_ttl(ttl);
+    ttl = clip_dns_ttl(ttl);
 
   if (exitname) {
     /* XXXX fails to ever get attempts to get an exit address of
@@ -903,7 +902,7 @@ get_random_virtual_addr(const virtual_addr_conf_t *conf, tor_addr_t *addr_out)
   }
 
   if (ipv6)
-    tor_addr_from_ipv6_bytes(addr_out, (char*) bytes);
+    tor_addr_from_ipv6_bytes(addr_out, bytes);
   else
     tor_addr_from_ipv4n(addr_out, get_uint32(bytes));
 
diff --git a/src/feature/client/addressmap.h b/src/feature/client/addressmap.h
index 9179aef1d0..7f1024e09a 100644
--- a/src/feature/client/addressmap.h
+++ b/src/feature/client/addressmap.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file addressmap.h
+ * @brief Header for addressmap.c
+ **/
+
 #ifndef TOR_ADDRESSMAP_H
 #define TOR_ADDRESSMAP_H
 
@@ -62,4 +67,3 @@ STATIC void get_random_virtual_addr(const virtual_addr_conf_t *conf,
 #endif /* defined(ADDRESSMAP_PRIVATE) */
 
 #endif /* !defined(TOR_ADDRESSMAP_H) */
-
diff --git a/src/feature/client/bridges.c b/src/feature/client/bridges.c
index 626c5efcae..66b04f3bc2 100644
--- a/src/feature/client/bridges.c
+++ b/src/feature/client/bridges.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -348,7 +348,7 @@ int
 node_is_a_configured_bridge(const node_t *node)
 {
   /* First, let's try searching for a bridge with matching identity. */
-  if (BUG(tor_digest_is_zero(node->identity)))
+  if (BUG(fast_mem_is_zero(node->identity, DIGEST_LEN)))
     return 0;
 
   if (find_bridge_by_digest(node->identity) != NULL)
diff --git a/src/feature/client/bridges.h b/src/feature/client/bridges.h
index 27b2750a45..174149cf97 100644
--- a/src/feature/client/bridges.h
+++ b/src/feature/client/bridges.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/client/circpathbias.c b/src/feature/client/circpathbias.c
index 1743ab5a81..74260171fe 100644
--- a/src/feature/client/circpathbias.c
+++ b/src/feature/client/circpathbias.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -176,6 +176,7 @@ pathbias_get_scale_threshold(const or_options_t *options)
 static double
 pathbias_get_scale_ratio(const or_options_t *options)
 {
+  (void) options;
   /*
    * The scale factor is the denominator for our scaling
    * of circuit counts for our path bias window.
@@ -185,7 +186,8 @@ pathbias_get_scale_ratio(const or_options_t *options)
    */
   int denominator = networkstatus_get_param(NULL, "pb_scalefactor",
                               2, 2, INT32_MAX);
-  (void) options;
+  tor_assert(denominator > 0);
+
   /**
    * The mult factor is the numerator for our scaling
    * of circuit counts for our path bias window. It
@@ -301,7 +303,7 @@ pathbias_is_new_circ_attempt(origin_circuit_t *circ)
   return circ->cpath &&
          circ->cpath->next != circ->cpath &&
          circ->cpath->next->state == CPATH_STATE_AWAITING_KEYS;
-#else /* !(defined(N2N_TAGGING_IS_POSSIBLE)) */
+#else /* !defined(N2N_TAGGING_IS_POSSIBLE) */
   /* If tagging attacks are no longer possible, we probably want to
    * count bias from the first hop. However, one could argue that
    * timing-based tagging is still more useful than per-hop failure.
@@ -369,8 +371,9 @@ pathbias_should_count(origin_circuit_t *circ)
         !circ->build_state->onehop_tunnel) {
       if ((rate_msg = rate_limit_log(&count_limit, approx_time()))) {
         log_info(LD_BUG,
-               "One-hop circuit has length %d. Path state is %s. "
+               "One-hop circuit %d has length %d. Path state is %s. "
                "Circuit is a %s currently %s.%s",
+               circ->global_identifier,
                circ->build_state->desired_path_len,
                pathbias_state_to_string(circ->path_state),
                circuit_purpose_to_string(circ->base_.purpose),
@@ -398,12 +401,13 @@ pathbias_should_count(origin_circuit_t *circ)
   /* Check to see if the shouldcount result has changed due to a
    * unexpected purpose change that would affect our results */
   if (circ->pathbias_shouldcount == PATHBIAS_SHOULDCOUNT_IGNORED) {
-      log_info(LD_BUG,
-              "Circuit %d is now being counted despite being ignored "
-              "in the past. Purpose is %s, path state is %s",
-              circ->global_identifier,
-              circuit_purpose_to_string(circ->base_.purpose),
-              pathbias_state_to_string(circ->path_state));
+    log_info(LD_CIRC,
+            "Circuit %d is not being counted by pathbias because it was "
+            "ignored in the past. Purpose is %s, path state is %s",
+            circ->global_identifier,
+            circuit_purpose_to_string(circ->base_.purpose),
+            pathbias_state_to_string(circ->path_state));
+    return 0;
   }
   circ->pathbias_shouldcount = PATHBIAS_SHOULDCOUNT_COUNTED;
 
@@ -434,8 +438,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                                      approx_time()))) {
         log_info(LD_BUG,
-                "Opened circuit is in strange path state %s. "
+                "Opened circuit %d is in strange path state %s. "
                 "Circuit is a %s currently %s.%s",
+                circ->global_identifier,
                 pathbias_state_to_string(circ->path_state),
                 circuit_purpose_to_string(circ->base_.purpose),
                 circuit_state_to_string(circ->base_.state),
@@ -468,8 +473,9 @@ pathbias_count_build_attempt(origin_circuit_t *circ)
           if ((rate_msg = rate_limit_log(&circ_attempt_notice_limit,
                   approx_time()))) {
             log_info(LD_BUG,
-                   "Unopened circuit has strange path state %s. "
+                   "Unopened circuit %d has strange path state %s. "
                    "Circuit is a %s currently %s.%s",
+                   circ->global_identifier,
                    pathbias_state_to_string(circ->path_state),
                    circuit_purpose_to_string(circ->base_.purpose),
                    circuit_state_to_string(circ->base_.state),
@@ -538,8 +544,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
         if ((rate_msg = rate_limit_log(&success_notice_limit,
                 approx_time()))) {
           log_info(LD_BUG,
-              "Succeeded circuit is in strange path state %s. "
+              "Succeeded circuit %d is in strange path state %s. "
               "Circuit is a %s currently %s.%s",
+              circ->global_identifier,
               pathbias_state_to_string(circ->path_state),
               circuit_purpose_to_string(circ->base_.purpose),
               circuit_state_to_string(circ->base_.state),
@@ -574,8 +581,9 @@ pathbias_count_build_success(origin_circuit_t *circ)
       if ((rate_msg = rate_limit_log(&success_notice_limit,
               approx_time()))) {
         log_info(LD_BUG,
-            "Opened circuit is in strange path state %s. "
+            "Opened circuit %d is in strange path state %s. "
             "Circuit is a %s currently %s.%s",
+            circ->global_identifier,
             pathbias_state_to_string(circ->path_state),
             circuit_purpose_to_string(circ->base_.purpose),
             circuit_state_to_string(circ->base_.state),
@@ -601,8 +609,9 @@ pathbias_count_use_attempt(origin_circuit_t *circ)
 
   if (circ->path_state < PATH_STATE_BUILD_SUCCEEDED) {
     log_notice(LD_BUG,
-        "Used circuit is in strange path state %s. "
+        "Used circuit %d is in strange path state %s. "
         "Circuit is a %s currently %s.",
+        circ->global_identifier,
         pathbias_state_to_string(circ->path_state),
         circuit_purpose_to_string(circ->base_.purpose),
         circuit_state_to_string(circ->base_.state));
@@ -817,6 +826,11 @@ pathbias_send_usable_probe(circuit_t *circ)
   ocirc->pathbias_probe_nonce &= 0x00ffffff;
   probe_nonce = tor_dup_ip(ocirc->pathbias_probe_nonce);
 
+  if (!probe_nonce) {
+    log_err(LD_BUG, "Failed to generate nonce");
+    return -1;
+  }
+
   tor_snprintf(payload,RELAY_PAYLOAD_SIZE, "%s:25", probe_nonce);
   payload_len = (int)strlen(payload)+1;
 
diff --git a/src/feature/client/circpathbias.h b/src/feature/client/circpathbias.h
index a9a8d18df2..88cc982dd4 100644
--- a/src/feature/client/circpathbias.h
+++ b/src/feature/client/circpathbias.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/client/dnsserv.c b/src/feature/client/dnsserv.c
index cf593cfb68..c1981ecde0 100644
--- a/src/feature/client/dnsserv.c
+++ b/src/feature/client/dnsserv.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,8 +26,9 @@
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/or/policies.h"
 
 #include "feature/control/control_connection_st.h"
@@ -213,6 +214,9 @@ dnsserv_launch_request(const char *name, int reverse,
   edge_connection_t *conn;
   char *q_name;
 
+  /* Launching a request for a user counts as user activity. */
+  note_user_activity(approx_time());
+
   /* Make a new dummy AP connection, and attach the request to it. */
   entry_conn = entry_connection_new(CONN_TYPE_AP, AF_INET);
   entry_conn->entry_cfg.dns_request = 1;
@@ -234,7 +238,7 @@ dnsserv_launch_request(const char *name, int reverse,
     TO_CONN(conn)->port = control_conn->base_.port;
     TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
   }
-#else /* !(defined(AF_UNIX)) */
+#else /* !defined(AF_UNIX) */
   TO_CONN(conn)->port = control_conn->base_.port;
   TO_CONN(conn)->address = tor_addr_to_str_dup(&control_conn->base_.addr);
 #endif /* defined(AF_UNIX) */
diff --git a/src/feature/client/dnsserv.h b/src/feature/client/dnsserv.h
index fff1ed2adb..4011cb4e02 100644
--- a/src/feature/client/dnsserv.h
+++ b/src/feature/client/dnsserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/client/entrynodes.c b/src/feature/client/entrynodes.c
index 6c7f8057bd..70ef64cc86 100644
--- a/src/feature/client/entrynodes.c
+++ b/src/feature/client/entrynodes.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -47,8 +47,7 @@
  * As a persistent ordered list whose elements are taken from the
  * sampled set, we track a CONFIRMED GUARDS LIST.  A guard becomes
  * confirmed when we successfully build a circuit through it, and decide
- * to use that circuit.  We order the guards on this list by the order
- * in which they became confirmed.
+ * to use that circuit.
  *
  * And as a final group, we have an ordered list of PRIMARY GUARDS,
  * whose elements are taken from the filtered set. We prefer
@@ -59,7 +58,7 @@
  *
  * To build circuits, we take a primary guard if possible -- or a
  * reachable filtered confirmed guard if no primary guard is possible --
- * or a random reachable filtered guard otherwise.  If the guard is
+ * or the first (by sampled order) filtered guard otherwise.  If the guard is
  * primary, we can use the circuit immediately on success.  Otherwise,
  * the guard is now "pending" -- we won't use its circuit unless all
  * of the circuits we're trying to build through better guards have
@@ -92,14 +91,18 @@
  * [x] Whenever we remove a guard from the sample, remove it from the primary
  * and confirmed lists.
  *
- * [x] When we make a guard confirmed, update the primary list.
+ * [x] When we make a guard confirmed, update the primary list, and sort them
+ * by sampled order.
  *
  * [x] When we make a guard filtered or unfiltered, update the primary list.
  *
  * [x] When we are about to pick a guard, make sure that the primary list is
  * full.
  *
- * [x] Before calling sample_reachable_filtered_entry_guards(), make sure
+ * [x] When we update the confirmed list, or when we re-build the primary list
+ * and detect a change, we sort those lists by sampled_idx
+ *
+ * [x] Before calling first_reachable_filtered_entry_guard(), make sure
  * that the filtered, primary, and confirmed flags are up-to-date.
  *
  * [x] Call entry_guard_consider_retry every time we are about to check
@@ -114,7 +117,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "app/config/statefile.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
@@ -128,7 +131,7 @@
 #include "feature/client/circpathbias.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/microdesc.h"
@@ -172,6 +175,7 @@ static entry_guard_t *get_sampled_guard_by_bridge_addr(guard_selection_t *gs,
                                               const tor_addr_port_t *addrport);
 static int entry_guard_obeys_restriction(const entry_guard_t *guard,
                                          const entry_guard_restriction_t *rst);
+static int compare_guards_by_sampled_idx(const void **a_, const void **b_);
 
 /** Return 0 if we should apply guardfraction information found in the
  *  consensus. A specific consensus can be specified with the
@@ -890,6 +894,7 @@ entry_guard_add_to_sample_impl(guard_selection_t *gs,
   tor_free(guard->sampled_by_version);
   guard->sampled_by_version = tor_strdup(VERSION);
   guard->currently_listed = 1;
+  guard->sampled_idx = gs->next_sampled_idx++;
   guard->confirmed_idx = -1;
 
   /* non-persistent fields */
@@ -901,6 +906,11 @@ entry_guard_add_to_sample_impl(guard_selection_t *gs,
   guard->in_selection = gs;
   entry_guard_set_filtered_flags(get_options(), gs, guard);
   entry_guards_changed_for_guard_selection(gs);
+
+  /* Just added this guard to the sampled set and hence it might be used as a
+   * guard in the future: send GUARD NEW control event. */
+  control_event_guard(guard->nickname, guard->identity, "NEW");
+
   return guard;
 }
 
@@ -1038,7 +1048,7 @@ get_max_sample_size(guard_selection_t *gs,
  * Return a smartlist of the all the guards that are not currently
  * members of the sample (GUARDS - SAMPLED_GUARDS).  The elements of
  * this list are node_t pointers in the non-bridge case, and
- * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out/b>
+ * bridge_info_t pointers in the bridge case.  Set *<b>n_guards_out</b>
  * to the number of guards that we found in GUARDS, including those
  * that were already sampled.
  */
@@ -1383,7 +1393,7 @@ sampled_guards_prune_obsolete_entries(guard_selection_t *gs,
 
     if (rmv) {
       ++n_changes;
-      SMARTLIST_DEL_CURRENT(gs->sampled_entry_guards, guard);
+      SMARTLIST_DEL_CURRENT_KEEPORDER(gs->sampled_entry_guards, guard);
       remove_guard_from_confirmed_and_primary_lists(gs, guard);
       entry_guard_free(guard);
     }
@@ -1707,7 +1717,7 @@ entry_guards_update_filtered_sets(guard_selection_t *gs)
 }
 
 /**
- * Return a random guard from the reachable filtered sample guards
+ * Return the first sampled guard from the reachable filtered sample guards
  * in <b>gs</b>, subject to the exclusion rules listed in <b>flags</b>.
  * Return NULL if no such guard can be found.
  *
@@ -1718,7 +1728,7 @@ entry_guards_update_filtered_sets(guard_selection_t *gs)
  * violate it.
  **/
 STATIC entry_guard_t *
-sample_reachable_filtered_entry_guards(guard_selection_t *gs,
+first_reachable_filtered_entry_guard(guard_selection_t *gs,
                                        const entry_guard_restriction_t *rst,
                                        unsigned flags)
 {
@@ -1771,7 +1781,17 @@ sample_reachable_filtered_entry_guards(guard_selection_t *gs,
            flags, smartlist_len(reachable_filtered_sample));
 
   if (smartlist_len(reachable_filtered_sample)) {
-    result = smartlist_choose(reachable_filtered_sample);
+    /**
+     * Get the first guard of the filtered set builds from
+     * sampled_entry_guards. Proposal 310 suggests this design to overcome
+     * performance and security issues linked to the previous selection
+     * method. The guard selected here should be filtered out if this function
+     * is called again in the same context. I.e., if we filter guards to add
+     * them into some list X, then the guards from list X will be filtered out
+     * when this function is called again. Hence it requires setting exclude
+     * flags in a appropriate way (depending of the context of the caller).
+     */
+    result = smartlist_get(reachable_filtered_sample, 0);
     log_info(LD_GUARD, "  (Selected %s.)",
              result ? entry_guard_describe(result) : "<null>");
   }
@@ -1780,10 +1800,6 @@ sample_reachable_filtered_entry_guards(guard_selection_t *gs,
   return result;
 }
 
-/**
- * Helper: compare two entry_guard_t by their confirmed_idx values.
- * Used to sort the confirmed list.
- */
 static int
 compare_guards_by_confirmed_idx(const void **a_, const void **b_)
 {
@@ -1795,6 +1811,21 @@ compare_guards_by_confirmed_idx(const void **a_, const void **b_)
   else
     return 0;
 }
+/**
+ * Helper: compare two entry_guard_t by their sampled_idx values.
+ * Used to sort the sampled list
+ */
+static int
+compare_guards_by_sampled_idx(const void **a_, const void **b_)
+{
+  const entry_guard_t *a = *a_, *b = *b_;
+  if (a->sampled_idx < b->sampled_idx)
+    return -1;
+  else if (a->sampled_idx > b->sampled_idx)
+    return 1;
+  else
+    return 0;
+}
 
 /**
  * Find the confirmed guards from among the sampled guards in <b>gs</b>,
@@ -1811,7 +1842,7 @@ entry_guards_update_confirmed(guard_selection_t *gs)
   } SMARTLIST_FOREACH_END(guard);
 
   smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_confirmed_idx);
-
+  /** Needed to keep a dense array of confirmed_idx */
   int any_changed = 0;
   SMARTLIST_FOREACH_BEGIN(gs->confirmed_entry_guards, entry_guard_t *, guard) {
     if (guard->confirmed_idx != guard_sl_idx) {
@@ -1821,6 +1852,8 @@ entry_guards_update_confirmed(guard_selection_t *gs)
   } SMARTLIST_FOREACH_END(guard);
 
   gs->next_confirmed_idx = smartlist_len(gs->confirmed_entry_guards);
+  // We need the confirmed list to always be give guards in sampled order
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_sampled_idx);
 
   if (any_changed) {
     entry_guards_changed_for_guard_selection(gs);
@@ -1849,6 +1882,9 @@ make_guard_confirmed(guard_selection_t *gs, entry_guard_t *guard)
 
   guard->confirmed_idx = gs->next_confirmed_idx++;
   smartlist_add(gs->confirmed_entry_guards, guard);
+  /** The confirmation ordering might not be the sample ording. We need to
+   * reorder */
+  smartlist_sort(gs->confirmed_entry_guards, compare_guards_by_sampled_idx);
 
   // This confirmed guard might kick something else out of the primary
   // guards.
@@ -1912,7 +1948,7 @@ entry_guards_update_primary(guard_selection_t *gs)
 
   /* Finally, fill out the list with sampled guards. */
   while (smartlist_len(new_primary_guards) < N_PRIMARY_GUARDS) {
-    entry_guard_t *guard = sample_reachable_filtered_entry_guards(gs, NULL,
+    entry_guard_t *guard = first_reachable_filtered_entry_guard(gs, NULL,
                                             SAMPLE_EXCLUDE_CONFIRMED|
                                             SAMPLE_EXCLUDE_PRIMARY|
                                             SAMPLE_NO_UPDATE_PRIMARY);
@@ -1943,6 +1979,7 @@ entry_guards_update_primary(guard_selection_t *gs)
                g->confirmed_idx >= 0 ? " (confirmed)" : "",
                g->is_filtered_guard ? "" : " (excluded by filter)");
     } SMARTLIST_FOREACH_END(g);
+    smartlist_sort(new_primary_guards, compare_guards_by_sampled_idx);
   }
 
   smartlist_free(old_primary_guards);
@@ -1974,10 +2011,12 @@ get_retry_schedule(time_t failing_since, time_t now,
   const struct {
     time_t maximum; int primary_delay; int nonprimary_delay;
   } delays[] = {
+    // clang-format off
     { SIX_HOURS,    10*60,  1*60*60 },
     { FOUR_DAYS,    90*60,  4*60*60 },
     { SEVEN_DAYS, 4*60*60, 18*60*60 },
     { TIME_MAX,   9*60*60, 36*60*60 }
+    // clang-format on
   };
 
   unsigned i;
@@ -2053,10 +2092,15 @@ select_primary_guard_for_circuit(guard_selection_t *gs,
 
   SMARTLIST_FOREACH_BEGIN(gs->primary_entry_guards, entry_guard_t *, guard) {
     entry_guard_consider_retry(guard);
-    if (! entry_guard_obeys_restriction(guard, rst))
+    if (!entry_guard_obeys_restriction(guard, rst)) {
+      log_info(LD_GUARD, "Entry guard %s doesn't obey restriction, we test the"
+          " next one", entry_guard_describe(guard));
       continue;
+    }
     if (guard->is_reachable != GUARD_REACHABLE_NO) {
       if (need_descriptor && !guard_has_descriptor(guard)) {
+        log_info(LD_GUARD, "Guard %s does not have a descriptor",
+            entry_guard_describe(guard));
         continue;
       }
       *state_out = GUARD_CIRC_STATE_USABLE_ON_COMPLETION;
@@ -2069,9 +2113,11 @@ select_primary_guard_for_circuit(guard_selection_t *gs,
 
   if (smartlist_len(usable_primary_guards)) {
     chosen_guard = smartlist_choose(usable_primary_guards);
+    log_info(LD_GUARD,
+        "Selected primary guard %s for circuit from a list size of %d.",
+        entry_guard_describe(chosen_guard),
+        smartlist_len(usable_primary_guards));
     smartlist_free(usable_primary_guards);
-    log_info(LD_GUARD, "Selected primary guard %s for circuit.",
-             entry_guard_describe(chosen_guard));
   }
 
   smartlist_free(usable_primary_guards);
@@ -2116,10 +2162,10 @@ select_confirmed_guard_for_circuit(guard_selection_t *gs,
 }
 
 /**
- * For use with a circuit, pick a confirmed usable filtered guard
- * at random. Update the <b>last_tried_to_connect</b> time and the
- * <b>is_pending</b> fields of the guard as appropriate. Set <b>state_out</b>
- * to the new guard-state of the circuit.
+ * For use with a circuit, pick a usable filtered guard. Update the
+ * <b>last_tried_to_connect</b> time and the <b>is_pending</b> fields of the
+ * guard as appropriate. Set <b>state_out</b> to the new guard-state of the
+ * circuit.
  */
 static entry_guard_t *
 select_filtered_guard_for_circuit(guard_selection_t *gs,
@@ -2132,7 +2178,7 @@ select_filtered_guard_for_circuit(guard_selection_t *gs,
   unsigned flags = 0;
   if (need_descriptor)
     flags |= SAMPLE_EXCLUDE_NO_DESCRIPTOR;
-  chosen_guard = sample_reachable_filtered_entry_guards(gs,
+  chosen_guard = first_reachable_filtered_entry_guard(gs,
                                                  rst,
                                                  SAMPLE_EXCLUDE_CONFIRMED |
                                                  SAMPLE_EXCLUDE_PRIMARY |
@@ -2146,7 +2192,7 @@ select_filtered_guard_for_circuit(guard_selection_t *gs,
   chosen_guard->last_tried_to_connect = approx_time();
   *state_out = GUARD_CIRC_STATE_USABLE_IF_NO_BETTER_GUARD;
   log_info(LD_GUARD, "No primary or confirmed guards available. Selected "
-           "random guard %s for circuit. Will try other guards before "
+           "guard %s for circuit. Will try other guards before "
            "using this circuit.",
            entry_guard_describe(chosen_guard));
   return chosen_guard;
@@ -2187,8 +2233,8 @@ select_entry_guard_for_circuit(guard_selection_t *gs,
   if (chosen_guard)
     return chosen_guard;
 
-  /* "Otherwise, if there is no such entry, select a member at
-      random from {USABLE_FILTERED_GUARDS}." */
+  /* "Otherwise, if there is no such entry, select a member
+   * {USABLE_FILTERED_GUARDS} following the sample ordering" */
   chosen_guard = select_filtered_guard_for_circuit(gs, usage, rst, state_out);
 
   if (chosen_guard == NULL) {
@@ -2218,6 +2264,9 @@ entry_guards_note_guard_failure(guard_selection_t *gs,
   if (guard->failing_since == 0)
     guard->failing_since = approx_time();
 
+  /* This guard not reachable: send GUARD DOWN event */
+  control_event_guard(guard->nickname, guard->identity, "DOWN");
+
   log_info(LD_GUARD, "Recorded failure for %s%sguard %s",
            guard->is_primary?"primary ":"",
            guard->confirmed_idx>=0?"confirmed ":"",
@@ -2243,6 +2292,11 @@ entry_guards_note_guard_success(guard_selection_t *gs,
   const time_t last_time_on_internet = gs->last_time_on_internet;
   gs->last_time_on_internet = approx_time();
 
+  /* If guard was not already marked as reachable, send a GUARD UP signal */
+  if (guard->is_reachable != GUARD_REACHABLE_YES) {
+    control_event_guard(guard->nickname, guard->identity, "UP");
+  }
+
   guard->is_reachable = GUARD_REACHABLE_YES;
   guard->failing_since = 0;
   guard->is_pending = 0;
@@ -2771,10 +2825,12 @@ entry_guards_update_all(guard_selection_t *gs)
 
 /**
  * Return a newly allocated string for encoding the persistent parts of
- * <b>guard</b> to the state file.
+ * <b>guard</b> to the state file. <b>dense_sampled_idx</b> refers to the
+ * sampled_idx made dense for this <b>guard</b>. Encoding all guards should
+ * lead to a dense array of sampled_idx in the state file.
  */
 STATIC char *
-entry_guard_encode_for_state(entry_guard_t *guard)
+entry_guard_encode_for_state(entry_guard_t *guard, int dense_sampled_idx)
 {
   /*
    * The meta-format we use is K=V K=V K=V... where K can be any
@@ -2803,7 +2859,8 @@ entry_guard_encode_for_state(entry_guard_t *guard)
 
   format_iso_time_nospace(tbuf, guard->sampled_on_date);
   smartlist_add_asprintf(result, "sampled_on=%s", tbuf);
-
+  // Replacing the sampled_idx by dense array
+  smartlist_add_asprintf(result, "sampled_idx=%d", dense_sampled_idx);
   if (guard->sampled_by_version) {
     smartlist_add_asprintf(result, "sampled_by=%s",
                            guard->sampled_by_version);
@@ -2859,6 +2916,78 @@ entry_guard_encode_for_state(entry_guard_t *guard)
 }
 
 /**
+ * Extract key=val from the state string <b>s</b> and duplicate the value to
+ * some string target declared in entry_guard_parse_from_state
+ */
+static void
+parse_from_state_set_vals(const char *s, smartlist_t *entries, smartlist_t
+    *extra, strmap_t *vals)
+{
+    smartlist_split_string(entries, s, " ",
+                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+
+    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
+      const char *eq = strchr(entry, '=');
+      if (!eq) {
+        smartlist_add(extra, entry);
+        continue;
+      }
+      char *key = tor_strndup(entry, eq-entry);
+      char **target = strmap_get(vals, key);
+      if (target == NULL || *target != NULL) {
+        /* unrecognized or already set */
+        smartlist_add(extra, entry);
+        tor_free(key);
+        continue;
+      }
+
+      *target = tor_strdup(eq+1);
+      tor_free(key);
+      tor_free(entry);
+    } SMARTLIST_FOREACH_END(entry);
+}
+
+/**
+ * Handle part of the parsing state file logic, focused on time related things
+ */
+static void
+parse_from_state_handle_time(entry_guard_t *guard, char *sampled_on, char
+    *unlisted_since, char *confirmed_on)
+{
+#define HANDLE_TIME(field) do {                                 \
+    if (field) {                                                \
+      int r = parse_iso_time_nospace(field, &field ## _time);   \
+      if (r < 0) {                                              \
+        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
+                 #field, escaped(field));                       \
+        field##_time = -1;                                      \
+      }                                                         \
+    }                                                           \
+  } while (0)
+
+  time_t sampled_on_time = 0;
+  time_t unlisted_since_time = 0;
+  time_t confirmed_on_time = 0;
+
+  HANDLE_TIME(sampled_on);
+  HANDLE_TIME(unlisted_since);
+  HANDLE_TIME(confirmed_on);
+
+  if (sampled_on_time <= 0)
+    sampled_on_time = approx_time();
+  if (unlisted_since_time < 0)
+    unlisted_since_time = 0;
+  if (confirmed_on_time < 0)
+    confirmed_on_time = 0;
+
+  #undef HANDLE_TIME
+
+  guard->sampled_on_date = sampled_on_time;
+  guard->unlisted_since_date = unlisted_since_time;
+  guard->confirmed_on_date = confirmed_on_time;
+}
+
+/**
  * Given a string generated by entry_guard_encode_for_state(), parse it
  * (if possible) and return an entry_guard_t object for it.  Return NULL
  * on complete failure.
@@ -2874,6 +3003,7 @@ entry_guard_parse_from_state(const char *s)
   char *rsa_id = NULL;
   char *nickname = NULL;
   char *sampled_on = NULL;
+  char *sampled_idx = NULL;
   char *sampled_by = NULL;
   char *unlisted_since = NULL;
   char *listed  = NULL;
@@ -2890,6 +3020,7 @@ entry_guard_parse_from_state(const char *s)
   char *pb_collapsed_circuits = NULL;
   char *pb_unusable_circuits = NULL;
   char *pb_timeouts = NULL;
+  int invalid_sampled_idx = get_max_sample_size_absolute();
 
   /* Split up the entries.  Put the ones we know about in strings and the
    * rest in "extra". */
@@ -2903,6 +3034,7 @@ entry_guard_parse_from_state(const char *s)
     FIELD(rsa_id);
     FIELD(nickname);
     FIELD(sampled_on);
+    FIELD(sampled_idx);
     FIELD(sampled_by);
     FIELD(unlisted_since);
     FIELD(listed);
@@ -2918,29 +3050,8 @@ entry_guard_parse_from_state(const char *s)
     FIELD(pb_unusable_circuits);
     FIELD(pb_timeouts);
 #undef FIELD
-
-    smartlist_split_string(entries, s, " ",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
-    SMARTLIST_FOREACH_BEGIN(entries, char *, entry) {
-      const char *eq = strchr(entry, '=');
-      if (!eq) {
-        smartlist_add(extra, entry);
-        continue;
-      }
-      char *key = tor_strndup(entry, eq-entry);
-      char **target = strmap_get(vals, key);
-      if (target == NULL || *target != NULL) {
-        /* unrecognized or already set */
-        smartlist_add(extra, entry);
-        tor_free(key);
-        continue;
-      }
-
-      *target = tor_strdup(eq+1);
-      tor_free(key);
-      tor_free(entry);
-    } SMARTLIST_FOREACH_END(entry);
+    /* Extract from s the key=val that we recognize, put the others in extra*/
+    parse_from_state_set_vals(s, entries, extra, vals);
 
     smartlist_free(entries);
     strmap_free(vals, NULL);
@@ -2988,43 +3099,12 @@ entry_guard_parse_from_state(const char *s)
   }
 
   /* Process the various time fields. */
-
-#define HANDLE_TIME(field) do {                                 \
-    if (field) {                                                \
-      int r = parse_iso_time_nospace(field, &field ## _time);   \
-      if (r < 0) {                                              \
-        log_warn(LD_CIRC, "Unable to parse %s %s from guard",   \
-                 #field, escaped(field));                       \
-        field##_time = -1;                                      \
-      }                                                         \
-    }                                                           \
-  } while (0)
-
-  time_t sampled_on_time = 0;
-  time_t unlisted_since_time = 0;
-  time_t confirmed_on_time = 0;
-
-  HANDLE_TIME(sampled_on);
-  HANDLE_TIME(unlisted_since);
-  HANDLE_TIME(confirmed_on);
-
-  if (sampled_on_time <= 0)
-    sampled_on_time = approx_time();
-  if (unlisted_since_time < 0)
-    unlisted_since_time = 0;
-  if (confirmed_on_time < 0)
-    confirmed_on_time = 0;
-
-  #undef HANDLE_TIME
-
-  guard->sampled_on_date = sampled_on_time;
-  guard->unlisted_since_date = unlisted_since_time;
-  guard->confirmed_on_date = confirmed_on_time;
+  parse_from_state_handle_time(guard, sampled_on, unlisted_since,
+      confirmed_on);
 
   /* Take sampled_by_version verbatim. */
   guard->sampled_by_version = sampled_by;
   sampled_by = NULL; /* prevent free */
-
   /* Listed is a boolean */
   if (listed && strcmp(listed, "0"))
     guard->currently_listed = 1;
@@ -3042,6 +3122,29 @@ entry_guard_parse_from_state(const char *s)
     }
   }
 
+  if (sampled_idx) {
+    int ok = 1;
+    long idx = tor_parse_long(sampled_idx, 10, 0, INT_MAX, &ok, NULL);
+    if (!ok) {
+      log_warn(LD_GUARD, "Guard has invalid sampled_idx %s",
+          escaped(sampled_idx));
+      /* set it to a idx higher than the max sample size */
+      guard->sampled_idx = invalid_sampled_idx++;
+    } else {
+      guard->sampled_idx = (int)idx;
+    }
+  } else if (confirmed_idx) {
+    /* This state has been written by an older Tor version which did not have
+     * sample ordering  */
+
+    guard->sampled_idx = guard->confirmed_idx;
+  } else {
+    log_info(LD_GUARD, "The state file seems to be into a status that could"
+             " yield to weird entry node selection: we're missing both a"
+             " sampled_idx and a confirmed_idx.");
+    guard->sampled_idx = invalid_sampled_idx++;
+  }
+
   /* Anything we didn't recognize gets crammed together */
   if (smartlist_len(extra) > 0) {
     guard->extra_state_fields = smartlist_join_strings(extra, " ", 0, NULL);
@@ -3096,6 +3199,7 @@ entry_guard_parse_from_state(const char *s)
   tor_free(listed);
   tor_free(confirmed_on);
   tor_free(confirmed_idx);
+  tor_free(sampled_idx);
   tor_free(bridge_addr);
   tor_free(pb_use_attempts);
   tor_free(pb_use_successes);
@@ -3125,13 +3229,15 @@ entry_guards_update_guards_in_state(or_state_t *state)
   config_line_t **nextline = &lines;
 
   SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+    int i = 0;
     SMARTLIST_FOREACH_BEGIN(gs->sampled_entry_guards, entry_guard_t *, guard) {
       if (guard->is_persistent == 0)
         continue;
       *nextline = tor_malloc_zero(sizeof(config_line_t));
       (*nextline)->key = tor_strdup("Guard");
-      (*nextline)->value = entry_guard_encode_for_state(guard);
+      (*nextline)->value = entry_guard_encode_for_state(guard, i);
       nextline = &(*nextline)->next;
+      i++;
     } SMARTLIST_FOREACH_END(guard);
   } SMARTLIST_FOREACH_END(gs);
 
@@ -3184,6 +3290,14 @@ entry_guards_load_guards_from_state(or_state_t *state, int set)
       tor_assert(gs);
       smartlist_add(gs->sampled_entry_guards, guard);
       guard->in_selection = gs;
+      /* Recompute the next_sampled_id from the state. We do not assume that
+       * sampled guards appear in the correct order within the file, and we
+       * need to know what would be the next sampled idx to give to any
+       * new sampled guard (i.e., max of guard->sampled_idx + 1)*/
+      if (gs->next_sampled_idx <= guard->sampled_idx) {
+        gs->next_sampled_idx = guard->sampled_idx + 1;
+      }
+
     } else {
       entry_guard_free(guard);
     }
@@ -3191,6 +3305,10 @@ entry_guards_load_guards_from_state(or_state_t *state, int set)
 
   if (set) {
     SMARTLIST_FOREACH_BEGIN(guard_contexts, guard_selection_t *, gs) {
+      /** Guards should be in sample order within the file, but it is maybe
+       * better NOT to assume that. Let's order them before updating lists
+       */
+      smartlist_sort(gs->sampled_entry_guards, compare_guards_by_sampled_idx);
       entry_guards_update_all(gs);
     } SMARTLIST_FOREACH_END(gs);
   }
diff --git a/src/feature/client/entrynodes.h b/src/feature/client/entrynodes.h
index 4e5eb4e960..4b236dc80c 100644
--- a/src/feature/client/entrynodes.h
+++ b/src/feature/client/entrynodes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "lib/container/handles.h"
 
 /* Forward declare for guard_selection_t; entrynodes.c has the real struct */
-typedef struct guard_selection_s guard_selection_t;
+typedef struct guard_selection_t guard_selection_t;
 
 /* Forward declare for entry_guard_t; the real declaration is private. */
 typedef struct entry_guard_t entry_guard_t;
@@ -28,7 +28,7 @@ typedef struct circuit_guard_state_t circuit_guard_state_t;
    private. */
 typedef struct entry_guard_restriction_t entry_guard_restriction_t;
 
-/* Information about a guard's pathbias status.
+/** Information about a guard's pathbias status.
  * These fields are used in circpathbias.c to try to detect entry
  * nodes that are failing circuits at a suspicious frequency.
  */
@@ -117,6 +117,13 @@ struct entry_guard_t {
    * confirmed guard. */
   time_t confirmed_on_date; /* 0 if not confirmed */
   /**
+   * In what order was this guard sampled? Guards with
+   * lower indices appear earlier on the sampled list, the confirmed list and
+   * the primary list as a result of Prop 310
+   */
+  int sampled_idx;
+
+  /**
    * In what order was this guard confirmed? Guards with lower indices
    * appear earlier on the confirmed list.  If the confirmed list is compacted,
    * this field corresponds to the index of this guard on the confirmed list.
@@ -210,7 +217,7 @@ typedef enum guard_selection_type_t {
  * See the module documentation for entrynodes.c for more information
  * about guard selection algorithms.
  */
-struct guard_selection_s {
+struct guard_selection_t {
   /**
    * The name for this guard-selection object. (Must not contain spaces).
    */
@@ -242,8 +249,9 @@ struct guard_selection_s {
    * Ordered list (from highest to lowest priority) of guards that we
    * have successfully contacted and decided to use. Every member of
    * this list is a member of sampled_entry_guards. Every member should
-   * have confirmed_on_date set, and have confirmed_idx greater than
-   * any earlier member of the list.
+   * have confirmed_on_date set.
+   * The ordering of the list should be by sampled idx. The reasoning behind
+   * it is linked to Proposal 310.
    *
    * This list is persistent. It is a subset of the elements in
    * sampled_entry_guards, and its pointers point to elements of
@@ -271,6 +279,12 @@ struct guard_selection_s {
    * confirmed_entry_guards receive? */
   int next_confirmed_idx;
 
+  /** What sampled_idx value should the next-added member of
+   * sampled_entry_guards receive? This should follow the size of the sampled
+   * list until sampled relays get pruned for some reason
+   */
+  int next_sampled_idx;
+
 };
 
 struct entry_guard_handle_t;
@@ -515,7 +529,8 @@ MOCK_DECL(STATIC circuit_guard_state_t *,
 STATIC entry_guard_t *entry_guard_add_to_sample(guard_selection_t *gs,
                                                 const node_t *node);
 STATIC entry_guard_t *entry_guards_expand_sample(guard_selection_t *gs);
-STATIC char *entry_guard_encode_for_state(entry_guard_t *guard);
+STATIC char *entry_guard_encode_for_state(entry_guard_t *guard, int
+    dense_sampled_index);
 STATIC entry_guard_t *entry_guard_parse_from_state(const char *s);
 #define entry_guard_free(e) \
   FREE_AND_NULL(entry_guard_t, entry_guard_free_, (e))
@@ -523,7 +538,7 @@ STATIC void entry_guard_free_(entry_guard_t *e);
 STATIC void entry_guards_update_filtered_sets(guard_selection_t *gs);
 STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
 /**
- * @name Flags for sample_reachable_filtered_entry_guards()
+ * @name Flags for first_reachable_filtered_entry_guard()
  */
 /**@{*/
 #define SAMPLE_EXCLUDE_CONFIRMED   (1u<<0)
@@ -532,7 +547,7 @@ STATIC int entry_guards_all_primary_guards_are_down(guard_selection_t *gs);
 #define SAMPLE_NO_UPDATE_PRIMARY   (1u<<3)
 #define SAMPLE_EXCLUDE_NO_DESCRIPTOR (1u<<4)
 /**@}*/
-STATIC entry_guard_t *sample_reachable_filtered_entry_guards(
+STATIC entry_guard_t *first_reachable_filtered_entry_guard(
                                     guard_selection_t *gs,
                                     const entry_guard_restriction_t *rst,
                                     unsigned flags);
diff --git a/src/feature/client/feature_client.md b/src/feature/client/feature_client.md
new file mode 100644
index 0000000000..dd4bf78ec8
--- /dev/null
+++ b/src/feature/client/feature_client.md
@@ -0,0 +1,5 @@
+@dir /feature/client
+@brief feature/client: Client-specific code
+
+(There is also a bunch of client-specific code in other modules.)
+
diff --git a/src/feature/client/include.am b/src/feature/client/include.am
new file mode 100644
index 0000000000..53c9f047d4
--- /dev/null
+++ b/src/feature/client/include.am
@@ -0,0 +1,20 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/client/addressmap.c		\
+	src/feature/client/bridges.c		\
+	src/feature/client/circpathbias.c	\
+	src/feature/client/dnsserv.c		\
+	src/feature/client/entrynodes.c		\
+	src/feature/client/proxymode.c		\
+	src/feature/client/transports.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/client/addressmap.h			\
+	src/feature/client/bridges.h			\
+	src/feature/client/circpathbias.h		\
+	src/feature/client/dnsserv.h			\
+	src/feature/client/entrynodes.h			\
+	src/feature/client/proxymode.h			\
+	src/feature/client/transports.h
diff --git a/src/feature/client/proxymode.c b/src/feature/client/proxymode.c
new file mode 100644
index 0000000000..aa269ec7fb
--- /dev/null
+++ b/src/feature/client/proxymode.c
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file proxymode.c
+ * @brief Determine whether we are trying to be a proxy.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/port_cfg_st.h"
+#include "feature/client/proxymode.h"
+
+/** Return true iff we are trying to proxy client connections. */
+int
+proxy_mode(const or_options_t *options)
+{
+  (void)options;
+  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
+    if (p->type == CONN_TYPE_AP_LISTENER ||
+        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
+        p->type == CONN_TYPE_AP_DNS_LISTENER ||
+        p->type == CONN_TYPE_AP_NATD_LISTENER)
+      return 1;
+  } SMARTLIST_FOREACH_END(p);
+  return 0;
+}
diff --git a/src/feature/client/proxymode.h b/src/feature/client/proxymode.h
new file mode 100644
index 0000000000..30be08ff78
--- /dev/null
+++ b/src/feature/client/proxymode.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file proxymode.h
+ * \brief Header file for proxymode.c.
+ **/
+
+#ifndef TOR_PROXYMODE_H
+#define TOR_PROXYMODE_H
+
+int proxy_mode(const or_options_t *options);
+
+#endif /* !defined(TOR_PROXYMODE_H) */
diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c
index 0d1342c87b..2bdc0ae151 100644
--- a/src/feature/client/transports.c
+++ b/src/feature/client/transports.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -97,15 +97,20 @@
 #include "core/or/circuitbuild.h"
 #include "feature/client/transports.h"
 #include "feature/relay/router.h"
+#include "feature/relay/relay_find_addr.h"
+/* 31851: split the server transport code out of the client module */
+#include "feature/relay/transport_config.h"
 #include "app/config/statefile.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
+#include "lib/process/process.h"
 #include "lib/process/env.h"
-#include "lib/process/subprocess.h"
 
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp);
 
 static inline int proxy_configuration_finished(const managed_proxy_t *mp);
@@ -127,6 +132,8 @@ static void parse_method_error(const char *line, int is_server_method);
 #define PROTO_SMETHODS_DONE "SMETHODS DONE"
 #define PROTO_PROXY_DONE "PROXY DONE"
 #define PROTO_PROXY_ERROR "PROXY-ERROR"
+#define PROTO_LOG "LOG"
+#define PROTO_STATUS "STATUS"
 
 /** The first and only supported - at the moment - configuration
     protocol version. */
@@ -490,8 +497,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
   tor_assert(mp->conf_state == PT_PROTO_COMPLETED);
 
   /* destroy the process handle and terminate the process. */
-  tor_process_handle_destroy(mp->process_handle, 1);
-  mp->process_handle = NULL;
+  process_set_data(mp->process, NULL);
+  process_terminate(mp->process);
 
   /* destroy all its registered transports, since we will no longer
      use them. */
@@ -520,34 +527,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp)
 static int
 launch_managed_proxy(managed_proxy_t *mp)
 {
-  int retval;
-
-  process_environment_t *env = create_managed_proxy_environment(mp);
-
-#ifdef _WIN32
-  /* Passing NULL as lpApplicationName makes Windows search for the .exe */
-  retval = tor_spawn_background(NULL,
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#else /* !(defined(_WIN32)) */
-  retval = tor_spawn_background(mp->argv[0],
-                                (const char **)mp->argv,
-                                env,
-                                &mp->process_handle);
-#endif /* defined(_WIN32) */
-
-  process_environment_free(env);
-
-  if (retval == PROCESS_STATUS_ERROR) {
-    log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.",
+  tor_assert(mp);
+
+  smartlist_t *env = create_managed_proxy_environment(mp);
+
+  /* Configure our process. */
+  process_set_data(mp->process, mp);
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback);
+  process_set_exit_callback(mp->process, managed_proxy_exit_callback);
+  process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE);
+  process_reset_environment(mp->process, env);
+
+  /* Cleanup our env. */
+  SMARTLIST_FOREACH(env, char *, x, tor_free(x));
+  smartlist_free(env);
+
+  /* Skip the argv[0] as we get that from process_new(argv[0]). */
+  for (int i = 1; mp->argv[i] != NULL; ++i)
+    process_append_argument(mp->process, mp->argv[i]);
+
+  if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.",
              mp->argv[0]);
     return -1;
   }
 
-  log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.",
-           mp->argv[0], tor_process_get_pid(mp->process_handle));
-
+  log_info(LD_CONFIG,
+           "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.",
+           mp->argv[0], process_get_pid(mp->process));
   mp->conf_state = PT_PROTO_LAUNCHED;
 
   return 0;
@@ -615,10 +623,6 @@ pt_configure_remaining_proxies(void)
 STATIC int
 configure_proxy(managed_proxy_t *mp)
 {
-  int configuration_finished = 0;
-  smartlist_t *proxy_output = NULL;
-  enum stream_status stream_status = 0;
-
   /* if we haven't launched the proxy yet, do it now */
   if (mp->conf_state == PT_PROTO_INFANT) {
     if (launch_managed_proxy(mp) < 0) { /* launch fail */
@@ -629,45 +633,8 @@ configure_proxy(managed_proxy_t *mp)
   }
 
   tor_assert(mp->conf_state != PT_PROTO_INFANT);
-  tor_assert(mp->process_handle);
-
-  proxy_output =
-    tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle),
-                              &stream_status);
-  if (!proxy_output) { /* failed to get input from proxy */
-    if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */
-      mp->conf_state = PT_PROTO_BROKEN;
-      log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' "
-               "is '%s'. Most probably the managed proxy stopped running. "
-               "This might be a bug of the managed proxy, a bug of Tor, or "
-               "a misconfiguration. Please enable logging on your managed "
-               "proxy and check the logs for errors.",
-               mp->argv[0], stream_status_to_string(stream_status));
-    }
-
-    goto done;
-  }
-
-  /* Handle lines. */
-  SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) {
-    handle_proxy_line(line, mp);
-    if (proxy_configuration_finished(mp))
-      goto done;
-  } SMARTLIST_FOREACH_END(line);
-
- done:
-  /* if the proxy finished configuring, exit the loop. */
-  if (proxy_configuration_finished(mp)) {
-    handle_finished_proxy(mp);
-    configuration_finished = 1;
-  }
-
-  if (proxy_output) {
-    SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp));
-    smartlist_free(proxy_output);
-  }
-
-  return configuration_finished;
+  tor_assert(mp->process);
+  return mp->conf_state == PT_PROTO_COMPLETED;
 }
 
 /** Register server managed proxy <b>mp</b> transports to state */
@@ -748,8 +715,14 @@ managed_proxy_destroy(managed_proxy_t *mp,
   /* free the outgoing proxy URI */
   tor_free(mp->proxy_uri);
 
-  tor_process_handle_destroy(mp->process_handle, also_terminate_process);
-  mp->process_handle = NULL;
+  /* do we want to terminate our process if it's still running? */
+  if (also_terminate_process && mp->process) {
+    /* Note that we do not call process_free(mp->process) here because we let
+     * the exit handler in managed_proxy_exit_callback() return `true` which
+     * makes the process subsystem deallocate the process_t. */
+    process_set_data(mp->process, NULL);
+    process_terminate(mp->process);
+  }
 
   tor_free(mp);
 }
@@ -763,6 +736,9 @@ get_pt_proxy_uri(void)
   const or_options_t *options = get_options();
   char *uri = NULL;
 
+  /* XXX: Currently TCPProxy is not supported in TOR_PT_PROXY because
+   * there isn't a standard URI scheme for some proxy protocols, such as
+   * haproxy. */
   if (options->Socks4Proxy || options->Socks5Proxy || options->HTTPSProxy) {
     char addr[TOR_ADDR_BUF_LEN+1];
 
@@ -945,21 +921,16 @@ handle_proxy_line(const char *line, managed_proxy_t *mp)
 
     parse_proxy_error(line);
     goto err;
-  } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) {
-    /* managed proxy launch failed: parse error message to learn why. */
-    int retval, child_state, saved_errno;
-    retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x",
-                        &child_state, &saved_errno);
-    if (retval == 2) {
-      log_warn(LD_GENERAL,
-               "Could not launch managed proxy executable at '%s' ('%s').",
-               mp->argv[0], strerror(saved_errno));
-    } else { /* failed to parse error message */
-      log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.",
-               mp->argv[0]);
-    }
 
-    mp->conf_state = PT_PROTO_FAILED_LAUNCH;
+    /* We check for the additional " " after the PROTO_LOG * PROTO_STATUS
+     * string to make sure we can later extend this big if/else-if table with
+     * something that begins with "LOG" without having to get the order right.
+     * */
+  } else if (!strcmpstart(line, PROTO_LOG " ")) {
+    parse_log_line(line, mp);
+    return;
+  } else if (!strcmpstart(line, PROTO_STATUS " ")) {
+    parse_status_line(line, mp);
     return;
   }
 
@@ -1182,6 +1153,121 @@ parse_proxy_error(const char *line)
            line+strlen(PROTO_PROXY_ERROR)+1);
 }
 
+/** Parses a LOG <b>line</b> and emit log events accordingly. */
+STATIC void
+parse_log_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *log_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_LOG) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_LOG);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_LOG) + 1;
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid LOG message: %s",
+             mp->argv[0], data);
+    goto done;
+  }
+
+  const config_line_t *severity = config_line_find(values, "SEVERITY");
+  const config_line_t *message = config_line_find(values, "MESSAGE");
+
+  /* Check if we got a message. */
+  if (! message) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "MESSAGE: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Check if severity is there and whether it's valid. */
+  if (! severity) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line without "
+                    "SEVERITY: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  int log_severity = managed_proxy_severity_parse(severity->value);
+
+  if (log_severity == -1) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a LOG line with an "
+                    "invalid severity level: %s",
+                    mp->argv[0], severity->value);
+    goto done;
+  }
+
+  tor_log(log_severity, LD_PT, "Managed proxy \"%s\": %s",
+          mp->argv[0], message->value);
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  log_message = kvline_encode(values, KV_QUOTED);
+
+  /* Emit control port event. */
+  control_event_pt_log(log_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(log_message);
+}
+
+/** Parses a STATUS <b>line</b> and emit control events accordingly. */
+STATIC void
+parse_status_line(const char *line, managed_proxy_t *mp)
+{
+  tor_assert(line);
+  tor_assert(mp);
+
+  config_line_t *values = NULL;
+  char *status_message = NULL;
+
+  if (strlen(line) < (strlen(PROTO_STATUS) + 1)) {
+    log_warn(LD_PT, "Managed proxy sent us a %s line "
+                    "with missing argument.", PROTO_STATUS);
+    goto done;
+  }
+
+  const char *data = line + strlen(PROTO_STATUS) + 1;
+
+  values = kvline_parse(data, KV_QUOTED);
+
+  if (! values) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote an invalid "
+             "STATUS message: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* We check if we received the TRANSPORT parameter, which is the only
+   * *required* value. */
+  const config_line_t *type = config_line_find(values, "TRANSPORT");
+
+  if (! type) {
+    log_warn(LD_PT, "Managed proxy \"%s\" wrote a STATUS line without "
+                    "TRANSPORT: %s", mp->argv[0], escaped(data));
+    goto done;
+  }
+
+  /* Prepend the PT name. */
+  config_line_prepend(&values, "PT", mp->argv[0]);
+  status_message = kvline_encode(values, KV_QUOTED);
+
+  /* We have checked that TRANSPORT is there, we can now emit the STATUS event
+   * via the control port. */
+  control_event_pt_status(status_message);
+
+ done:
+  config_free_lines(values);
+  tor_free(status_message);
+}
+
 /** Return a newly allocated string that tor should place in
  * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server
  * manged proxy in <b>mp</b>. Return NULL if no such options are found. */
@@ -1199,7 +1285,7 @@ get_transport_options_for_server_proxy(const managed_proxy_t *mp)
       string. */
   SMARTLIST_FOREACH_BEGIN(mp->transports_to_launch, const char *, transport) {
     smartlist_t *options_tmp_sl = NULL;
-    options_tmp_sl = get_options_for_server_transport(transport);
+    options_tmp_sl = pt_get_options_for_server_transport(transport);
     if (!options_tmp_sl)
       continue;
 
@@ -1257,7 +1343,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp)
 
 /** Return a newly allocated process_environment_t * for <b>mp</b>'s
  * process. */
-static process_environment_t *
+static smartlist_t *
 create_managed_proxy_environment(const managed_proxy_t *mp)
 {
   const or_options_t *options = get_options();
@@ -1272,8 +1358,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
   /* The final environment to be passed to mp. */
   smartlist_t *merged_env_vars = get_current_process_environment_variables();
 
-  process_environment_t *env;
-
   {
     char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */
     smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp);
@@ -1337,8 +1421,10 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
         smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=%s",
                                ext_or_addrport_tmp);
       }
-      smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s",
-                             cookie_file_loc);
+      if (cookie_file_loc) {
+        smartlist_add_asprintf(envs, "TOR_PT_AUTH_COOKIE_FILE=%s",
+                               cookie_file_loc);
+      }
 
       tor_free(ext_or_addrport_tmp);
       tor_free(cookie_file_loc);
@@ -1346,11 +1432,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     } else {
       smartlist_add_asprintf(envs, "TOR_PT_EXTENDED_SERVER_PORT=");
     }
-
-    /* All new versions of tor will keep stdin open, so PTs can use it
-     * as a reliable termination detection mechanism.
-     */
-    smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
   } else {
     /* If ClientTransportPlugin has a HTTPS/SOCKS proxy configured, set the
      * TOR_PT_PROXY line.
@@ -1361,19 +1442,19 @@ create_managed_proxy_environment(const managed_proxy_t *mp)
     }
   }
 
+  /* All new versions of tor will keep stdin open, so PTs can use it
+   * as a reliable termination detection mechanism.
+   */
+  smartlist_add_asprintf(envs, "TOR_PT_EXIT_ON_STDIN_CLOSE=1");
+
   SMARTLIST_FOREACH_BEGIN(envs, const char *, env_var) {
     set_environment_variable_in_smartlist(merged_env_vars, env_var,
                                           tor_free_, 1);
   } SMARTLIST_FOREACH_END(env_var);
 
-  env = process_environment_make(merged_env_vars);
-
   smartlist_free(envs);
 
-  SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x));
-  smartlist_free(merged_env_vars);
-
-  return env;
+  return merged_env_vars;
 }
 
 /** Create and return a new managed proxy for <b>transport</b> using
@@ -1392,6 +1473,7 @@ managed_proxy_create(const smartlist_t *with_transport_list,
   mp->argv = proxy_argv;
   mp->transports = smartlist_new();
   mp->proxy_uri = get_pt_proxy_uri();
+  mp->process = process_new(proxy_argv[0]);
 
   mp->transports_to_launch = smartlist_new();
   SMARTLIST_FOREACH(with_transport_list, const char *, transport,
@@ -1736,3 +1818,95 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape)
 
   return new_string;
 }
+
+/** Callback function that is called when our PT process have data on its
+ * stdout. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stdout_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (mp == NULL)
+    return;
+
+  handle_proxy_line(line, mp);
+
+  if (proxy_configuration_finished(mp))
+    handle_finished_proxy(mp);
+}
+
+/** Callback function that is called when our PT process have data on its
+ * stderr. Our process can be found in <b>process</b>, the data can be found in
+ * <b>line</b> and the length of our line is given in <b>size</b>. */
+STATIC void
+managed_proxy_stderr_callback(process_t *process,
+                              const char *line,
+                              size_t size)
+{
+  tor_assert(process);
+  tor_assert(line);
+
+  (void)size;
+
+  managed_proxy_t *mp = process_get_data(process);
+
+  if (BUG(mp == NULL))
+    return;
+
+  log_info(LD_PT,
+           "Managed proxy at '%s' reported via standard error: %s",
+           mp->argv[0], line);
+}
+
+/** Callback function that is called when our PT process terminates. The
+ * process exit code can be found in <b>exit_code</b> and our process can be
+ * found in <b>process</b>. Returns true iff we want the process subsystem to
+ * free our process_t handle for us. */
+STATIC bool
+managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_warn(LD_PT,
+          "Pluggable Transport process terminated with status code %" PRIu64,
+          exit_code);
+
+  /* Returning true here means that the process subsystem will take care of
+   * calling process_free() on our process_t. */
+  return true;
+}
+
+/** Returns a valid integer log severity level from <b>severity</b> that
+ * is compatible with Tor's logging functions. Returns <b>-1</b> on
+ * error. */
+STATIC int
+managed_proxy_severity_parse(const char *severity)
+{
+  tor_assert(severity);
+
+  /* Slightly different than log.c's parse_log_level :-( */
+  if (! strcmp(severity, "debug"))
+    return LOG_DEBUG;
+
+  if (! strcmp(severity, "info"))
+    return LOG_INFO;
+
+  if (! strcmp(severity, "notice"))
+    return LOG_NOTICE;
+
+  if (! strcmp(severity, "warning"))
+    return LOG_WARN;
+
+  if (! strcmp(severity, "error"))
+    return LOG_ERR;
+
+  return -1;
+}
diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h
index e2fa45828f..1ed942c175 100644
--- a/src/feature/client/transports.h
+++ b/src/feature/client/transports.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,8 @@
 #ifndef TOR_TRANSPORTS_H
 #define TOR_TRANSPORTS_H
 
+#include "lib/process/process.h"
+
 /** Represents a pluggable transport used by a bridge. */
 typedef struct transport_t {
   /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */
@@ -81,7 +83,7 @@ enum pt_proto_state {
   PT_PROTO_FAILED_LAUNCH /* failed while launching */
 };
 
-struct process_handle_t;
+struct process_t;
 
 /** Structure containing information of a managed proxy. */
 typedef struct {
@@ -94,10 +96,8 @@ typedef struct {
 
   int is_server; /* is it a server proxy? */
 
-  /* A pointer to the process handle of this managed proxy. */
-  struct process_handle_t *process_handle;
-
-  int pid; /* The Process ID this managed proxy is using. */
+  /* A pointer to the process of this managed proxy. */
+  struct process_t *process;
 
   /** Boolean: We are re-parsing our config, and we are going to
    * remove this managed proxy if we don't find it any transport
@@ -128,6 +128,8 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp);
 STATIC void parse_env_error(const char *line);
 STATIC void parse_proxy_error(const char *line);
 STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_log_line(const char *line, managed_proxy_t *mp);
+STATIC void parse_status_line(const char *line, managed_proxy_t *mp);
 STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp);
 
 STATIC void managed_proxy_destroy(managed_proxy_t *mp,
@@ -142,6 +144,12 @@ STATIC char* get_pt_proxy_uri(void);
 
 STATIC void free_execve_args(char **arg);
 
+STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t);
+STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t);
+STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t);
+
+STATIC int managed_proxy_severity_parse(const char *);
+
 #endif /* defined(PT_PRIVATE) */
 
 #endif /* !defined(TOR_TRANSPORTS_H) */
diff --git a/src/feature/control/.may_include b/src/feature/control/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/control/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/control/btrack.c b/src/feature/control/btrack.c
new file mode 100644
index 0000000000..405630ecd4
--- /dev/null
+++ b/src/feature/control/btrack.c
@@ -0,0 +1,65 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack.c
+ * \brief Bootstrap trackers
+ *
+ * Initializes and shuts down the specific bootstrap trackers.  These
+ * trackers help the reporting of bootstrap progress by maintaining
+ * state information about various subsystems within tor.  When the
+ * correct state changes happen, these trackers emit controller
+ * events.
+ *
+ * These trackers avoid referring directly to the internals of state
+ * objects of other subsystems.
+ *
+ * btrack_circuit.c contains the tracker for origin circuits.
+ *
+ * btrack_orconn.c contains the tracker for OR connections.
+ *
+ * Eventually there will be a tracker for directory downloads as well.
+ **/
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_sys.h"
+#include "lib/pubsub/pubsub.h"
+#include "lib/subsys/subsys.h"
+
+static int
+btrack_init(void)
+{
+  if (btrack_orconn_init())
+    return -1;
+
+  return 0;
+}
+
+static void
+btrack_fini(void)
+{
+  btrack_orconn_fini();
+  btrack_circ_fini();
+}
+
+static int
+btrack_add_pubsub(pubsub_connector_t *connector)
+{
+  if (btrack_orconn_add_pubsub(connector))
+    return -1;
+  if (btrack_circ_add_pubsub(connector))
+    return -1;
+
+  return 0;
+}
+
+const subsys_fns_t sys_btrack = {
+  .name = "btrack",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = 55,
+  .initialize = btrack_init,
+  .shutdown = btrack_fini,
+  .add_pubsub = btrack_add_pubsub,
+};
diff --git a/src/feature/control/btrack_circuit.c b/src/feature/control/btrack_circuit.c
new file mode 100644
index 0000000000..be51b51046
--- /dev/null
+++ b/src/feature/control/btrack_circuit.c
@@ -0,0 +1,166 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.c
+ * \brief Bootstrap tracker for origin circuits
+ *
+ * Track state changes of origin circuits, as published by the circuit
+ * subsystem.
+ **/
+
+#include "core/or/or.h"
+
+#include "core/or/ocirc_event.h"
+
+#include "feature/control/btrack_circuit.h"
+#include "feature/control/control.h"
+#include "lib/log/log.h"
+
+/** Pair of a best origin circuit GID with its state or status */
+typedef struct btc_best_t {
+  uint32_t gid;
+  int val;
+} btc_best_t;
+
+/** GID and state of the best origin circuit we've seen so far */
+static btc_best_t best_any_state = { 0, -1 };
+/** GID and state of the best application circuit we've seen so far */
+static btc_best_t best_ap_state = { 0, -1 };
+/** GID and status of the best origin circuit we've seen so far */
+static btc_best_t best_any_evtype = { 0, -1 };
+/** GID and status of the best application circuit we've seen so far */
+static btc_best_t best_ap_evtype = { 0, -1 };
+
+/** Reset cached "best" values */
+static void
+btc_reset_bests(void)
+{
+  best_any_state.gid = best_ap_state.gid = 0;
+  best_any_state.val = best_ap_state.val = -1;
+  best_any_evtype.gid = best_ap_state.gid = 0;
+  best_any_evtype.val = best_ap_evtype.val = -1;
+}
+
+/** True if @a state is a "better" origin circuit state than @a best->val */
+static bool
+btc_state_better(int state, const btc_best_t *best)
+{
+  return state > best->val;
+}
+
+/**
+ * Definine an ordering on circuit status events
+ *
+ * The CIRC_EVENT_ constants aren't sorted in a useful order, so this
+ * array helps to decode them.  This approach depends on the statuses
+ * being nonnegative and dense.
+ **/
+static int circ_event_order[] = {
+  [CIRC_EVENT_FAILED] = -1,
+  [CIRC_EVENT_CLOSED] = -1,
+  [CIRC_EVENT_LAUNCHED] = 1,
+  [CIRC_EVENT_EXTENDED] = 2,
+  [CIRC_EVENT_BUILT] = 3,
+};
+#define N_CIRC_EVENT_ORDER \
+  (sizeof(circ_event_order) / sizeof(circ_event_order[0]))
+
+/** True if @a state is a "better" origin circuit event status than @a
+    best->val */
+static bool
+btc_evtype_better(int state, const btc_best_t *best)
+{
+  if (state < 0)
+    return false;
+  if (best->val < 0)
+    return true;
+
+  tor_assert(state >= 0 && (unsigned)state < N_CIRC_EVENT_ORDER);
+  tor_assert(best->val >= 0 && (unsigned)best->val < N_CIRC_EVENT_ORDER);
+  return circ_event_order[state] > circ_event_order[best->val];
+}
+
+static bool
+btc_update_state(const ocirc_state_msg_t *msg, btc_best_t *best,
+                 const char *type)
+{
+  if (btc_state_better(msg->state, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s state %d->%d gid=%"PRIu32, type,
+             best->val, msg->state, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->state;
+    return true;
+  }
+  return false;
+}
+
+static bool
+btc_update_evtype(const ocirc_cevent_msg_t *msg, btc_best_t *best,
+                  const char *type)
+{
+  if (btc_evtype_better(msg->evtype, best)) {
+    log_info(LD_BTRACK, "CIRC BEST_%s evtype %d->%d gid=%"PRIu32, type,
+             best->val, msg->evtype, msg->gid);
+    best->gid = msg->gid;
+    best->val = msg->evtype;
+    return true;
+  }
+  return false;
+}
+
+DECLARE_SUBSCRIBE(ocirc_state, btc_state_rcvr);
+DECLARE_SUBSCRIBE(ocirc_cevent, btc_cevent_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, btc_chan_rcvr);
+
+static void
+btc_state_rcvr(const msg_t *msg, const ocirc_state_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" state=%d onehop=%d",
+            arg->gid, arg->state, arg->onehop);
+
+  btc_update_state(arg, &best_any_state, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_state(arg, &best_ap_state, "AP");
+}
+
+static void
+btc_cevent_rcvr(const msg_t *msg, const ocirc_cevent_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" evtype=%d reason=%d onehop=%d",
+            arg->gid, arg->evtype, arg->reason, arg->onehop);
+
+  btc_update_evtype(arg, &best_any_evtype, "ANY");
+  if (arg->onehop)
+    return;
+  btc_update_evtype(arg, &best_ap_evtype, "AP");
+}
+
+static void
+btc_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  (void)msg;
+  log_debug(LD_BTRACK, "CIRC gid=%"PRIu32" chan=%"PRIu64" onehop=%d",
+            arg->gid, arg->chan, arg->onehop);
+}
+
+int
+btrack_circ_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_cevent))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_state))
+    return -1;
+  return 0;
+}
+
+void
+btrack_circ_fini(void)
+{
+  btc_reset_bests();
+}
diff --git a/src/feature/control/btrack_circuit.h b/src/feature/control/btrack_circuit.h
new file mode 100644
index 0000000000..75699450c3
--- /dev/null
+++ b/src/feature/control/btrack_circuit.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_circuit.h
+ * \brief Header file for btrack_circuit.c
+ **/
+
+#ifndef TOR_BTRACK_CIRCUIT_H
+#define TOR_BTRACK_CIRCUIT_H
+
+#include "lib/pubsub/pubsub.h"
+
+int btrack_circ_init(void);
+void btrack_circ_fini(void);
+int btrack_circ_add_pubsub(pubsub_connector_t *);
+
+#endif /* !defined(TOR_BTRACK_CIRCUIT_H) */
diff --git a/src/feature/control/btrack_orconn.c b/src/feature/control/btrack_orconn.c
new file mode 100644
index 0000000000..104c8af230
--- /dev/null
+++ b/src/feature/control/btrack_orconn.c
@@ -0,0 +1,206 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.c
+ * \brief Bootstrap tracker for OR connections
+ *
+ * Track state changes of OR connections, as published by the
+ * connection subsystem.  Also track circuit launch events, because
+ * they're one of the few ways to discover the association between a
+ * channel (and OR connection) and a circuit.
+ *
+ * We track all OR connections that we receive events for, whether or
+ * not they're carrying origin circuits.  (An OR connection might
+ * carry origin circuits only after we first find out about that
+ * connection.)
+ *
+ * All origin ORCONN events update the "any" state variables, while
+ * only application ORCONN events update the "ap" state variables (and
+ * also update the "any") variables.
+ *
+ * We do this because we want to report the first increments of
+ * connection progress as the earliest bootstrap phases.  This results
+ * in a better user experience because failures here translate into
+ * zero or very small amounts of displayed progress, instead of
+ * progress stuck near completion.  The first connection to a relay
+ * might be a one-hop circuit for directory lookups, or it might be a
+ * connection for an application circuit because we already have
+ * enough directory info to build an application circuit.
+ *
+ * We call functions in btrack_orconn_cevent.c to generate the actual
+ * controller events, because some of the state decoding we need to do
+ * is complicated.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+#include "lib/pubsub/pubsub.h"
+
+DECLARE_SUBSCRIBE(orconn_state, bto_state_rcvr);
+DECLARE_SUBSCRIBE(orconn_status, bto_status_rcvr);
+DECLARE_SUBSCRIBE(ocirc_chan, bto_chan_rcvr);
+
+/** Pair of a best ORCONN GID and with its state */
+typedef struct bto_best_t {
+  uint64_t gid;
+  int state;
+} bto_best_t;
+
+/** GID and state of the best ORCONN we've seen so far */
+static bto_best_t best_any = { 0, -1 };
+/** GID and state of the best application circuit ORCONN we've seen so far */
+static bto_best_t best_ap = { 0, -1 };
+
+/**
+ * Update a cached state of a best ORCONN progress we've seen so far.
+ *
+ * Return true if the new state is better than the old.
+ **/
+static bool
+bto_update_best(const bt_orconn_t *bto, bto_best_t *best, const char *type)
+{
+  if (bto->state < best->state)
+    return false;
+  /* Update even if we won't change best->state, because it's more
+   * recent information that a particular connection transitioned to
+   * that state. */
+  best->gid = bto->gid;
+  if (bto->state > best->state) {
+    log_info(LD_BTRACK, "ORCONN BEST_%s state %d->%d gid=%"PRIu64, type,
+             best->state, bto->state, bto->gid);
+    best->state = bto->state;
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Update cached states of best ORCONN progress we've seen
+ *
+ * Only update the application ORCONN state if we know it's carrying
+ * an application circuit.
+ **/
+static void
+bto_update_bests(const bt_orconn_t *bto)
+{
+  tor_assert(bto->is_orig);
+
+  if (bto_update_best(bto, &best_any, "ANY"))
+    bto_cevent_anyconn(bto);
+  if (!bto->is_onehop && bto_update_best(bto, &best_ap, "AP"))
+    bto_cevent_apconn(bto);
+}
+
+/** Reset cached "best" values */
+static void
+bto_reset_bests(void)
+{
+  best_any.gid = best_ap.gid = 0;
+  best_any.state = best_ap.state = -1;
+}
+
+/**
+ * Update cached states of ORCONNs from the incoming message.  This
+ * message comes from code in connection_or.c.
+ **/
+static void
+bto_state_rcvr(const msg_t *msg, const orconn_state_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(arg->gid, arg->chan);
+  log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" chan=%"PRIu64
+            " proxy_type=%d state=%d",
+            arg->gid, arg->chan, arg->proxy_type, arg->state);
+  bto->proxy_type = arg->proxy_type;
+  bto->state = arg->state;
+  if (bto->is_orig)
+    bto_update_bests(bto);
+}
+
+/**
+ * Delete a cached ORCONN state if we get an incoming message saying
+ * the ORCONN is failed or closed.  This message comes from code in
+ * control.c.
+ **/
+static void
+bto_status_rcvr(const msg_t *msg, const orconn_status_msg_t *arg)
+{
+  (void)msg;
+  switch (arg->status) {
+  case OR_CONN_EVENT_FAILED:
+  case OR_CONN_EVENT_CLOSED:
+    log_info(LD_BTRACK, "ORCONN DELETE gid=%"PRIu64" status=%d reason=%d",
+             arg->gid, arg->status, arg->reason);
+    return bto_delete(arg->gid);
+  default:
+    break;
+  }
+}
+
+/**
+ * Create or update a cached ORCONN state for a newly launched
+ * connection, including whether it's launched by an origin circuit
+ * and whether it's a one-hop circuit.
+ **/
+static void
+bto_chan_rcvr(const msg_t *msg, const ocirc_chan_msg_t *arg)
+{
+  bt_orconn_t *bto;
+
+  (void)msg;
+  bto = bto_find_or_new(0, arg->chan);
+  if (!bto->is_orig || (bto->is_onehop && !arg->onehop)) {
+    log_debug(LD_BTRACK, "ORCONN LAUNCH chan=%"PRIu64" onehop=%d",
+              arg->chan, arg->onehop);
+  }
+  bto->is_orig = true;
+  if (!arg->onehop)
+    bto->is_onehop = false;
+  bto_update_bests(bto);
+}
+
+/**
+ * Initialize the hash maps and subscribe to ORCONN and origin
+ * circuit events.
+ **/
+int
+btrack_orconn_init(void)
+{
+  bto_init_maps();
+
+  return 0;
+}
+
+int
+btrack_orconn_add_pubsub(pubsub_connector_t *connector)
+{
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_state))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, orconn, orconn_status))
+    return -1;
+  if (DISPATCH_ADD_SUB(connector, ocirc, ocirc_chan))
+    return -1;
+  return 0;
+}
+
+/** Clear the hash maps and reset the "best" states */
+void
+btrack_orconn_fini(void)
+{
+  bto_clear_maps();
+  bto_reset_bests();
+  bto_cevent_reset();
+}
diff --git a/src/feature/control/btrack_orconn.h b/src/feature/control/btrack_orconn.h
new file mode 100644
index 0000000000..8b3d8be37d
--- /dev/null
+++ b/src/feature/control/btrack_orconn.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn.h
+ * \brief Header file for btrack_orconn.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_H
+#define TOR_BTRACK_ORCONN_H
+
+#include "lib/pubsub/pubsub.h"
+
+#ifdef BTRACK_ORCONN_PRIVATE
+
+#include "ht.h"
+
+/**
+ * Structure for tracking OR connection states
+ *
+ * This gets linked into two hash maps: one with connection IDs, and
+ * another with channel IDs.
+ **/
+typedef struct bt_orconn_t {
+  HT_ENTRY(bt_orconn_t) node;   /**< Hash map entry indexed by gid */
+  HT_ENTRY(bt_orconn_t) chan_node; /**< Hash map entry indexed by channel ID */
+  uint64_t gid;                    /**< Global ID of this ORCONN */
+  uint64_t chan;                   /**< Channel ID, if known */
+  int proxy_type;                  /**< Proxy type */
+  uint8_t state;                   /**< State of this ORCONN */
+  bool is_orig;             /**< Does this carry an origin circuit? */
+  bool is_onehop;           /**< Is this for a one-hop circuit? */
+} bt_orconn_t;
+
+#endif /* defined(BTRACK_ORCONN_PRIVATE) */
+
+int btrack_orconn_init(void);
+int btrack_orconn_add_pubsub(pubsub_connector_t *);
+void btrack_orconn_fini(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_H) */
diff --git a/src/feature/control/btrack_orconn_cevent.c b/src/feature/control/btrack_orconn_cevent.c
new file mode 100644
index 0000000000..d11be59280
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.c
@@ -0,0 +1,161 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.c
+ * \brief Emit bootstrap status events for OR connections
+ *
+ * We do some decoding of the raw OR_CONN_STATE_* values.  For
+ * example, OR_CONN_STATE_CONNECTING means the first TCP connect()
+ * completing, regardless of whether it's directly to a relay instead
+ * of a proxy or a PT.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "core/or/orconn_event.h"
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_cevent.h"
+#include "feature/control/control_events.h"
+
+/**
+ * Have we completed our first OR connection?
+ *
+ * Block display of application circuit progress until we do, to avoid
+ * some misleading behavior of jumping to high progress.
+ **/
+static bool bto_first_orconn = false;
+
+/** Is the ORCONN using a pluggable transport? */
+static bool
+using_pt(const bt_orconn_t *bto)
+{
+  return bto->proxy_type == PROXY_PLUGGABLE;
+}
+
+/** Is the ORCONN using a non-PT proxy? */
+static bool
+using_proxy(const bt_orconn_t *bto)
+{
+  switch (bto->proxy_type) {
+  case PROXY_CONNECT:
+  case PROXY_SOCKS4:
+  case PROXY_SOCKS5:
+  case PROXY_HAPROXY:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_anyconn(const bt_orconn_t *bto)
+{
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE_DONE, 0);
+    /* Unblock directory progress display */
+    control_event_boot_first_orconn();
+    /* Unblock apconn progress display */
+    bto_first_orconn = true;
+    break;
+  default:
+    break;
+  }
+}
+
+/**
+ * Emit control events when we have updated our idea of the best state
+ * that any application circuit OR connection has reached.
+ *
+ * Do some decoding of the ORCONN states depending on whether a PT or
+ * a proxy is in use.
+ **/
+void
+bto_cevent_apconn(const bt_orconn_t *bto)
+{
+  if (!bto_first_orconn)
+    return;
+
+  switch (bto->state) {
+  case OR_CONN_STATE_CONNECTING:
+    /* Exactly what kind of thing we're connecting to isn't
+     * information we directly get from the states in connection_or.c,
+     * so decode it here. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_PROXY, 0);
+    else
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN, 0);
+    break;
+  case OR_CONN_STATE_PROXY_HANDSHAKING:
+    /* Similarly, starting a proxy handshake means the TCP connect()
+     * succeeded to the proxy.  Let's be specific about what kind of
+     * proxy. */
+    if (using_pt(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PT, 0);
+    else if (using_proxy(bto))
+      control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, 0);
+    break;
+  case OR_CONN_STATE_TLS_HANDSHAKING:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_CONN_DONE, 0);
+    break;
+  case OR_CONN_STATE_TLS_CLIENT_RENEGOTIATING:
+  case OR_CONN_STATE_OR_HANDSHAKING_V2:
+  case OR_CONN_STATE_OR_HANDSHAKING_V3:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE, 0);
+    break;
+  case OR_CONN_STATE_OPEN:
+    control_event_bootstrap(BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, 0);
+    break;
+  default:
+    break;
+  }
+}
+
+/** Forget that we completed our first OR connection */
+void
+bto_cevent_reset(void)
+{
+  bto_first_orconn = false;
+}
diff --git a/src/feature/control/btrack_orconn_cevent.h b/src/feature/control/btrack_orconn_cevent.h
new file mode 100644
index 0000000000..8b2207721e
--- /dev/null
+++ b/src/feature/control/btrack_orconn_cevent.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_cevent.h
+ * \brief Header file for btrack_orconn_cevent.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_CEVENT_H
+#define TOR_BTRACK_ORCONN_CEVENT_H
+
+#include "feature/control/btrack_orconn.h"
+
+void bto_cevent_anyconn(const bt_orconn_t *);
+void bto_cevent_apconn(const bt_orconn_t *);
+void bto_cevent_reset(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_CEVENT_H) */
diff --git a/src/feature/control/btrack_orconn_maps.c b/src/feature/control/btrack_orconn_maps.c
new file mode 100644
index 0000000000..a60dffb8c4
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.c
@@ -0,0 +1,224 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.c
+ * \brief Hash map implementation for btrack_orconn.c
+ *
+ * These functions manipulate the hash maps that contain bt_orconn
+ * objects.
+ **/
+
+#include <stdbool.h>
+
+#include "core/or/or.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#define BTRACK_ORCONN_PRIVATE
+
+#include "feature/control/btrack_orconn.h"
+#include "feature/control/btrack_orconn_maps.h"
+#include "lib/log/log.h"
+
+static inline unsigned int
+bto_gid_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->gid, sizeof(elm->gid));
+}
+
+static inline int
+bto_gid_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->gid == b->gid;
+}
+
+static inline unsigned int
+bto_chan_hash_(bt_orconn_t *elm)
+{
+  return (unsigned)siphash24g(&elm->chan, sizeof(elm->chan));
+}
+
+static inline int
+bto_chan_eq_(bt_orconn_t *a, bt_orconn_t *b)
+{
+  return a->chan == b->chan;
+}
+
+HT_HEAD(bto_gid_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_gid_ht, bt_orconn_t, node, bto_gid_hash_, bto_gid_eq_);
+HT_GENERATE2(bto_gid_ht, bt_orconn_t, node,
+             bto_gid_hash_, bto_gid_eq_, 0.6,
+             tor_reallocarray_, tor_free_);
+static struct bto_gid_ht *bto_gid_map;
+
+HT_HEAD(bto_chan_ht, bt_orconn_t);
+HT_PROTOTYPE(bto_chan_ht, bt_orconn_t, chan_node, bto_chan_hash_,
+             bto_chan_eq_);
+HT_GENERATE2(bto_chan_ht, bt_orconn_t, chan_node,
+             bto_chan_hash_, bto_chan_eq_, 0.6,
+             tor_reallocarray_, tor_free_);
+static struct bto_chan_ht *bto_chan_map;
+
+/** Clear the GID hash map, freeing any bt_orconn_t objects that become
+ * unreferenced */
+static void
+bto_gid_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_gid_ht, bto_gid_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_gid_ht, bto_gid_map, elt);
+
+    c->gid = 0;
+    /* Don't delete if chan ID isn't zero: it's still in the chan hash map */
+    if (!c->chan)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_gid_ht, bto_gid_map);
+  tor_free(bto_gid_map);
+}
+
+/** Clear the chan ID hash map, freeing any bt_orconn_t objects that
+ * become unreferenced */
+static void
+bto_chan_clear_map(void)
+{
+  bt_orconn_t **elt, **next, *c;
+
+  for (elt = HT_START(bto_chan_ht, bto_chan_map);
+       elt;
+       elt = next) {
+    c = *elt;
+    next = HT_NEXT_RMV(bto_chan_ht, bto_chan_map, elt);
+
+    c->chan = 0;
+    /* Don't delete if GID isn't zero, it's still in the GID hash map */
+    if (!c->gid)
+      tor_free(c);
+  }
+  HT_CLEAR(bto_chan_ht, bto_chan_map);
+  tor_free(bto_chan_map);
+}
+
+/** Delete a bt_orconn from the hash maps by GID */
+void
+bto_delete(uint64_t gid)
+{
+  bt_orconn_t key, *bto;
+
+  key.gid = gid;
+  key.chan = 0;
+  bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto) {
+    /* The orconn might be unregistered because it's an EXT_OR_CONN? */
+    log_debug(LD_BTRACK, "tried to delete unregistered ORCONN gid=%"PRIu64,
+              gid);
+    return;
+  }
+  HT_REMOVE(bto_gid_ht, bto_gid_map, &key);
+  if (bto->chan) {
+    key.chan = bto->chan;
+    HT_REMOVE(bto_chan_ht, bto_chan_map, &key);
+  }
+  tor_free(bto);
+}
+
+/**
+ * Helper for bto_find_or_new().
+ *
+ * Update GID and chan ID of an existing bt_orconn object if needed,
+ * given a search key previously used within bto_find_or_new().
+ **/
+static bt_orconn_t *
+bto_update(bt_orconn_t *bto, const bt_orconn_t *key)
+{
+  /* ORCONN GIDs shouldn't change once assigned */
+  tor_assert(!bto->gid || !key->gid || bto->gid == key->gid);
+  if (!bto->gid && key->gid) {
+    /* Got a gid when we didn't already have one; insert into gid map */
+    log_debug(LD_BTRACK, "ORCONN chan=%"PRIu64" newgid=%"PRIu64, key->chan,
+              key->gid);
+    bto->gid = key->gid;
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  }
+  /* association of ORCONN with channel shouldn't change */
+  tor_assert(!bto->chan || !key->chan || bto->chan == key->chan);
+  if (!bto->chan && key->chan) {
+    /* Got a chan when we didn't already have one; insert into chan map */
+    log_debug(LD_BTRACK, "ORCONN gid=%"PRIu64" newchan=%"PRIu64,
+              bto->gid, key->chan);
+    bto->chan = key->chan;
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+  }
+  return bto;
+}
+
+/** Helper for bto_find_or_new() */
+static bt_orconn_t *
+bto_new(const bt_orconn_t *key)
+{
+  struct bt_orconn_t *bto = tor_malloc(sizeof(*bto));
+
+  bto->gid = key->gid;
+  bto->chan = key->chan;
+  bto->state = 0;
+  bto->proxy_type = 0;
+  bto->is_orig = false;
+  bto->is_onehop = true;
+
+  if (bto->gid)
+    HT_INSERT(bto_gid_ht, bto_gid_map, bto);
+  if (bto->chan)
+    HT_INSERT(bto_chan_ht, bto_chan_map, bto);
+
+  return bto;
+}
+
+/**
+ * Insert a new bt_orconn with the given GID and chan ID, or update
+ * the GID and chan ID if one already exists.
+ *
+ * Return the found or allocated bt_orconn.
+ **/
+bt_orconn_t *
+bto_find_or_new(uint64_t gid, uint64_t chan)
+{
+  bt_orconn_t key, *bto = NULL;
+
+  tor_assert(gid || chan);
+  key.gid = gid;
+  key.chan = chan;
+  if (key.gid)
+    bto = HT_FIND(bto_gid_ht, bto_gid_map, &key);
+  if (!bto && key.chan) {
+    /* Not found by GID; look up by chan ID */
+    bto = HT_FIND(bto_chan_ht, bto_chan_map, &key);
+  }
+  if (bto)
+    return bto_update(bto, &key);
+  else
+    return bto_new(&key);
+}
+
+/** Initialize the hash maps  */
+void
+bto_init_maps(void)
+{
+  bto_gid_map = tor_malloc(sizeof(*bto_gid_map));
+  HT_INIT(bto_gid_ht, bto_gid_map);
+  bto_chan_map = tor_malloc(sizeof(*bto_chan_map));
+  HT_INIT(bto_chan_ht, bto_chan_map);
+}
+
+/** Clear the hash maps, freeing all associated storage */
+void
+bto_clear_maps(void)
+{
+  bto_gid_clear_map();
+  bto_chan_clear_map();
+}
diff --git a/src/feature/control/btrack_orconn_maps.h b/src/feature/control/btrack_orconn_maps.h
new file mode 100644
index 0000000000..c83b22b1e8
--- /dev/null
+++ b/src/feature/control/btrack_orconn_maps.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_orconn_maps.h
+ * \brief Header file for btrack_orconn_maps.c
+ **/
+
+#ifndef TOR_BTRACK_ORCONN_MAPS_H
+#define TOR_BTRACK_ORCONN_MAPS_H
+
+void bto_delete(uint64_t);
+bt_orconn_t *bto_find_or_new(uint64_t, uint64_t);
+
+void bto_init_maps(void);
+void bto_clear_maps(void);
+
+#endif /* !defined(TOR_BTRACK_ORCONN_MAPS_H) */
diff --git a/src/feature/control/btrack_sys.h b/src/feature/control/btrack_sys.h
new file mode 100644
index 0000000000..5a157b7b54
--- /dev/null
+++ b/src/feature/control/btrack_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file btrack_sys.h
+ * \brief Declare subsystem object for the bootstrap tracker susbystem.
+ **/
+
+#ifndef TOR_BTRACK_SYS_H
+#define TOR_BTRACK_SYS_H
+
+extern const struct subsys_fns_t sys_btrack;
+
+#endif /* !defined(TOR_BTRACK_SYS_H) */
diff --git a/src/feature/control/control.c b/src/feature/control/control.c
index 26ac12d307..ee1026359d 100644
--- a/src/feature/control/control.c
+++ b/src/feature/control/control.c
@@ -1,6 +1,5 @@
-
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,83 +32,27 @@
  * stack.
  **/
 
+#define CONTROL_MODULE_PRIVATE
 #define CONTROL_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
 #include "app/main/main.h"
 #include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
-#include "core/or/channel.h"
-#include "core/or/channeltls.h"
-#include "core/or/circuitbuild.h"
-#include "core/or/circuitlist.h"
-#include "core/or/circuitstats.h"
-#include "core/or/circuituse.h"
-#include "core/or/command.h"
-#include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "core/or/policies.h"
-#include "core/or/reasons.h"
-#include "core/or/versions.h"
 #include "core/proto/proto_control0.h"
 #include "core/proto/proto_http.h"
-#include "feature/client/addressmap.h"
-#include "feature/client/bridges.h"
-#include "feature/client/dnsserv.h"
-#include "feature/client/entrynodes.h"
 #include "feature/control/control.h"
-#include "feature/control/fmt_serverstatus.h"
-#include "feature/control/getinfo_geoip.h"
-#include "feature/dircache/dirserv.h"
-#include "feature/dirclient/dirclient.h"
-#include "feature/dirclient/dlstatus.h"
-#include "feature/dircommon/directory.h"
-#include "feature/hibernate/hibernate.h"
-#include "feature/hs/hs_cache.h"
-#include "feature/hs/hs_common.h"
-#include "feature/hs/hs_control.h"
-#include "feature/hs_common/shared_random_client.h"
-#include "feature/nodelist/authcert.h"
-#include "feature/nodelist/dirlist.h"
-#include "feature/nodelist/microdesc.h"
-#include "feature/nodelist/networkstatus.h"
-#include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
-#include "feature/nodelist/routerlist.h"
-#include "feature/relay/router.h"
-#include "feature/relay/routermode.h"
-#include "feature/relay/selftest.h"
-#include "feature/rend/rendclient.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_proto.h"
 #include "feature/rend/rendcommon.h"
-#include "feature/rend/rendparse.h"
 #include "feature/rend/rendservice.h"
-#include "feature/stats/geoip_stats.h"
-#include "feature/stats/predict_ports.h"
-#include "lib/container/buffers.h"
-#include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_util.h"
-#include "lib/encoding/confline.h"
-#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/procmon.h"
 
-#include "feature/dircache/cached_dir_st.h"
 #include "feature/control/control_connection_st.h"
-#include "core/or/cpath_build_state_st.h"
-#include "core/or/entry_connection_st.h"
-#include "feature/nodelist/extrainfo_st.h"
-#include "feature/nodelist/networkstatus_st.h"
-#include "feature/nodelist/node_st.h"
-#include "core/or/or_connection_st.h"
-#include "core/or/or_circuit_st.h"
-#include "core/or/origin_circuit_st.h"
-#include "feature/nodelist/microdesc_st.h"
-#include "feature/rend/rend_authorized_client_st.h"
-#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
-#include "feature/rend/rend_service_descriptor_st.h"
-#include "feature/nodelist/routerinfo_st.h"
-#include "feature/nodelist/routerlist_st.h"
-#include "core/or/socks_request_st.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -118,152 +61,6 @@
 #include <sys/stat.h>
 #endif
 
-#ifndef _WIN32
-#include <pwd.h>
-#include <sys/resource.h>
-#endif
-
-#include "lib/crypt_ops/crypto_s2k.h"
-#include "lib/evloop/procmon.h"
-#include "lib/evloop/compat_libevent.h"
-
-/** Yield true iff <b>s</b> is the state of a control_connection_t that has
- * finished authentication and is accepting commands. */
-#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
-
-/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
- * connection is interested in events of type <b>e</b>.  We use this
- * so that we can decide to skip generating event messages that nobody
- * has interest in without having to walk over the global connection
- * list to find out.
- **/
-typedef uint64_t event_mask_t;
-
-/** An event mask of all the events that any controller is interested in
- * receiving. */
-static event_mask_t global_event_mask = 0;
-
-/** True iff we have disabled log messages from being sent to the controller */
-static int disable_log_messages = 0;
-
-/** Macro: true if any control connection is interested in events of type
- * <b>e</b>. */
-#define EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & EVENT_MASK_(e)))
-
-/** Macro: true if any event from the bitfield 'e' is interesting. */
-#define ANY_EVENT_IS_INTERESTING(e) \
-  (!! (global_event_mask & (e)))
-
-/** If we're using cookie-type authentication, how long should our cookies be?
- */
-#define AUTHENTICATION_COOKIE_LEN 32
-
-/** If true, we've set authentication_cookie to a secret code and
- * stored it to disk. */
-static int authentication_cookie_is_set = 0;
-/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
- * and which we're using to authenticate controllers.  (If the controller can
- * read it off disk, it has permission to connect.) */
-static uint8_t *authentication_cookie = NULL;
-
-#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
-  "Tor safe cookie authentication server-to-controller hash"
-#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
-  "Tor safe cookie authentication controller-to-server hash"
-#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
-
-/** The list of onion services that have been added via ADD_ONION that do not
- * belong to any particular control connection.
- */
-static smartlist_t *detached_onion_services = NULL;
-
-/** A sufficiently large size to record the last bootstrap phase string. */
-#define BOOTSTRAP_MSG_LEN 1024
-
-/** What was the last bootstrap phase message we sent? We keep track
- * of this so we can respond to getinfo status/bootstrap-phase queries. */
-static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
-
-static void connection_printf_to_buf(control_connection_t *conn,
-                                     const char *format, ...)
-  CHECK_PRINTF(2,3);
-static void send_control_event_impl(uint16_t event,
-                                    const char *format, va_list ap)
-  CHECK_PRINTF(2,0);
-static int control_event_status(int type, int severity, const char *format,
-                                va_list args)
-  CHECK_PRINTF(3,0);
-
-static void send_control_done(control_connection_t *conn);
-static void send_control_event(uint16_t event,
-                               const char *format, ...)
-  CHECK_PRINTF(2,3);
-static int handle_control_setconf(control_connection_t *conn, uint32_t len,
-                                  char *body);
-static int handle_control_resetconf(control_connection_t *conn, uint32_t len,
-                                    char *body);
-static int handle_control_getconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_setevents(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_authenticate(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_signal(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                                     const char *body);
-static char *list_getinfo_options(void);
-static int handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_extendcircuit(control_connection_t *conn,
-                                        uint32_t len,
-                                        const char *body);
-static int handle_control_setcircuitpurpose(control_connection_t *conn,
-                                            uint32_t len, const char *body);
-static int handle_control_attachstream(control_connection_t *conn,
-                                       uint32_t len,
-                                        const char *body);
-static int handle_control_postdescriptor(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_redirectstream(control_connection_t *conn,
-                                         uint32_t len,
-                                         const char *body);
-static int handle_control_closestream(control_connection_t *conn, uint32_t len,
-                                      const char *body);
-static int handle_control_closecircuit(control_connection_t *conn,
-                                       uint32_t len,
-                                       const char *body);
-static int handle_control_resolve(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_usefeature(control_connection_t *conn,
-                                     uint32_t len,
-                                     const char *body);
-static int handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                                  const char *body);
-static int handle_control_hspost(control_connection_t *conn, uint32_t len,
-                                 const char *body);
-static int handle_control_add_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int handle_control_del_onion(control_connection_t *conn, uint32_t len,
-                                    const char *body);
-static int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
-                                      size_t len);
-static void orconn_target_get_name(char *buf, size_t len,
-                                   or_connection_t *conn);
-
-static int get_cached_network_liveness(void);
-static void set_cached_network_liveness(int liveness);
-
-static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
-
-static char * download_status_to_string(const download_status_t *dl);
-static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
-
 /** Convert a connection_t* to an control_connection_t*; assert if the cast is
  * invalid. */
 control_connection_t *
@@ -273,410 +70,6 @@ TO_CONTROL_CONN(connection_t *c)
   return DOWNCAST(control_connection_t, c);
 }
 
-/** Given a control event code for a message event, return the corresponding
- * log severity. */
-static inline int
-event_to_log_severity(int event)
-{
-  switch (event) {
-    case EVENT_DEBUG_MSG: return LOG_DEBUG;
-    case EVENT_INFO_MSG: return LOG_INFO;
-    case EVENT_NOTICE_MSG: return LOG_NOTICE;
-    case EVENT_WARN_MSG: return LOG_WARN;
-    case EVENT_ERR_MSG: return LOG_ERR;
-    default: return -1;
-  }
-}
-
-/** Given a log severity, return the corresponding control event code. */
-static inline int
-log_severity_to_event(int severity)
-{
-  switch (severity) {
-    case LOG_DEBUG: return EVENT_DEBUG_MSG;
-    case LOG_INFO: return EVENT_INFO_MSG;
-    case LOG_NOTICE: return EVENT_NOTICE_MSG;
-    case LOG_WARN: return EVENT_WARN_MSG;
-    case LOG_ERR: return EVENT_ERR_MSG;
-    default: return -1;
-  }
-}
-
-/** Helper: clear bandwidth counters of all origin circuits. */
-static void
-clear_circ_bw_fields(void)
-{
-  origin_circuit_t *ocirc;
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-    ocirc = TO_ORIGIN_CIRCUIT(circ);
-    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-  }
-  SMARTLIST_FOREACH_END(circ);
-}
-
-/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
- * connection's event_mask field. */
-void
-control_update_global_event_mask(void)
-{
-  smartlist_t *conns = get_connection_array();
-  event_mask_t old_mask, new_mask;
-  old_mask = global_event_mask;
-  int any_old_per_sec_events = control_any_per_second_event_enabled();
-
-  global_event_mask = 0;
-  SMARTLIST_FOREACH(conns, connection_t *, _conn,
-  {
-    if (_conn->type == CONN_TYPE_CONTROL &&
-        STATE_IS_OPEN(_conn->state)) {
-      control_connection_t *conn = TO_CONTROL_CONN(_conn);
-      global_event_mask |= conn->event_mask;
-    }
-  });
-
-  new_mask = global_event_mask;
-
-  /* Handle the aftermath.  Set up the log callback to tell us only what
-   * we want to hear...*/
-  control_adjust_event_log_severity();
-
-  /* Macro: true if ev was false before and is true now. */
-#define NEWLY_ENABLED(ev) \
-  (! (old_mask & (ev)) && (new_mask & (ev)))
-
-  /* ...then, if we've started logging stream or circ bw, clear the
-   * appropriate fields. */
-  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
-    SMARTLIST_FOREACH(conns, connection_t *, conn,
-    {
-      if (conn->type == CONN_TYPE_AP) {
-        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
-        edge_conn->n_written = edge_conn->n_read = 0;
-      }
-    });
-  }
-  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
-    clear_circ_bw_fields();
-  }
-  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
-    uint64_t r, w;
-    control_get_bytes_rw_last_sec(&r, &w);
-  }
-  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
-    reschedule_per_second_timer();
-  }
-
-#undef NEWLY_ENABLED
-}
-
-/** Adjust the log severities that result in control_event_logmsg being called
- * to match the severity of log messages that any controllers are interested
- * in. */
-void
-control_adjust_event_log_severity(void)
-{
-  int i;
-  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
-
-  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      min_log_event = i;
-      break;
-    }
-  }
-  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
-    if (EVENT_IS_INTERESTING(i)) {
-      max_log_event = i;
-      break;
-    }
-  }
-  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
-    if (min_log_event > EVENT_NOTICE_MSG)
-      min_log_event = EVENT_NOTICE_MSG;
-    if (max_log_event < EVENT_ERR_MSG)
-      max_log_event = EVENT_ERR_MSG;
-  }
-  if (min_log_event <= max_log_event)
-    change_callback_log_severity(event_to_log_severity(min_log_event),
-                                 event_to_log_severity(max_log_event),
-                                 control_event_logmsg);
-  else
-    change_callback_log_severity(LOG_ERR, LOG_ERR,
-                                 control_event_logmsg);
-}
-
-/** Return true iff the event with code <b>c</b> is being sent to any current
- * control connection.  This is useful if the amount of work needed to prepare
- * to call the appropriate control_event_...() function is high.
- */
-int
-control_event_is_interesting(int event)
-{
-  return EVENT_IS_INTERESTING(event);
-}
-
-/** Return true if any event that needs to fire once a second is enabled. */
-int
-control_any_per_second_event_enabled(void)
-{
-  return ANY_EVENT_IS_INTERESTING(
-      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CELL_STATS) |
-      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
-      EVENT_MASK_(EVENT_CONN_BW) |
-      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
-  );
-}
-
-/* The value of 'get_bytes_read()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_read = 0;
-/* The value of 'get_bytes_written()' the previous time that
- * control_get_bytes_rw_last_sec() as called. */
-static uint64_t stats_prev_n_written = 0;
-
-/**
- * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
- * and written by Tor since the last call to this function.
- *
- * Call this only from the main thread.
- */
-static void
-control_get_bytes_rw_last_sec(uint64_t *n_read,
-                              uint64_t *n_written)
-{
-  const uint64_t stats_n_bytes_read = get_bytes_read();
-  const uint64_t stats_n_bytes_written = get_bytes_written();
-
-  *n_read = stats_n_bytes_read - stats_prev_n_read;
-  *n_written = stats_n_bytes_written - stats_prev_n_written;
-  stats_prev_n_read = stats_n_bytes_read;
-  stats_prev_n_written = stats_n_bytes_written;
-}
-
-/**
- * Run all the controller events (if any) that are scheduled to trigger once
- * per second.
- */
-void
-control_per_second_events(void)
-{
-  if (!control_any_per_second_event_enabled())
-    return;
-
-  uint64_t bytes_read, bytes_written;
-  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
-  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
-
-  control_event_stream_bandwidth_used();
-  control_event_conn_bandwidth_used();
-  control_event_circ_bandwidth_used();
-  control_event_circuit_cell_stats();
-}
-
-/** Append a NUL-terminated string <b>s</b> to the end of
- * <b>conn</b>-\>outbuf.
- */
-static inline void
-connection_write_str_to_buf(const char *s, control_connection_t *conn)
-{
-  size_t len = strlen(s);
-  connection_buf_add(s, len, TO_CONN(conn));
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
- * contents of <b>data</b> into *<b>out</b>, adding a period before any period
- * that appears at the start of a line, and adding a period-CRLF line at
- * the end. Replace all LF characters sequences with CRLF.  Return the number
- * of bytes in *<b>out</b>.
- */
-STATIC size_t
-write_escaped_data(const char *data, size_t len, char **out)
-{
-  tor_assert(len < SIZE_MAX - 9);
-  size_t sz_out = len+8+1;
-  char *outp;
-  const char *start = data, *end;
-  size_t i;
-  int start_of_line;
-  for (i=0; i < len; ++i) {
-    if (data[i] == '\n') {
-      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
-      if (sz_out >= SIZE_T_CEILING) {
-        log_warn(LD_BUG, "Input to write_escaped_data was too long");
-        *out = tor_strdup(".\r\n");
-        return 3;
-      }
-    }
-  }
-  *out = outp = tor_malloc(sz_out);
-  end = data+len;
-  start_of_line = 1;
-  while (data < end) {
-    if (*data == '\n') {
-      if (data > start && data[-1] != '\r')
-        *outp++ = '\r';
-      start_of_line = 1;
-    } else if (*data == '.') {
-      if (start_of_line) {
-        start_of_line = 0;
-        *outp++ = '.';
-      }
-    } else {
-      start_of_line = 0;
-    }
-    *outp++ = *data++;
-  }
-  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
-    *outp++ = '\r';
-    *outp++ = '\n';
-  }
-  *outp++ = '.';
-  *outp++ = '\r';
-  *outp++ = '\n';
-  *outp = '\0'; /* NUL-terminate just in case. */
-  tor_assert(outp >= *out);
-  tor_assert((size_t)(outp - *out) <= sz_out);
-  return outp - *out;
-}
-
-/** Given a <b>len</b>-character string in <b>data</b>, made of lines
- * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
- * the contents of <b>data</b> into *<b>out</b>, removing any period
- * that appears at the start of a line, and replacing all CRLF sequences
- * with LF.   Return the number of
- * bytes in *<b>out</b>. */
-STATIC size_t
-read_escaped_data(const char *data, size_t len, char **out)
-{
-  char *outp;
-  const char *next;
-  const char *end;
-
-  *out = outp = tor_malloc(len+1);
-
-  end = data+len;
-
-  while (data < end) {
-    /* we're at the start of a line. */
-    if (*data == '.')
-      ++data;
-    next = memchr(data, '\n', end-data);
-    if (next) {
-      size_t n_to_copy = next-data;
-      /* Don't copy a CR that precedes this LF. */
-      if (n_to_copy && *(next-1) == '\r')
-        --n_to_copy;
-      memcpy(outp, data, n_to_copy);
-      outp += n_to_copy;
-      data = next+1; /* This will point at the start of the next line,
-                      * or the end of the string, or a period. */
-    } else {
-      memcpy(outp, data, end-data);
-      outp += (end-data);
-      *outp = '\0';
-      return outp - *out;
-    }
-    *outp++ = '\n';
-  }
-
-  *outp = '\0';
-  return outp - *out;
-}
-
-/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
- * double-quoted string with escaped characters, return the length of that
- * string (as encoded, including quotes).  Otherwise return -1. */
-static inline int
-get_escaped_string_length(const char *start, size_t in_len_max,
-                          int *chars_out)
-{
-  const char *cp, *end;
-  int chars = 0;
-
-  if (*start != '\"')
-    return -1;
-
-  cp = start+1;
-  end = start+in_len_max;
-
-  /* Calculate length. */
-  while (1) {
-    if (cp >= end) {
-      return -1; /* Too long. */
-    } else if (*cp == '\\') {
-      if (++cp == end)
-        return -1; /* Can't escape EOS. */
-      ++cp;
-      ++chars;
-    } else if (*cp == '\"') {
-      break;
-    } else {
-      ++cp;
-      ++chars;
-    }
-  }
-  if (chars_out)
-    *chars_out = chars;
-  return (int)(cp - start+1);
-}
-
-/** As decode_escaped_string, but does not decode the string: copies the
- * entire thing, including quotation marks. */
-static const char *
-extract_escaped_string(const char *start, size_t in_len_max,
-                       char **out, size_t *out_len)
-{
-  int length = get_escaped_string_length(start, in_len_max, NULL);
-  if (length<0)
-    return NULL;
-  *out_len = length;
-  *out = tor_strndup(start, *out_len);
-  return start+length;
-}
-
-/** Given a pointer to a string starting at <b>start</b> containing
- * <b>in_len_max</b> characters, decode a string beginning with one double
- * quote, containing any number of non-quote characters or characters escaped
- * with a backslash, and ending with a final double quote.  Place the resulting
- * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
- * store its length in <b>out_len</b>.  On success, return a pointer to the
- * character immediately following the escaped string.  On failure, return
- * NULL. */
-static const char *
-decode_escaped_string(const char *start, size_t in_len_max,
-                   char **out, size_t *out_len)
-{
-  const char *cp, *end;
-  char *outp;
-  int len, n_chars = 0;
-
-  len = get_escaped_string_length(start, in_len_max, &n_chars);
-  if (len<0)
-    return NULL;
-
-  end = start+len-1; /* Index of last quote. */
-  tor_assert(*end == '\"');
-  outp = *out = tor_malloc(len+1);
-  *out_len = n_chars;
-
-  cp = start+1;
-  while (cp < end) {
-    if (*cp == '\\')
-      ++cp;
-    *outp++ = *cp++;
-  }
-  *outp = '\0';
-  tor_assert((outp - *out) == (int)*out_len);
-
-  return end+1;
-}
-
 /** Create and add a new controller connection on <b>sock</b>.  If
  * <b>CC_LOCAL_FD_IS_OWNER</b> is set in <b>flags</b>, this Tor process should
  * exit when the connection closes.  If <b>CC_LOCAL_FD_IS_AUTHENTICATED</b>
@@ -720,29 +113,6 @@ control_connection_add_local_fd(tor_socket_t sock, unsigned flags)
   return 0;
 }
 
-/** Acts like sprintf, but writes its formatted string to the end of
- * <b>conn</b>-\>outbuf. */
-static void
-connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
-{
-  va_list ap;
-  char *buf = NULL;
-  int len;
-
-  va_start(ap,format);
-  len = tor_vasprintf(&buf, format, ap);
-  va_end(ap);
-
-  if (len < 0) {
-    log_err(LD_BUG, "Unable to format string for controller.");
-    tor_assert(0);
-  }
-
-  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
-
-  tor_free(buf);
-}
-
 /** Write all of the open control ports to ControlPortWriteToFile */
 void
 control_ports_write_to_file(void)
@@ -787,886 +157,11 @@ control_ports_write_to_file(void)
   smartlist_free(lines);
 }
 
-/** Send a "DONE" message down the control connection <b>conn</b>. */
-static void
-send_control_done(control_connection_t *conn)
-{
-  connection_write_str_to_buf("250 OK\r\n", conn);
-}
-
-/** Represents an event that's queued to be sent to one or more
- * controllers. */
-typedef struct queued_event_s {
-  uint16_t event;
-  char *msg;
-} queued_event_t;
-
-/** Pointer to int. If this is greater than 0, we don't allow new events to be
- * queued. */
-static tor_threadlocal_t block_event_queue_flag;
-
-/** Holds a smartlist of queued_event_t objects that may need to be sent
- * to one or more controllers */
-static smartlist_t *queued_control_events = NULL;
-
-/** True if the flush_queued_events_event is pending. */
-static int flush_queued_event_pending = 0;
-
-/** Lock to protect the above fields. */
-static tor_mutex_t *queued_control_events_lock = NULL;
-
-/** An event that should fire in order to flush the contents of
- * queued_control_events. */
-static mainloop_event_t *flush_queued_events_event = NULL;
-
-void
-control_initialize_event_queue(void)
-{
-  if (queued_control_events == NULL) {
-    queued_control_events = smartlist_new();
-  }
-
-  if (flush_queued_events_event == NULL) {
-    struct event_base *b = tor_libevent_get_base();
-    if (b) {
-      flush_queued_events_event =
-        mainloop_event_new(flush_queued_events_cb, NULL);
-      tor_assert(flush_queued_events_event);
-    }
-  }
-
-  if (queued_control_events_lock == NULL) {
-    queued_control_events_lock = tor_mutex_new();
-    tor_threadlocal_init(&block_event_queue_flag);
-  }
-}
-
-static int *
-get_block_event_queue(void)
-{
-  int *val = tor_threadlocal_get(&block_event_queue_flag);
-  if (PREDICT_UNLIKELY(val == NULL)) {
-    val = tor_malloc_zero(sizeof(int));
-    tor_threadlocal_set(&block_event_queue_flag, val);
-  }
-  return val;
-}
-
-/** Helper: inserts an event on the list of events queued to be sent to
- * one or more controllers, and schedules the events to be flushed if needed.
- *
- * This function takes ownership of <b>msg</b>, and may free it.
- *
- * We queue these events rather than send them immediately in order to break
- * the dependency in our callgraph from code that generates events for the
- * controller, and the network layer at large.  Otherwise, nearly every
- * interesting part of Tor would potentially call every other interesting part
- * of Tor.
- */
-MOCK_IMPL(STATIC void,
-queue_control_event_string,(uint16_t event, char *msg))
-{
-  /* This is redundant with checks done elsewhere, but it's a last-ditch
-   * attempt to avoid queueing something we shouldn't have to queue. */
-  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
-    tor_free(msg);
-    return;
-  }
-
-  int *block_event_queue = get_block_event_queue();
-  if (*block_event_queue) {
-    tor_free(msg);
-    return;
-  }
-
-  queued_event_t *ev = tor_malloc(sizeof(*ev));
-  ev->event = event;
-  ev->msg = msg;
-
-  /* No queueing an event while queueing an event */
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  tor_assert(queued_control_events);
-  smartlist_add(queued_control_events, ev);
-
-  int activate_event = 0;
-  if (! flush_queued_event_pending && in_main_thread()) {
-    activate_event = 1;
-    flush_queued_event_pending = 1;
-  }
-
-  tor_mutex_release(queued_control_events_lock);
-
-  --*block_event_queue;
-
-  /* We just put an event on the queue; mark the queue to be
-   * flushed.  We only do this from the main thread for now; otherwise,
-   * we'd need to incur locking overhead in Libevent or use a socket.
-   */
-  if (activate_event) {
-    tor_assert(flush_queued_events_event);
-    mainloop_event_activate(flush_queued_events_event);
-  }
-}
-
-#define queued_event_free(ev) \
-  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
-
-/** Release all storage held by <b>ev</b>. */
-static void
-queued_event_free_(queued_event_t *ev)
-{
-  if (ev == NULL)
-    return;
-
-  tor_free(ev->msg);
-  tor_free(ev);
-}
-
-/** Send every queued event to every controller that's interested in it,
- * and remove the events from the queue.  If <b>force</b> is true,
- * then make all controllers send their data out immediately, since we
- * may be about to shut down. */
-static void
-queued_events_flush_all(int force)
-{
-  /* Make sure that we get all the pending log events, if there are any. */
-  flush_pending_log_callbacks();
-
-  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
-    return;
-  }
-  smartlist_t *all_conns = get_connection_array();
-  smartlist_t *controllers = smartlist_new();
-  smartlist_t *queued_events;
-
-  int *block_event_queue = get_block_event_queue();
-  ++*block_event_queue;
-
-  tor_mutex_acquire(queued_control_events_lock);
-  /* No queueing an event while flushing events. */
-  flush_queued_event_pending = 0;
-  queued_events = queued_control_events;
-  queued_control_events = smartlist_new();
-  tor_mutex_release(queued_control_events_lock);
-
-  /* Gather all the controllers that will care... */
-  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
-    if (conn->type == CONN_TYPE_CONTROL &&
-        !conn->marked_for_close &&
-        conn->state == CONTROL_CONN_STATE_OPEN) {
-      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
-
-      smartlist_add(controllers, control_conn);
-    }
-  } SMARTLIST_FOREACH_END(conn);
-
-  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
-    const event_mask_t bit = ((event_mask_t)1) << ev->event;
-    const size_t msg_len = strlen(ev->msg);
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      if (control_conn->event_mask & bit) {
-        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
-      }
-    } SMARTLIST_FOREACH_END(control_conn);
-
-    queued_event_free(ev);
-  } SMARTLIST_FOREACH_END(ev);
-
-  if (force) {
-    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
-                            control_conn) {
-      connection_flush(TO_CONN(control_conn));
-    } SMARTLIST_FOREACH_END(control_conn);
-  }
-
-  smartlist_free(queued_events);
-  smartlist_free(controllers);
-
-  --*block_event_queue;
-}
-
-/** Libevent callback: Flushes pending events to controllers that are
- * interested in them. */
-static void
-flush_queued_events_cb(mainloop_event_t *event, void *arg)
-{
-  (void) event;
-  (void) arg;
-  queued_events_flush_all(0);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is given by <b>msg</b>.
- *
- * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
- * respect to the EXTENDED_EVENTS feature. */
-MOCK_IMPL(STATIC void,
-send_control_event_string,(uint16_t event,
-                           const char *msg))
-{
-  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
-  queue_control_event_string(event, tor_strdup(msg));
-}
-
-/** Helper for send_control_event and control_event_status:
- * Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event_impl(uint16_t event,
-                        const char *format, va_list ap)
-{
-  char *buf = NULL;
-  int len;
-
-  len = tor_vasprintf(&buf, format, ap);
-  if (len < 0) {
-    log_warn(LD_BUG, "Unable to format event for controller.");
-    return;
-  }
-
-  queue_control_event_string(event, buf);
-}
-
-/** Send an event to all v1 controllers that are listening for code
- * <b>event</b>.  The event's body is created by the printf-style format in
- * <b>format</b>, and other arguments as provided. */
-static void
-send_control_event(uint16_t event,
-                   const char *format, ...)
-{
-  va_list ap;
-  va_start(ap, format);
-  send_control_event_impl(event, format, ap);
-  va_end(ap);
-}
-
-/** Given a text circuit <b>id</b>, return the corresponding circuit. */
-static origin_circuit_t *
-get_circ(const char *id)
-{
-  uint32_t n_id;
-  int ok;
-  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  return circuit_get_by_global_id(n_id);
-}
-
-/** Given a text stream <b>id</b>, return the corresponding AP connection. */
-static entry_connection_t *
-get_stream(const char *id)
-{
-  uint64_t n_id;
-  int ok;
-  connection_t *conn;
-  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
-  if (!ok)
-    return NULL;
-  conn = connection_get_by_global_id(n_id);
-  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
-    return NULL;
-  return TO_ENTRY_CONN(conn);
-}
-
-/** Helper for setconf and resetconf. Acts like setconf, except
- * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
- * contents of body.
- */
-static int
-control_setconf_helper(control_connection_t *conn, uint32_t len, char *body,
-                       int use_defaults)
-{
-  setopt_err_t opt_err;
-  config_line_t *lines=NULL;
-  char *start = body;
-  char *errstring = NULL;
-  const unsigned flags =
-    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
-
-  char *config;
-  smartlist_t *entries = smartlist_new();
-
-  /* We have a string, "body", of the format '(key(=val|="val")?)' entries
-   * separated by space.  break it into a list of configuration entries. */
-  while (*body) {
-    char *eq = body;
-    char *key;
-    char *entry;
-    while (!TOR_ISSPACE(*eq) && *eq != '=')
-      ++eq;
-    key = tor_strndup(body, eq-body);
-    body = eq+1;
-    if (*eq == '=') {
-      char *val=NULL;
-      size_t val_len=0;
-      if (*body != '\"') {
-        char *val_start = body;
-        while (!TOR_ISSPACE(*body))
-          body++;
-        val = tor_strndup(val_start, body-val_start);
-        val_len = strlen(val);
-      } else {
-        body = (char*)extract_escaped_string(body, (len - (body-start)),
-                                             &val, &val_len);
-        if (!body) {
-          connection_write_str_to_buf("551 Couldn't parse string\r\n", conn);
-          SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-          smartlist_free(entries);
-          tor_free(key);
-          return 0;
-        }
-      }
-      tor_asprintf(&entry, "%s %s", key, val);
-      tor_free(key);
-      tor_free(val);
-    } else {
-      entry = key;
-    }
-    smartlist_add(entries, entry);
-    while (TOR_ISSPACE(*body))
-      ++body;
-  }
-
-  smartlist_add_strdup(entries, "");
-  config = smartlist_join_strings(entries, "\n", 0, NULL);
-  SMARTLIST_FOREACH(entries, char *, cp, tor_free(cp));
-  smartlist_free(entries);
-
-  if (config_get_lines(config, &lines, 0) < 0) {
-    log_warn(LD_CONTROL,"Controller gave us config lines we can't parse.");
-    connection_write_str_to_buf("551 Couldn't parse configuration\r\n",
-                                conn);
-    tor_free(config);
-    return 0;
-  }
-  tor_free(config);
-
-  opt_err = options_trial_assign(lines, flags, &errstring);
-  {
-    const char *msg;
-    switch (opt_err) {
-      case SETOPT_ERR_MISC:
-        msg = "552 Unrecognized option";
-        break;
-      case SETOPT_ERR_PARSE:
-        msg = "513 Unacceptable option value";
-        break;
-      case SETOPT_ERR_TRANSITION:
-        msg = "553 Transition not allowed";
-        break;
-      case SETOPT_ERR_SETTING:
-      default:
-        msg = "553 Unable to set option";
-        break;
-      case SETOPT_OK:
-        config_free_lines(lines);
-        send_control_done(conn);
-        return 0;
-    }
-    log_warn(LD_CONTROL,
-             "Controller gave us config lines that didn't validate: %s",
-             errstring);
-    connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    config_free_lines(lines);
-    tor_free(errstring);
-    return 0;
-  }
-}
-
-/** Called when we receive a SETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body.*/
-static int
-handle_control_setconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 0);
-}
-
-/** Called when we receive a RESETCONF message: parse the body and try
- * to update our configuration.  Reply with a DONE or ERROR message.
- * Modifies the contents of body. */
-static int
-handle_control_resetconf(control_connection_t *conn, uint32_t len, char *body)
-{
-  return control_setconf_helper(conn, len, body, 1);
-}
-
-/** Called when we receive a GETCONF message.  Parse the request, and
- * reply with a CONFVALUE or an ERROR message */
-static int
-handle_control_getconf(control_connection_t *conn, uint32_t body_len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *msg = NULL;
-  size_t msg_len;
-  const or_options_t *options = get_options();
-  int i, len;
-
-  (void) body_len; /* body is NUL-terminated; so we can ignore len. */
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    if (!option_is_recognized(q)) {
-      smartlist_add(unrecognized, (char*) q);
-    } else {
-      config_line_t *answer = option_get_assignment(options,q);
-      if (!answer) {
-        const char *name = option_get_canonical_name(q);
-        smartlist_add_asprintf(answers, "250-%s\r\n", name);
-      }
-
-      while (answer) {
-        config_line_t *next;
-        smartlist_add_asprintf(answers, "250-%s=%s\r\n",
-                     answer->key, answer->value);
-
-        next = answer->next;
-        tor_free(answer->key);
-        tor_free(answer->value);
-        tor_free(answer);
-        answer = next;
-      }
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if ((len = smartlist_len(unrecognized))) {
-    for (i=0; i < len-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-Unrecognized configuration key \"%s\"\r\n",
-                               (char*)smartlist_get(unrecognized, i));
-    connection_printf_to_buf(conn,
-                             "552 Unrecognized configuration key \"%s\"\r\n",
-                             (char*)smartlist_get(unrecognized, len-1));
-  } else if ((len = smartlist_len(answers))) {
-    char *tmp = smartlist_get(answers, len-1);
-    tor_assert(strlen(tmp)>4);
-    tmp[3] = ' ';
-    msg = smartlist_join_strings(answers, "", 0, &msg_len);
-    connection_buf_add(msg, msg_len, TO_CONN(conn));
-  } else {
-    connection_write_str_to_buf("250 OK\r\n", conn);
-  }
-
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  smartlist_free(unrecognized);
-
-  tor_free(msg);
-
-  return 0;
-}
-
-/** Called when we get a +LOADCONF message. */
-static int
-handle_control_loadconf(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  setopt_err_t retval;
-  char *errstring = NULL;
-  const char *msg = NULL;
-  (void) len;
-
-  retval = options_init_from_string(NULL, body, CMD_RUN_TOR, NULL, &errstring);
-
-  if (retval != SETOPT_OK)
-    log_warn(LD_CONTROL,
-             "Controller gave us config file that didn't validate: %s",
-             errstring);
-
-  switch (retval) {
-  case SETOPT_ERR_PARSE:
-    msg = "552 Invalid config file";
-    break;
-  case SETOPT_ERR_TRANSITION:
-    msg = "553 Transition not allowed";
-    break;
-  case SETOPT_ERR_SETTING:
-    msg = "553 Unable to set option";
-    break;
-  case SETOPT_ERR_MISC:
-  default:
-    msg = "550 Unable to load config";
-    break;
-  case SETOPT_OK:
-    break;
-  }
-  if (msg) {
-    if (errstring)
-      connection_printf_to_buf(conn, "%s: %s\r\n", msg, errstring);
-    else
-      connection_printf_to_buf(conn, "%s\r\n", msg);
-  } else {
-    send_control_done(conn);
-  }
-  tor_free(errstring);
-  return 0;
-}
-
-/** Helper structure: maps event values to their names. */
-struct control_event_t {
-  uint16_t event_code;
-  const char *event_name;
-};
-/** Table mapping event values to their names.  Used to implement SETEVENTS
- * and GETINFO events/names, and to keep they in sync. */
-static const struct control_event_t control_event_table[] = {
-  { EVENT_CIRCUIT_STATUS, "CIRC" },
-  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
-  { EVENT_STREAM_STATUS, "STREAM" },
-  { EVENT_OR_CONN_STATUS, "ORCONN" },
-  { EVENT_BANDWIDTH_USED, "BW" },
-  { EVENT_DEBUG_MSG, "DEBUG" },
-  { EVENT_INFO_MSG, "INFO" },
-  { EVENT_NOTICE_MSG, "NOTICE" },
-  { EVENT_WARN_MSG, "WARN" },
-  { EVENT_ERR_MSG, "ERR" },
-  { EVENT_NEW_DESC, "NEWDESC" },
-  { EVENT_ADDRMAP, "ADDRMAP" },
-  { EVENT_DESCCHANGED, "DESCCHANGED" },
-  { EVENT_NS, "NS" },
-  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
-  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
-  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
-  { EVENT_GUARD, "GUARD" },
-  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
-  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
-  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
-  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
-  { EVENT_GOT_SIGNAL, "SIGNAL" },
-  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
-  { EVENT_CONN_BW, "CONN_BW" },
-  { EVENT_CELL_STATS, "CELL_STATS" },
-  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
-  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
-  { EVENT_HS_DESC, "HS_DESC" },
-  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
-  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
-  { 0, NULL },
-};
-
-/** Called when we get a SETEVENTS message: update conn->event_mask,
- * and reply with DONE or ERROR. */
-static int
-handle_control_setevents(control_connection_t *conn, uint32_t len,
-                         const char *body)
-{
-  int event_code;
-  event_mask_t event_mask = 0;
-  smartlist_t *events = smartlist_new();
-
-  (void) len;
-
-  smartlist_split_string(events, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
-    {
-      if (!strcasecmp(ev, "EXTENDED") ||
-          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
-        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
-                 "supported.", ev);
-        continue;
-      } else {
-        int i;
-        event_code = -1;
-
-        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-          if (!strcasecmp(ev, control_event_table[i].event_name)) {
-            event_code = control_event_table[i].event_code;
-            break;
-          }
-        }
-
-        if (event_code == -1) {
-          connection_printf_to_buf(conn, "552 Unrecognized event \"%s\"\r\n",
-                                   ev);
-          SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-          smartlist_free(events);
-          return 0;
-        }
-      }
-      event_mask |= (((event_mask_t)1) << event_code);
-    }
-  SMARTLIST_FOREACH_END(ev);
-  SMARTLIST_FOREACH(events, char *, e, tor_free(e));
-  smartlist_free(events);
-
-  conn->event_mask = event_mask;
-
-  control_update_global_event_mask();
-  send_control_done(conn);
-  return 0;
-}
-
-/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
- * Return a smartlist of acceptable passwords (unterminated strings of
- * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
- * failure.
- */
-smartlist_t *
-decode_hashed_passwords(config_line_t *passwords)
-{
-  char decoded[64];
-  config_line_t *cl;
-  smartlist_t *sl = smartlist_new();
-
-  tor_assert(passwords);
-
-  for (cl = passwords; cl; cl = cl->next) {
-    const char *hashed = cl->value;
-
-    if (!strcmpstart(hashed, "16:")) {
-      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
-                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
-          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
-        goto err;
-      }
-    } else {
-        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
-            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
-          goto err;
-        }
-    }
-    smartlist_add(sl,
-                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
-  }
-
-  return sl;
-
- err:
-  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
-  smartlist_free(sl);
-  return NULL;
-}
-
-/** Called when we get an AUTHENTICATE message.  Check whether the
- * authentication is valid, and if so, update the connection's state to
- * OPEN.  Reply with DONE or ERROR.
- */
-static int
-handle_control_authenticate(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  int used_quoted_string = 0;
-  const or_options_t *options = get_options();
-  const char *errstr = "Unknown error";
-  char *password;
-  size_t password_len;
-  const char *cp;
-  int i;
-  int bad_cookie=0, bad_password=0;
-  smartlist_t *sl = NULL;
-
-  if (!len) {
-    password = tor_strdup("");
-    password_len = 0;
-  } else if (TOR_ISXDIGIT(body[0])) {
-    cp = body;
-    while (TOR_ISXDIGIT(*cp))
-      ++cp;
-    i = (int)(cp - body);
-    tor_assert(i>0);
-    password_len = i/2;
-    password = tor_malloc(password_len + 1);
-    if (base16_decode(password, password_len+1, body, i)
-                      != (int) password_len) {
-      connection_write_str_to_buf(
-            "551 Invalid hexadecimal encoding.  Maybe you tried a plain text "
-            "password?  If so, the standard requires that you put it in "
-            "double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(password);
-      return 0;
-    }
-  } else {
-    if (!decode_escaped_string(body, len, &password, &password_len)) {
-      connection_write_str_to_buf("551 Invalid quoted string.  You need "
-            "to put the password in double quotes.\r\n", conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return 0;
-    }
-    used_quoted_string = 1;
-  }
-
-  if (conn->safecookie_client_hash != NULL) {
-    /* The controller has chosen safe cookie authentication; the only
-     * acceptable authentication value is the controller-to-server
-     * response. */
-
-    tor_assert(authentication_cookie_is_set);
-
-    if (password_len != DIGEST256_LEN) {
-      log_warn(LD_CONTROL,
-               "Got safe cookie authentication response with wrong length "
-               "(%d)", (int)password_len);
-      errstr = "Wrong length for safe cookie response.";
-      goto err;
-    }
-
-    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
-      log_warn(LD_CONTROL,
-               "Got incorrect safe cookie authentication response");
-      errstr = "Safe cookie response did not match expected value.";
-      goto err;
-    }
-
-    tor_free(conn->safecookie_client_hash);
-    goto ok;
-  }
-
-  if (!options->CookieAuthentication && !options->HashedControlPassword &&
-      !options->HashedControlSessionPassword) {
-    /* if Tor doesn't demand any stronger authentication, then
-     * the controller can get in with anything. */
-    goto ok;
-  }
-
-  if (options->CookieAuthentication) {
-    int also_password = options->HashedControlPassword != NULL ||
-      options->HashedControlSessionPassword != NULL;
-    if (password_len != AUTHENTICATION_COOKIE_LEN) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
-                 "(%d)", (int)password_len);
-        errstr = "Wrong length on authentication cookie.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else if (tor_memneq(authentication_cookie, password, password_len)) {
-      if (!also_password) {
-        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
-        errstr = "Authentication cookie did not match expected value.";
-        goto err;
-      }
-      bad_cookie = 1;
-    } else {
-      goto ok;
-    }
-  }
-
-  if (options->HashedControlPassword ||
-      options->HashedControlSessionPassword) {
-    int bad = 0;
-    smartlist_t *sl_tmp;
-    char received[DIGEST_LEN];
-    int also_cookie = options->CookieAuthentication;
-    sl = smartlist_new();
-    if (options->HashedControlPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (options->HashedControlSessionPassword) {
-      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
-      if (!sl_tmp)
-        bad = 1;
-      else {
-        smartlist_add_all(sl, sl_tmp);
-        smartlist_free(sl_tmp);
-      }
-    }
-    if (bad) {
-      if (!also_cookie) {
-        log_warn(LD_BUG,
-                 "Couldn't decode HashedControlPassword: invalid base16");
-        errstr="Couldn't decode HashedControlPassword value in configuration.";
-        goto err;
-      }
-      bad_password = 1;
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-    } else {
-      SMARTLIST_FOREACH(sl, char *, expected,
-      {
-        secret_to_key_rfc2440(received,DIGEST_LEN,
-                              password,password_len,expected);
-        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
-                      received, DIGEST_LEN))
-          goto ok;
-      });
-      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-      smartlist_free(sl);
-      sl = NULL;
-
-      if (used_quoted_string)
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration";
-      else
-        errstr = "Password did not match HashedControlPassword value from "
-          "configuration. Maybe you tried a plain text password? "
-          "If so, the standard requires that you put it in double quotes.";
-      bad_password = 1;
-      if (!also_cookie)
-        goto err;
-    }
-  }
-
-  /** We only get here if both kinds of authentication failed. */
-  tor_assert(bad_password && bad_cookie);
-  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
-  errstr = "Password did not match HashedControlPassword *or* authentication "
-    "cookie.";
-
- err:
-  tor_free(password);
-  connection_printf_to_buf(conn, "515 Authentication failed: %s\r\n", errstr);
-  connection_mark_for_close(TO_CONN(conn));
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
- ok:
-  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
-           ")", conn->base_.s);
-  send_control_done(conn);
-  conn->base_.state = CONTROL_CONN_STATE_OPEN;
-  tor_free(password);
-  if (sl) { /* clean up */
-    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
-    smartlist_free(sl);
-  }
-  return 0;
-}
-
-/** Called when we get a SAVECONF command. Try to flush the current options to
- * disk, and report success or failure. */
-static int
-handle_control_saveconf(control_connection_t *conn, uint32_t len,
-                        const char *body)
-{
-  (void) len;
-
-  int force = !strcmpstart(body, "FORCE");
-  const or_options_t *options = get_options();
-  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
-    connection_write_str_to_buf(
-      "551 Unable to write configuration to disk.\r\n", conn);
-  } else {
-    send_control_done(conn);
-  }
-  return 0;
-}
-
-struct signal_t {
-  int sig;
-  const char *signal_name;
-};
-
-static const struct signal_t signal_table[] = {
+const struct signal_name_t signal_table[] = {
+  /* NOTE: this table is used for handling SIGNAL commands and generating
+   * SIGNAL events.  Order is significant: if there are two entries for the
+   * same numeric signal, the first one is the canonical name generated
+   * for the events. */
   { SIGHUP, "RELOAD" },
   { SIGHUP, "HUP" },
   { SIGINT, "SHUTDOWN" },
@@ -1680,3576 +175,11 @@ static const struct signal_t signal_table[] = {
   { SIGNEWNYM, "NEWNYM" },
   { SIGCLEARDNSCACHE, "CLEARDNSCACHE"},
   { SIGHEARTBEAT, "HEARTBEAT"},
+  { SIGACTIVE, "ACTIVE" },
+  { SIGDORMANT, "DORMANT" },
   { 0, NULL },
 };
 
-/** Called when we get a SIGNAL command. React to the provided signal, and
- * report success or failure. (If the signal results in a shutdown, success
- * may not be reported.) */
-static int
-handle_control_signal(control_connection_t *conn, uint32_t len,
-                      const char *body)
-{
-  int sig = -1;
-  int i;
-  int n = 0;
-  char *s;
-
-  (void) len;
-
-  while (body[n] && ! TOR_ISSPACE(body[n]))
-    ++n;
-  s = tor_strndup(body, n);
-
-  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
-    if (!strcasecmp(s, signal_table[i].signal_name)) {
-      sig = signal_table[i].sig;
-      break;
-    }
-  }
-
-  if (sig < 0)
-    connection_printf_to_buf(conn, "552 Unrecognized signal code \"%s\"\r\n",
-                             s);
-  tor_free(s);
-  if (sig < 0)
-    return 0;
-
-  send_control_done(conn);
-  /* Flush the "done" first if the signal might make us shut down. */
-  if (sig == SIGTERM || sig == SIGINT)
-    connection_flush(TO_CONN(conn));
-
-  activate_signal(sig);
-
-  return 0;
-}
-
-/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
- * as an owning connection, so that we will exit if the connection
- * closes. */
-static int
-handle_control_takeownership(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  (void)len;
-  (void)body;
-
-  conn->is_owning_control_connection = 1;
-
-  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
-           "Tor instance.",
-           (int)(conn->base_.s));
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
- * containing funny characters. */
-static int
-address_is_invalid_mapaddress_target(const char *addr)
-{
-  if (!strcmpstart(addr, "*."))
-    return address_is_invalid_destination(addr+2, 1);
-  else
-    return address_is_invalid_destination(addr, 1);
-}
-
-/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
- * and report success or failure. */
-static int
-handle_control_mapaddress(control_connection_t *conn, uint32_t len,
-                          const char *body)
-{
-  smartlist_t *elts;
-  smartlist_t *lines;
-  smartlist_t *reply;
-  char *r;
-  size_t sz;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  lines = smartlist_new();
-  elts = smartlist_new();
-  reply = smartlist_new();
-  smartlist_split_string(lines, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(lines, char *, line) {
-    tor_strlower(line);
-    smartlist_split_string(elts, line, "=", 0, 2);
-    if (smartlist_len(elts) == 2) {
-      const char *from = smartlist_get(elts,0);
-      const char *to = smartlist_get(elts,1);
-      if (address_is_invalid_mapaddress_target(to)) {
-        smartlist_add_asprintf(reply,
-                     "512-syntax error: invalid address '%s'", to);
-        log_warn(LD_CONTROL,
-                 "Skipping invalid argument '%s' in MapAddress msg", to);
-      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
-                 !strcmp(from, "::")) {
-        const char type =
-          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
-          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
-        const char *address = addressmap_register_virtual_address(
-                                                     type, tor_strdup(to));
-        if (!address) {
-          smartlist_add_asprintf(reply,
-                       "451-resource exhausted: skipping '%s'", line);
-          log_warn(LD_CONTROL,
-                   "Unable to allocate address for '%s' in MapAddress msg",
-                   safe_str_client(line));
-        } else {
-          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
-        }
-      } else {
-        const char *msg;
-        if (addressmap_register_auto(from, to, 1,
-                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
-          smartlist_add_asprintf(reply,
-                                 "512-syntax error: invalid address mapping "
-                                 " '%s': %s", line, msg);
-          log_warn(LD_CONTROL,
-                   "Skipping invalid argument '%s' in MapAddress msg: %s",
-                   line, msg);
-        } else {
-          smartlist_add_asprintf(reply, "250-%s", line);
-        }
-      }
-    } else {
-      smartlist_add_asprintf(reply, "512-syntax error: mapping '%s' is "
-                   "not of expected form 'foo=bar'.", line);
-      log_info(LD_CONTROL, "Skipping MapAddress '%s': wrong "
-                           "number of items.",
-                           safe_str_client(line));
-    }
-    SMARTLIST_FOREACH(elts, char *, cp, tor_free(cp));
-    smartlist_clear(elts);
-  } SMARTLIST_FOREACH_END(line);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  smartlist_free(elts);
-
-  if (smartlist_len(reply)) {
-    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
-    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
-    connection_buf_add(r, sz, TO_CONN(conn));
-    tor_free(r);
-  } else {
-    const char *response =
-      "512 syntax error: not enough arguments to mapaddress.\r\n";
-    connection_buf_add(response, strlen(response), TO_CONN(conn));
-  }
-
-  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
-  smartlist_free(reply);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for various
- * trivial-to-implement questions. */
-static int
-getinfo_helper_misc(control_connection_t *conn, const char *question,
-                    char **answer, const char **errmsg)
-{
-  (void) conn;
-  if (!strcmp(question, "version")) {
-    *answer = tor_strdup(get_version());
-  } else if (!strcmp(question, "bw-event-cache")) {
-    *answer = get_bw_samples();
-  } else if (!strcmp(question, "config-file")) {
-    const char *a = get_torrc_fname(0);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-defaults-file")) {
-    const char *a = get_torrc_fname(1);
-    if (a)
-      *answer = tor_strdup(a);
-  } else if (!strcmp(question, "config-text")) {
-    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
-  } else if (!strcmp(question, "config-can-saveconf")) {
-    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
-  } else if (!strcmp(question, "info/names")) {
-    *answer = list_getinfo_options();
-  } else if (!strcmp(question, "dormant")) {
-    int dormant = rep_hist_circbuilding_dormant(time(NULL));
-    *answer = tor_strdup(dormant ? "1" : "0");
-  } else if (!strcmp(question, "events/names")) {
-    int i;
-    smartlist_t *event_names = smartlist_new();
-
-    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
-      smartlist_add(event_names, (char *)control_event_table[i].event_name);
-    }
-
-    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
-
-    smartlist_free(event_names);
-  } else if (!strcmp(question, "signal/names")) {
-    smartlist_t *signal_names = smartlist_new();
-    int j;
-    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
-      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
-    }
-
-    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
-
-    smartlist_free(signal_names);
-  } else if (!strcmp(question, "features/names")) {
-    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
-  } else if (!strcmp(question, "address")) {
-    uint32_t addr;
-    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
-      *errmsg = "Address unknown";
-      return -1;
-    }
-    *answer = tor_dup_ip(addr);
-  } else if (!strcmp(question, "traffic/read")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
-  } else if (!strcmp(question, "traffic/written")) {
-    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
-  } else if (!strcmp(question, "uptime")) {
-    long uptime_secs = get_uptime();
-    tor_asprintf(answer, "%ld", uptime_secs);
-  } else if (!strcmp(question, "process/pid")) {
-    int myPid = -1;
-
-#ifdef _WIN32
-      myPid = _getpid();
-#else
-      myPid = getpid();
-#endif
-
-    tor_asprintf(answer, "%d", myPid);
-  } else if (!strcmp(question, "process/uid")) {
-#ifdef _WIN32
-      *answer = tor_strdup("-1");
-#else
-      int myUid = geteuid();
-      tor_asprintf(answer, "%d", myUid);
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/user")) {
-#ifdef _WIN32
-      *answer = tor_strdup("");
-#else
-      int myUid = geteuid();
-      const struct passwd *myPwEntry = tor_getpwuid(myUid);
-
-      if (myPwEntry) {
-        *answer = tor_strdup(myPwEntry->pw_name);
-      } else {
-        *answer = tor_strdup("");
-      }
-#endif /* defined(_WIN32) */
-  } else if (!strcmp(question, "process/descriptor-limit")) {
-    int max_fds = get_max_sockets();
-    tor_asprintf(answer, "%d", max_fds);
-  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
-    tor_asprintf(answer, "%"PRIu64,
-                 (get_options()->MaxMemInQueues));
-  } else if (!strcmp(question, "fingerprint")) {
-    crypto_pk_t *server_key;
-    if (!server_mode(get_options())) {
-      *errmsg = "Not running in server mode";
-      return -1;
-    }
-    server_key = get_server_identity_key();
-    *answer = tor_malloc(HEX_DIGEST_LEN+1);
-    crypto_pk_get_fingerprint(server_key, *answer, 0);
-  }
-  return 0;
-}
-
-/** Awful hack: return a newly allocated string based on a routerinfo and
- * (possibly) an extrainfo, sticking the read-history and write-history from
- * <b>ei</b> into the resulting string.  The thing you get back won't
- * necessarily have a valid signature.
- *
- * New code should never use this; it's for backward compatibility.
- *
- * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
- * not be NUL-terminated. */
-static char *
-munge_extrainfo_into_routerinfo(const char *ri_body,
-                                const signed_descriptor_t *ri,
-                                const signed_descriptor_t *ei)
-{
-  char *out = NULL, *outp;
-  int i;
-  const char *router_sig;
-  const char *ei_body = signed_descriptor_get_body(ei);
-  size_t ri_len = ri->signed_descriptor_len;
-  size_t ei_len = ei->signed_descriptor_len;
-  if (!ei_body)
-    goto bail;
-
-  outp = out = tor_malloc(ri_len+ei_len+1);
-  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
-    goto bail;
-  ++router_sig;
-  memcpy(out, ri_body, router_sig-ri_body);
-  outp += router_sig-ri_body;
-
-  for (i=0; i < 2; ++i) {
-    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
-    const char *cp, *eol;
-    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
-      continue;
-    ++cp;
-    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
-      continue;
-    memcpy(outp, cp, eol-cp+1);
-    outp += eol-cp+1;
-  }
-  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
-  *outp++ = '\0';
-  tor_assert(outp-out < (int)(ri_len+ei_len+1));
-
-  return out;
- bail:
-  tor_free(out);
-  return tor_strndup(ri_body, ri->signed_descriptor_len);
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * which ports are bound. */
-static int
-getinfo_helper_listeners(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  int type;
-  smartlist_t *res;
-
-  (void)control_conn;
-  (void)errmsg;
-
-  if (!strcmp(question, "net/listeners/or"))
-    type = CONN_TYPE_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/extor"))
-    type = CONN_TYPE_EXT_OR_LISTENER;
-  else if (!strcmp(question, "net/listeners/dir"))
-    type = CONN_TYPE_DIR_LISTENER;
-  else if (!strcmp(question, "net/listeners/socks"))
-    type = CONN_TYPE_AP_LISTENER;
-  else if (!strcmp(question, "net/listeners/trans"))
-    type = CONN_TYPE_AP_TRANS_LISTENER;
-  else if (!strcmp(question, "net/listeners/natd"))
-    type = CONN_TYPE_AP_NATD_LISTENER;
-  else if (!strcmp(question, "net/listeners/httptunnel"))
-    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
-  else if (!strcmp(question, "net/listeners/dns"))
-    type = CONN_TYPE_AP_DNS_LISTENER;
-  else if (!strcmp(question, "net/listeners/control"))
-    type = CONN_TYPE_CONTROL_LISTENER;
-  else
-    return 0; /* unknown key */
-
-  res = smartlist_new();
-  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
-    struct sockaddr_storage ss;
-    socklen_t ss_len = sizeof(ss);
-
-    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
-      continue;
-
-    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
-      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
-    } else {
-      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
-      smartlist_add(res, esc_for_log(tmp));
-      tor_free(tmp);
-    }
-
-  } SMARTLIST_FOREACH_END(conn);
-
-  *answer = smartlist_join_strings(res, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
-  smartlist_free(res);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers requests for information about
- * the current time in both local and UTC forms. */
-STATIC int
-getinfo_helper_current_time(control_connection_t *control_conn,
-                         const char *question,
-                         char **answer, const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-
-  struct timeval now;
-  tor_gettimeofday(&now);
-  char timebuf[ISO_TIME_LEN+1];
-
-  if (!strcmp(question, "current-time/local"))
-    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else if (!strcmp(question, "current-time/utc"))
-    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
-  else
-    return 0;
-
-  *answer = tor_strdup(timebuf);
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * directory information. */
-STATIC int
-getinfo_helper_dir(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  (void) control_conn;
-  if (!strcmpstart(question, "desc/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/id/* "
-                "instead of desc/id/*.";
-      return 0;
-    }
-  } else if (!strcmpstart(question, "desc/name/")) {
-    const routerinfo_t *ri = NULL;
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node =
-      node_get_by_nickname(question+strlen("desc/name/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *body = signed_descriptor_get_body(&ri->cache_info);
-      if (body)
-        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
-    } else if (! we_fetch_router_descriptors(get_options())) {
-      /* Descriptors won't be available, provide proper error */
-      *errmsg = "We fetch microdescriptors, not router "
-                "descriptors. You'll need to use md/name/* "
-                "instead of desc/name/*.";
-      return 0;
-    }
-  } else if (!strcmp(question, "desc/download-enabled")) {
-    int r = we_fetch_router_descriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmp(question, "desc/all-recent")) {
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
-      {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        if (body)
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-      });
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
-    /* XXXX Remove this once Torstat asks for extrainfos. */
-    routerlist_t *routerlist = router_get_routerlist();
-    smartlist_t *sl = smartlist_new();
-    if (routerlist && routerlist->routers) {
-      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
-        const char *body = signed_descriptor_get_body(&ri->cache_info);
-        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
-                                     ri->cache_info.extra_info_digest);
-        if (ei && body) {
-          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
-                                                        &ri->cache_info, ei));
-        } else if (body) {
-          smartlist_add(sl,
-                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
-        }
-      } SMARTLIST_FOREACH_END(ri);
-    }
-    *answer = smartlist_join_strings(sl, "", 0, NULL);
-    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
-    smartlist_free(sl);
-  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/client/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      rend_cache_entry_t *e = NULL;
-      if (!rend_cache_lookup_entry(question, -1, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      const char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_cache_lookup_encoded_as_client(&service_pk);
-      if (desc) {
-        *answer = tor_strdup(desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
-    hostname_type_t addr_type;
-
-    question += strlen("hs/service/desc/id/");
-    if (rend_valid_v2_service_id(question)) {
-      addr_type = ONION_V2_HOSTNAME;
-    } else if (hs_address_is_valid(question)) {
-      addr_type = ONION_V3_HOSTNAME;
-    } else {
-      *errmsg = "Invalid address";
-      return -1;
-    }
-    rend_cache_entry_t *e = NULL;
-
-    if (addr_type == ONION_V2_HOSTNAME) {
-      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
-        /* Descriptor found in cache */
-        *answer = tor_strdup(e->desc);
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    } else {
-      ed25519_public_key_t service_pk;
-      char *desc;
-
-      /* The check before this if/else makes sure of this. */
-      tor_assert(addr_type == ONION_V3_HOSTNAME);
-
-      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
-        *errmsg = "Invalid v3 address";
-        return -1;
-      }
-
-      desc = hs_service_lookup_current_desc(&service_pk);
-      if (desc) {
-        /* Newly allocated string, we have ownership. */
-        *answer = desc;
-      } else {
-        *errmsg = "Not found in cache";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "md/all")) {
-    const smartlist_t *nodes = nodelist_get_list();
-    tor_assert(nodes);
-
-    if (smartlist_len(nodes) == 0) {
-      *answer = tor_strdup("");
-      return 0;
-    }
-
-    smartlist_t *microdescs = smartlist_new();
-
-    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
-      if (n->md && n->md->body) {
-        char *copy = tor_strndup(n->md->body, n->md->bodylen);
-        smartlist_add(microdescs, copy);
-      }
-    } SMARTLIST_FOREACH_END(n);
-
-    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
-    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
-    smartlist_free(microdescs);
-  } else if (!strcmpstart(question, "md/id/")) {
-    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmpstart(question, "md/name/")) {
-    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
-     * warning goes to the user, not to the controller. */
-    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
-    /* XXXX duplicated code */
-    const microdesc_t *md = NULL;
-    if (node) md = node->md;
-    if (md && md->body) {
-      *answer = tor_strndup(md->body, md->bodylen);
-    }
-  } else if (!strcmp(question, "md/download-enabled")) {
-    int r = we_fetch_microdescriptors(get_options());
-    tor_asprintf(answer, "%d", !!r);
-  } else if (!strcmpstart(question, "desc-annotations/id/")) {
-    const routerinfo_t *ri = NULL;
-    const node_t *node =
-      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
-    if (node)
-      ri = node->ri;
-    if (ri) {
-      const char *annotations =
-        signed_descriptor_get_annotations(&ri->cache_info);
-      if (annotations)
-        *answer = tor_strndup(annotations,
-                              ri->cache_info.annotations_len);
-    }
-  } else if (!strcmpstart(question, "dir/server/")) {
-    size_t answer_len = 0;
-    char *url = NULL;
-    smartlist_t *descs = smartlist_new();
-    const char *msg;
-    int res;
-    char *cp;
-    tor_asprintf(&url, "/tor/%s", question+4);
-    res = dirserv_get_routerdescs(descs, url, &msg);
-    if (res) {
-      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
-      smartlist_free(descs);
-      tor_free(url);
-      *errmsg = msg;
-      return -1;
-    }
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      answer_len += sd->signed_descriptor_len);
-    cp = *answer = tor_malloc(answer_len+1);
-    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
-                      {
-                        memcpy(cp, signed_descriptor_get_body(sd),
-                               sd->signed_descriptor_len);
-                        cp += sd->signed_descriptor_len;
-                      });
-    *cp = '\0';
-    tor_free(url);
-    smartlist_free(descs);
-  } else if (!strcmpstart(question, "dir/status/")) {
-    *answer = tor_strdup("");
-  } else if (!strcmp(question, "dir/status-vote/current/consensus")) { /* v3 */
-    if (we_want_to_fetch_flavor(get_options(), FLAV_NS)) {
-      const cached_dir_t *consensus = dirserv_get_consensus("ns");
-      if (consensus)
-        *answer = tor_strdup(consensus->dir);
-    }
-    if (!*answer) { /* try loading it from disk */
-      *answer = networkstatus_read_cached_consensus("ns");
-      if (!*answer) { /* generate an error */
-        *errmsg = "Could not open cached consensus. "
-          "Make sure FetchUselessDescriptors is set to 1.";
-        return -1;
-      }
-    }
-  } else if (!strcmp(question, "network-status")) { /* v1 */
-    static int network_status_warned = 0;
-    if (!network_status_warned) {
-      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
-               "go away in a future version of Tor.");
-      network_status_warned = 1;
-    }
-    routerlist_t *routerlist = router_get_routerlist();
-    if (!routerlist || !routerlist->routers ||
-        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
-      return -1;
-    }
-  } else if (!strcmpstart(question, "extra-info/digest/")) {
-    question += strlen("extra-info/digest/");
-    if (strlen(question) == HEX_DIGEST_LEN) {
-      char d[DIGEST_LEN];
-      signed_descriptor_t *sd = NULL;
-      if (base16_decode(d, sizeof(d), question, strlen(question))
-                        == sizeof(d)) {
-        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
-         * but I don't want to risk affecting other parts of the code,
-         * especially since the rules for using our own extrainfo (including
-         * when it might be freed) are different from those for using one
-         * we have downloaded. */
-        if (router_extrainfo_digest_is_me(d))
-          sd = &(router_get_my_extrainfo()->cache_info);
-        else
-          sd = extrainfo_get_by_descriptor_digest(d);
-      }
-      if (sd) {
-        const char *body = signed_descriptor_get_body(sd);
-        if (body)
-          *answer = tor_strndup(body, sd->signed_descriptor_len);
-      }
-    }
-  }
-
-  return 0;
-}
-
-/** Given a smartlist of 20-byte digests, return a newly allocated string
- * containing each of those digests in order, formatted in HEX, and terminated
- * with a newline. */
-static char *
-digest_list_to_string(const smartlist_t *sl)
-{
-  int len;
-  char *result, *s;
-
-  /* Allow for newlines, and a \0 at the end */
-  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
-  result = tor_malloc_zero(len);
-
-  s = result;
-  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
-    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
-    s[HEX_DIGEST_LEN] = '\n';
-    s += HEX_DIGEST_LEN + 1;
-  } SMARTLIST_FOREACH_END(digest);
-  *s = '\0';
-
-  return result;
-}
-
-/** Turn a download_status_t into a human-readable description in a newly
- * allocated string.  The format is specified in control-spec.txt, under
- * the documentation for "GETINFO download/..." .  */
-static char *
-download_status_to_string(const download_status_t *dl)
-{
-  char *rv = NULL;
-  char tbuf[ISO_TIME_LEN+1];
-  const char *schedule_str, *want_authority_str;
-  const char *increment_on_str, *backoff_str;
-
-  if (dl) {
-    /* Get some substrings of the eventual output ready */
-    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
-
-    switch (dl->schedule) {
-      case DL_SCHED_GENERIC:
-        schedule_str = "DL_SCHED_GENERIC";
-        break;
-      case DL_SCHED_CONSENSUS:
-        schedule_str = "DL_SCHED_CONSENSUS";
-        break;
-      case DL_SCHED_BRIDGE:
-        schedule_str = "DL_SCHED_BRIDGE";
-        break;
-      default:
-        schedule_str = "unknown";
-        break;
-    }
-
-    switch (dl->want_authority) {
-      case DL_WANT_ANY_DIRSERVER:
-        want_authority_str = "DL_WANT_ANY_DIRSERVER";
-        break;
-      case DL_WANT_AUTHORITY:
-        want_authority_str = "DL_WANT_AUTHORITY";
-        break;
-      default:
-        want_authority_str = "unknown";
-        break;
-    }
-
-    switch (dl->increment_on) {
-      case DL_SCHED_INCREMENT_FAILURE:
-        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
-        break;
-      case DL_SCHED_INCREMENT_ATTEMPT:
-        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
-        break;
-      default:
-        increment_on_str = "unknown";
-        break;
-    }
-
-    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
-
-    /* Now assemble them */
-    tor_asprintf(&rv,
-                 "next-attempt-at %s\n"
-                 "n-download-failures %u\n"
-                 "n-download-attempts %u\n"
-                 "schedule %s\n"
-                 "want-authority %s\n"
-                 "increment-on %s\n"
-                 "backoff %s\n"
-                 "last-backoff-position %u\n"
-                 "last-delay-used %d\n",
-                 tbuf,
-                 dl->n_download_failures,
-                 dl->n_download_attempts,
-                 schedule_str,
-                 want_authority_str,
-                 increment_on_str,
-                 backoff_str,
-                 dl->last_backoff_position,
-                 dl->last_delay_used);
-  }
-
-  return rv;
-}
-
-/** Handle the consensus download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_networkstatus(const char *flavor,
-                                       download_status_t **dl_to_emit,
-                                       const char **errmsg)
-{
-  /*
-   * We get the one for the current bootstrapped status by default, or
-   * take an extra /bootstrap or /running suffix
-   */
-  if (strcmp(flavor, "ns") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
-  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
-  } else if (strcmp(flavor, "ns/running") == 0 ) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
-  } else if (strcmp(flavor, "microdesc") == 0) {
-    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
-  } else if (strcmp(flavor, "microdesc/running") == 0) {
-    *dl_to_emit =
-      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
-  } else {
-    *errmsg = "Unknown flavor";
-  }
-}
-
-/** Handle the cert download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_cert(const char *fp_sk_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  const char *sk_req;
-  char id_digest[DIGEST_LEN];
-  char sk_digest[DIGEST_LEN];
-
-  /*
-   * We have to handle four cases; fp_sk_req is the request with
-   * a prefix of "downloads/cert/" snipped off.
-   *
-   * Case 1: fp_sk_req = "fps"
-   *  - We should emit a digest_list with a list of all the identity
-   *    fingerprints that can be queried for certificate download status;
-   *    get it by calling list_authority_ids_with_downloads().
-   *
-   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
-   *  - We want the default certificate for this identity fingerprint's
-   *    download status; this is the download we get from URLs starting
-   *    in /fp/ on the directory server.  We can get it with
-   *    id_only_download_status_for_authority_id().
-   *
-   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
-   *  - We want a list of all signing key digests for this identity
-   *    fingerprint which can be queried for certificate download status.
-   *    Get it with list_sk_digests_for_authority_id().
-   *
-   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
-   *         signing key digest sk
-   *   - We want the download status for the certificate for this specific
-   *     signing key and fingerprint.  These correspond to the ones we get
-   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
-   *     list_sk_digests_for_authority_id().
-   */
-
-  if (strcmp(fp_sk_req, "fps") == 0) {
-    *digest_list = list_authority_ids_with_downloads();
-    if (!(*digest_list)) {
-      *errmsg = "Failed to get list of authority identity digests (!)";
-    }
-  } else if (!strcmpstart(fp_sk_req, "fp/")) {
-    fp_sk_req += strlen("fp/");
-    /* Okay, look for another / to tell the fp from fp-sk cases */
-    sk_req = strchr(fp_sk_req, '/');
-    if (sk_req) {
-      /* okay, split it here and try to parse <fp> */
-      if (base16_decode(id_digest, DIGEST_LEN,
-                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
-        /* Skip past the '/' */
-        ++sk_req;
-        if (strcmp(sk_req, "sks") == 0) {
-          /* We're asking for the list of signing key fingerprints */
-          *digest_list = list_sk_digests_for_authority_id(id_digest);
-          if (!(*digest_list)) {
-            *errmsg = "Failed to get list of signing key digests for this "
-                      "authority identity digest";
-          }
-        } else {
-          /* We've got a signing key digest */
-          if (base16_decode(sk_digest, DIGEST_LEN,
-                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
-            *dl_to_emit =
-              download_status_for_authority_id_and_sk(id_digest, sk_digest);
-            if (!(*dl_to_emit)) {
-              *errmsg = "Failed to get download status for this identity/"
-                        "signing key digest pair";
-            }
-          } else {
-            *errmsg = "That didn't look like a signing key digest";
-          }
-        }
-      } else {
-        *errmsg = "That didn't look like an identity digest";
-      }
-    } else {
-      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
-      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
-        if (base16_decode(id_digest, DIGEST_LEN,
-                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
-          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
-          if (!(*dl_to_emit)) {
-            *errmsg = "Failed to get download status for this authority "
-                      "identity digest";
-          }
-        } else {
-          *errmsg = "That didn't look like a digest";
-        }
-      } else {
-        *errmsg = "That didn't look like a digest";
-      }
-    }
-  } else {
-    *errmsg = "Unknown certificate download status query";
-  }
-}
-
-/** Handle the routerdesc download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_desc(const char *desc_req,
-                              download_status_t **dl_to_emit,
-                              smartlist_t **digest_list,
-                              const char **errmsg)
-{
-  char desc_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: desc_req = "descs"
-   *   - Emit a list of all router descriptor digests, which we get by
-   *     calling router_get_descriptor_digests(); this can return NULL
-   *     if we have no current ns-flavor consensus.
-   *
-   * Case 2: desc_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using router_get_dl_status_by_descriptor_digest().
-   */
-
-  if (strcmp(desc_req, "descs") == 0) {
-    *digest_list = router_get_descriptor_digests();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
-    }
-    /*
-     * Microdescs don't use the download_status_t mechanism, so we don't
-     * answer queries about their downloads here; see microdesc.c.
-     */
-  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(desc_digest, DIGEST_LEN,
-                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such descriptor digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown router descriptor download status query";
-  }
-}
-
-/** Handle the bridge download cases for getinfo_helper_downloads() */
-STATIC void
-getinfo_helper_downloads_bridge(const char *bridge_req,
-                                download_status_t **dl_to_emit,
-                                smartlist_t **digest_list,
-                                const char **errmsg)
-{
-  char bridge_digest[DIGEST_LEN];
-  /*
-   * Two cases to handle here:
-   *
-   * Case 1: bridge_req = "bridges"
-   *   - Emit a list of all bridge identity digests, which we get by
-   *     calling list_bridge_identities(); this can return NULL if we are
-   *     not using bridges.
-   *
-   * Case 2: bridge_req = <fp>
-   *   - Check on the specified fingerprint and emit its download_status_t
-   *     using get_bridge_dl_status_by_id().
-   */
-
-  if (strcmp(bridge_req, "bridges") == 0) {
-    *digest_list = list_bridge_identities();
-    if (!(*digest_list)) {
-      *errmsg = "We don't seem to be using bridges";
-    }
-  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
-    if (base16_decode(bridge_digest, DIGEST_LEN,
-                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
-      /* Okay we got a digest-shaped thing; try asking for it */
-      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
-      if (!(*dl_to_emit)) {
-        *errmsg = "No such bridge identity digest found";
-      }
-    } else {
-      *errmsg = "That didn't look like a digest";
-    }
-  } else {
-    *errmsg = "Unknown bridge descriptor download status query";
-  }
-}
-
-/** Implementation helper for GETINFO: knows the answers for questions about
- * download status information. */
-STATIC int
-getinfo_helper_downloads(control_connection_t *control_conn,
-                   const char *question, char **answer,
-                   const char **errmsg)
-{
-  download_status_t *dl_to_emit = NULL;
-  smartlist_t *digest_list = NULL;
-
-  /* Assert args are sane */
-  tor_assert(control_conn != NULL);
-  tor_assert(question != NULL);
-  tor_assert(answer != NULL);
-  tor_assert(errmsg != NULL);
-
-  /* We check for this later to see if we should supply a default */
-  *errmsg = NULL;
-
-  /* Are we after networkstatus downloads? */
-  if (!strcmpstart(question, "downloads/networkstatus/")) {
-    getinfo_helper_downloads_networkstatus(
-        question + strlen("downloads/networkstatus/"),
-        &dl_to_emit, errmsg);
-  /* Certificates? */
-  } else if (!strcmpstart(question, "downloads/cert/")) {
-    getinfo_helper_downloads_cert(
-        question + strlen("downloads/cert/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Router descriptors? */
-  } else if (!strcmpstart(question, "downloads/desc/")) {
-    getinfo_helper_downloads_desc(
-        question + strlen("downloads/desc/"),
-        &dl_to_emit, &digest_list, errmsg);
-  /* Bridge descriptors? */
-  } else if (!strcmpstart(question, "downloads/bridge/")) {
-    getinfo_helper_downloads_bridge(
-        question + strlen("downloads/bridge/"),
-        &dl_to_emit, &digest_list, errmsg);
-  } else {
-    *errmsg = "Unknown download status query";
-  }
-
-  if (dl_to_emit) {
-    *answer = download_status_to_string(dl_to_emit);
-
-    return 0;
-  } else if (digest_list) {
-    *answer = digest_list_to_string(digest_list);
-    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
-    smartlist_free(digest_list);
-
-    return 0;
-  } else {
-    if (!(*errmsg)) {
-      *errmsg = "Unknown error";
-    }
-
-    return -1;
-  }
-}
-
-/** Allocate and return a description of <b>circ</b>'s current status,
- * including its path (if any). */
-static char *
-circuit_describe_status_for_controller(origin_circuit_t *circ)
-{
-  char *rv;
-  smartlist_t *descparts = smartlist_new();
-
-  {
-    char *vpath = circuit_list_path_for_controller(circ);
-    if (*vpath) {
-      smartlist_add(descparts, vpath);
-    } else {
-      tor_free(vpath); /* empty path; don't put an extra space in the result */
-    }
-  }
-
-  {
-    cpath_build_state_t *build_state = circ->build_state;
-    smartlist_t *flaglist = smartlist_new();
-    char *flaglist_joined;
-
-    if (build_state->onehop_tunnel)
-      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
-    if (build_state->is_internal)
-      smartlist_add(flaglist, (void *)"IS_INTERNAL");
-    if (build_state->need_capacity)
-      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
-    if (build_state->need_uptime)
-      smartlist_add(flaglist, (void *)"NEED_UPTIME");
-
-    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
-    if (smartlist_len(flaglist)) {
-      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
-
-      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
-
-      tor_free(flaglist_joined);
-    }
-
-    smartlist_free(flaglist);
-  }
-
-  smartlist_add_asprintf(descparts, "PURPOSE=%s",
-                    circuit_purpose_to_controller_string(circ->base_.purpose));
-
-  {
-    const char *hs_state =
-      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
-
-    if (hs_state != NULL) {
-      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
-    }
-  }
-
-  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
-    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
-    const char *onion_address;
-    if (circ->rend_data) {
-      onion_address = rend_data_get_address(circ->rend_data);
-    } else {
-      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
-      onion_address = addr;
-    }
-    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
-  }
-
-  {
-    char tbuf[ISO_TIME_USEC_LEN+1];
-    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
-
-    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
-  }
-
-  // Show username and/or password if available.
-  if (circ->socks_username_len > 0) {
-    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
-                                     (size_t) circ->socks_username_len);
-    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
-                           socks_username_escaped);
-    tor_free(socks_username_escaped);
-  }
-  if (circ->socks_password_len > 0) {
-    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
-                                     (size_t) circ->socks_password_len);
-    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
-                           socks_password_escaped);
-    tor_free(socks_password_escaped);
-  }
-
-  rv = smartlist_join_strings(descparts, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
-  smartlist_free(descparts);
-
-  return rv;
-}
-
-/** Implementation helper for GETINFO: knows how to generate summaries of the
- * current states of things we send events about. */
-static int
-getinfo_helper_events(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  const or_options_t *options = get_options();
-  (void) control_conn;
-  if (!strcmp(question, "circuit-status")) {
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
-      origin_circuit_t *circ;
-      char *circdesc;
-      const char *state;
-      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
-        continue;
-      circ = TO_ORIGIN_CIRCUIT(circ_);
-
-      if (circ->base_.state == CIRCUIT_STATE_OPEN)
-        state = "BUILT";
-      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
-        state = "GUARD_WAIT";
-      else if (circ->cpath)
-        state = "EXTENDED";
-      else
-        state = "LAUNCHED";
-
-      circdesc = circuit_describe_status_for_controller(circ);
-
-      smartlist_add_asprintf(status, "%lu %s%s%s",
-                   (unsigned long)circ->global_identifier,
-                   state, *circdesc ? " " : "", circdesc);
-      tor_free(circdesc);
-    }
-    SMARTLIST_FOREACH_END(circ_);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "stream-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    char buf[256];
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      entry_connection_t *conn;
-      circuit_t *circ;
-      origin_circuit_t *origin_circ = NULL;
-      if (base_conn->type != CONN_TYPE_AP ||
-          base_conn->marked_for_close ||
-          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
-          base_conn->state == AP_CONN_STATE_NATD_WAIT)
-        continue;
-      conn = TO_ENTRY_CONN(base_conn);
-      switch (base_conn->state)
-        {
-        case AP_CONN_STATE_CONTROLLER_WAIT:
-        case AP_CONN_STATE_CIRCUIT_WAIT:
-          if (conn->socks_request &&
-              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
-            state = "NEWRESOLVE";
-          else
-            state = "NEW";
-          break;
-        case AP_CONN_STATE_RENDDESC_WAIT:
-        case AP_CONN_STATE_CONNECT_WAIT:
-          state = "SENTCONNECT"; break;
-        case AP_CONN_STATE_RESOLVE_WAIT:
-          state = "SENTRESOLVE"; break;
-        case AP_CONN_STATE_OPEN:
-          state = "SUCCEEDED"; break;
-        default:
-          log_warn(LD_BUG, "Asked for stream in unknown state %d",
-                   base_conn->state);
-          continue;
-        }
-      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-      if (circ && CIRCUIT_IS_ORIGIN(circ))
-        origin_circ = TO_ORIGIN_CIRCUIT(circ);
-      write_stream_target_to_buf(conn, buf, sizeof(buf));
-      smartlist_add_asprintf(status, "%lu %s %lu %s",
-                   (unsigned long) base_conn->global_identifier,state,
-                   origin_circ?
-                         (unsigned long)origin_circ->global_identifier : 0ul,
-                   buf);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmp(question, "orconn-status")) {
-    smartlist_t *conns = get_connection_array();
-    smartlist_t *status = smartlist_new();
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-      const char *state;
-      char name[128];
-      or_connection_t *conn;
-      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
-        continue;
-      conn = TO_OR_CONN(base_conn);
-      if (conn->base_.state == OR_CONN_STATE_OPEN)
-        state = "CONNECTED";
-      else if (conn->nickname)
-        state = "LAUNCHED";
-      else
-        state = "NEW";
-      orconn_target_get_name(name, sizeof(name), conn);
-      smartlist_add_asprintf(status, "%s %s", name, state);
-    } SMARTLIST_FOREACH_END(base_conn);
-    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
-    smartlist_free(status);
-  } else if (!strcmpstart(question, "address-mappings/")) {
-    time_t min_e, max_e;
-    smartlist_t *mappings;
-    question += strlen("address-mappings/");
-    if (!strcmp(question, "all")) {
-      min_e = 0; max_e = TIME_MAX;
-    } else if (!strcmp(question, "cache")) {
-      min_e = 2; max_e = TIME_MAX;
-    } else if (!strcmp(question, "config")) {
-      min_e = 0; max_e = 0;
-    } else if (!strcmp(question, "control")) {
-      min_e = 1; max_e = 1;
-    } else {
-      return 0;
-    }
-    mappings = smartlist_new();
-    addressmap_get_mappings(mappings, min_e, max_e, 1);
-    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
-    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
-    smartlist_free(mappings);
-  } else if (!strcmpstart(question, "status/")) {
-    /* Note that status/ is not a catch-all for events; there's only supposed
-     * to be a status GETINFO if there's a corresponding STATUS event. */
-    if (!strcmp(question, "status/circuit-established")) {
-      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
-    } else if (!strcmp(question, "status/enough-dir-info")) {
-      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
-    } else if (!strcmp(question, "status/good-server-descriptor") ||
-               !strcmp(question, "status/accepted-server-descriptor")) {
-      /* They're equivalent for now, until we can figure out how to make
-       * good-server-descriptor be what we want. See comment in
-       * control-spec.txt. */
-      *answer = tor_strdup(directories_have_accepted_server_descriptor()
-                           ? "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
-      *answer = tor_strdup(check_whether_orport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
-      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
-                            "1" : "0");
-    } else if (!strcmp(question, "status/reachability-succeeded")) {
-      tor_asprintf(answer, "OR=%d DIR=%d",
-                   check_whether_orport_reachable(options) ? 1 : 0,
-                   check_whether_dirport_reachable(options) ? 1 : 0);
-    } else if (!strcmp(question, "status/bootstrap-phase")) {
-      *answer = tor_strdup(last_sent_bootstrap_message);
-    } else if (!strcmpstart(question, "status/version/")) {
-      int is_server = server_mode(options);
-      networkstatus_t *c = networkstatus_get_latest_consensus();
-      version_status_t status;
-      const char *recommended;
-      if (c) {
-        recommended = is_server ? c->server_versions : c->client_versions;
-        status = tor_version_is_obsolete(VERSION, recommended);
-      } else {
-        recommended = "?";
-        status = VS_UNKNOWN;
-      }
-
-      if (!strcmp(question, "status/version/recommended")) {
-        *answer = tor_strdup(recommended);
-        return 0;
-      }
-      if (!strcmp(question, "status/version/current")) {
-        switch (status)
-          {
-          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
-          case VS_OLD: *answer = tor_strdup("obsolete"); break;
-          case VS_NEW: *answer = tor_strdup("new"); break;
-          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
-          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
-          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
-          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
-          default: tor_fragile_assert();
-          }
-      } else if (!strcmp(question, "status/version/num-versioning") ||
-                 !strcmp(question, "status/version/num-concurring")) {
-        tor_asprintf(answer, "%d", get_n_authorities(V3_DIRINFO));
-        log_warn(LD_GENERAL, "%s is deprecated; it no longer gives useful "
-                 "information", question);
-      }
-    } else if (!strcmp(question, "status/clients-seen")) {
-      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
-      if (!bridge_stats) {
-        *errmsg = "No bridge-client stats available";
-        return -1;
-      }
-      *answer = bridge_stats;
-    } else if (!strcmp(question, "status/fresh-relay-descs")) {
-      if (!server_mode(options)) {
-        *errmsg = "Only relays have descriptors";
-        return -1;
-      }
-      routerinfo_t *r;
-      extrainfo_t *e;
-      if (router_build_fresh_descriptor(&r, &e) < 0) {
-        *errmsg = "Error generating descriptor";
-        return -1;
-      }
-      size_t size = r->cache_info.signed_descriptor_len + 1;
-      if (e) {
-        size += e->cache_info.signed_descriptor_len + 1;
-      }
-      tor_assert(r->cache_info.signed_descriptor_len);
-      char *descs = tor_malloc(size);
-      char *cp = descs;
-      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
-             r->cache_info.signed_descriptor_len);
-      cp += r->cache_info.signed_descriptor_len - 1;
-      if (e) {
-        if (cp[0] == '\0') {
-          cp[0] = '\n';
-        } else if (cp[0] != '\n') {
-          cp[1] = '\n';
-          cp++;
-        }
-        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
-               e->cache_info.signed_descriptor_len);
-        cp += e->cache_info.signed_descriptor_len - 1;
-      }
-      if (cp[0] == '\n') {
-        cp[0] = '\0';
-      } else if (cp[0] != '\0') {
-        cp[1] = '\0';
-      }
-      *answer = descs;
-      routerinfo_free(r);
-      extrainfo_free(e);
-    } else {
-      return 0;
-    }
-  }
-  return 0;
-}
-
-/** Implementation helper for GETINFO: knows how to enumerate hidden services
- * created via the control port. */
-STATIC int
-getinfo_helper_onions(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  smartlist_t *onion_list = NULL;
-  (void) errmsg;  /* no errors from this method */
-
-  if (control_conn && !strcmp(question, "onions/current")) {
-    onion_list = control_conn->ephemeral_onion_services;
-  } else if (!strcmp(question, "onions/detached")) {
-    onion_list = detached_onion_services;
-  } else {
-    return 0;
-  }
-  if (!onion_list || smartlist_len(onion_list) == 0) {
-    if (answer) {
-      *answer = tor_strdup("");
-    }
-  } else {
-    if (answer) {
-      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about network
- * liveness. */
-static int
-getinfo_helper_liveness(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **errmsg)
-{
-  (void)control_conn;
-  (void)errmsg;
-  if (strcmp(question, "network-liveness") == 0) {
-    if (get_cached_network_liveness()) {
-      *answer = tor_strdup("up");
-    } else {
-      *answer = tor_strdup("down");
-    }
-  }
-
-  return 0;
-}
-
-/** Implementation helper for GETINFO: answers queries about shared random
- * value. */
-static int
-getinfo_helper_sr(control_connection_t *control_conn,
-                  const char *question, char **answer,
-                  const char **errmsg)
-{
-  (void) control_conn;
-  (void) errmsg;
-
-  if (!strcmp(question, "sr/current")) {
-    *answer = sr_get_current_for_control();
-  } else if (!strcmp(question, "sr/previous")) {
-    *answer = sr_get_previous_for_control();
-  }
-  /* Else statement here is unrecognized key so do nothing. */
-
-  return 0;
-}
-
-/** Callback function for GETINFO: on a given control connection, try to
- * answer the question <b>q</b> and store the newly-allocated answer in
- * *<b>a</b>. If an internal error occurs, return -1 and optionally set
- * *<b>error_out</b> to point to an error message to be delivered to the
- * controller. On success, _or if the key is not recognized_, return 0. Do not
- * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
- * to improve the error message.
- */
-typedef int (*getinfo_helper_t)(control_connection_t *,
-                                const char *q, char **a,
-                                const char **error_out);
-
-/** A single item for the GETINFO question-to-answer-function table. */
-typedef struct getinfo_item_t {
-  const char *varname; /**< The value (or prefix) of the question. */
-  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
-                        * this entry is documentation-only. */
-  const char *desc; /**< Description of the variable. */
-  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
-} getinfo_item_t;
-
-#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
-#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
-#define DOC(name, desc) { name, NULL, desc, 0 }
-
-/** Table mapping questions accepted by GETINFO to the functions that know how
- * to answer them. */
-static const getinfo_item_t getinfo_items[] = {
-  ITEM("version", misc, "The current version of Tor."),
-  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
-  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
-  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
-  ITEM("config-text", misc,
-       "Return the string that would be written by a saveconf command."),
-  ITEM("config-can-saveconf", misc,
-       "Is it possible to save the configuration to the \"torrc\" file?"),
-  ITEM("accounting/bytes", accounting,
-       "Number of bytes read/written so far in the accounting interval."),
-  ITEM("accounting/bytes-left", accounting,
-      "Number of bytes left to write/read so far in the accounting interval."),
-  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
-  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
-  ITEM("accounting/interval-start", accounting,
-       "Time when the accounting period starts."),
-  ITEM("accounting/interval-end", accounting,
-       "Time when the accounting period ends."),
-  ITEM("accounting/interval-wake", accounting,
-       "Time to wake up in this accounting period."),
-  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
-  ITEM("entry-guards", entry_guards,
-       "Which nodes are we using as entry guards?"),
-  ITEM("fingerprint", misc, NULL),
-  PREFIX("config/", config, "Current configuration values."),
-  DOC("config/names",
-      "List of configuration options, types, and documentation."),
-  DOC("config/defaults",
-      "List of default values for configuration options. "
-      "See also config/names"),
-  PREFIX("current-time/", current_time, "Current time."),
-  DOC("current-time/local", "Current time on the local system."),
-  DOC("current-time/utc", "Current UTC time."),
-  PREFIX("downloads/networkstatus/", downloads,
-         "Download statuses for networkstatus objects"),
-  DOC("downloads/networkstatus/ns",
-      "Download status for current-mode networkstatus download"),
-  DOC("downloads/networkstatus/ns/bootstrap",
-      "Download status for bootstrap-time networkstatus download"),
-  DOC("downloads/networkstatus/ns/running",
-      "Download status for run-time networkstatus download"),
-  DOC("downloads/networkstatus/microdesc",
-      "Download status for current-mode microdesc download"),
-  DOC("downloads/networkstatus/microdesc/bootstrap",
-      "Download status for bootstrap-time microdesc download"),
-  DOC("downloads/networkstatus/microdesc/running",
-      "Download status for run-time microdesc download"),
-  PREFIX("downloads/cert/", downloads,
-         "Download statuses for certificates, by id fingerprint and "
-         "signing key"),
-  DOC("downloads/cert/fps",
-      "List of authority fingerprints for which any download statuses "
-      "exist"),
-  DOC("downloads/cert/fp/<fp>",
-      "Download status for <fp> with the default signing key; corresponds "
-      "to /fp/ URLs on directory server."),
-  DOC("downloads/cert/fp/<fp>/sks",
-      "List of signing keys for which specific download statuses are "
-      "available for this id fingerprint"),
-  DOC("downloads/cert/fp/<fp>/<sk>",
-      "Download status for <fp> with signing key <sk>; corresponds "
-      "to /fp-sk/ URLs on directory server."),
-  PREFIX("downloads/desc/", downloads,
-         "Download statuses for router descriptors, by descriptor digest"),
-  DOC("downloads/desc/descs",
-      "Return a list of known router descriptor digests"),
-  DOC("downloads/desc/<desc>",
-      "Return a download status for a given descriptor digest"),
-  PREFIX("downloads/bridge/", downloads,
-         "Download statuses for bridge descriptors, by bridge identity "
-         "digest"),
-  DOC("downloads/bridge/bridges",
-      "Return a list of configured bridge identity digests with download "
-      "statuses"),
-  DOC("downloads/bridge/<desc>",
-      "Return a download status for a given bridge identity digest"),
-  ITEM("info/names", misc,
-       "List of GETINFO options, types, and documentation."),
-  ITEM("events/names", misc,
-       "Events that the controller can ask for with SETEVENTS."),
-  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
-  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
-  PREFIX("desc/id/", dir, "Router descriptors by ID."),
-  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
-  ITEM("desc/all-recent", dir,
-       "All non-expired, non-superseded router descriptors."),
-  ITEM("desc/download-enabled", dir,
-       "Do we try to download router descriptors?"),
-  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
-  ITEM("md/all", dir, "All known microdescriptors."),
-  PREFIX("md/id/", dir, "Microdescriptors by ID"),
-  PREFIX("md/name/", dir, "Microdescriptors by name"),
-  ITEM("md/download-enabled", dir,
-       "Do we try to download microdescriptors?"),
-  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
-  PREFIX("hs/client/desc/id", dir,
-         "Hidden Service descriptor in client's cache by onion."),
-  PREFIX("hs/service/desc/id/", dir,
-         "Hidden Service descriptor in services's cache by onion."),
-  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
-  ITEM("ns/all", networkstatus,
-       "Brief summary of router status (v2 directory format)"),
-  PREFIX("ns/id/", networkstatus,
-         "Brief summary of router status by ID (v2 directory format)."),
-  PREFIX("ns/name/", networkstatus,
-         "Brief summary of router status by nickname (v2 directory format)."),
-  PREFIX("ns/purpose/", networkstatus,
-         "Brief summary of router status by purpose (v2 directory format)."),
-  PREFIX("consensus/", networkstatus,
-         "Information about and from the ns consensus."),
-  ITEM("network-status", dir,
-       "Brief summary of router status (v1 directory format)"),
-  ITEM("network-liveness", liveness,
-       "Current opinion on whether the network is live"),
-  ITEM("circuit-status", events, "List of current circuits originating here."),
-  ITEM("stream-status", events,"List of current streams."),
-  ITEM("orconn-status", events, "A list of current OR connections."),
-  ITEM("dormant", misc,
-       "Is Tor dormant (not building circuits because it's idle)?"),
-  PREFIX("address-mappings/", events, NULL),
-  DOC("address-mappings/all", "Current address mappings."),
-  DOC("address-mappings/cache", "Current cached DNS replies."),
-  DOC("address-mappings/config",
-      "Current address mappings from configuration."),
-  DOC("address-mappings/control", "Current address mappings from controller."),
-  PREFIX("status/", events, NULL),
-  DOC("status/circuit-established",
-      "Whether we think client functionality is working."),
-  DOC("status/enough-dir-info",
-      "Whether we have enough up-to-date directory information to build "
-      "circuits."),
-  DOC("status/bootstrap-phase",
-      "The last bootstrap phase status event that Tor sent."),
-  DOC("status/clients-seen",
-      "Breakdown of client countries seen by a bridge."),
-  DOC("status/fresh-relay-descs",
-      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
-  DOC("status/version/recommended", "List of currently recommended versions."),
-  DOC("status/version/current", "Status of the current version."),
-  DOC("status/version/num-versioning", "Number of versioning authorities."),
-  DOC("status/version/num-concurring",
-      "Number of versioning authorities agreeing on the status of the "
-      "current version"),
-  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
-  ITEM("traffic/read", misc,"Bytes read since the process was started."),
-  ITEM("traffic/written", misc,
-       "Bytes written since the process was started."),
-  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
-  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
-  ITEM("process/uid", misc, "User id running the tor process."),
-  ITEM("process/user", misc,
-       "Username under which the tor process is running."),
-  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
-  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
-  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
-  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
-  PREFIX("dir/status/", dir,
-         "v2 networkstatus docs as retrieved from a DirPort."),
-  ITEM("dir/status-vote/current/consensus", dir,
-       "v3 Networkstatus consensus as retrieved from a DirPort."),
-  ITEM("exit-policy/default", policies,
-       "The default value appended to the configured exit policy."),
-  ITEM("exit-policy/reject-private/default", policies,
-       "The default rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate."),
-  ITEM("exit-policy/reject-private/relay", policies,
-       "The relay-specific rules appended to the configured exit policy by"
-       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
-  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
-  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
-  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
-  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
-  ITEM("onions/current", onions,
-       "Onion services owned by the current control connection."),
-  ITEM("onions/detached", onions,
-       "Onion services detached from the control connection."),
-  ITEM("sr/current", sr, "Get current shared random value."),
-  ITEM("sr/previous", sr, "Get previous shared random value."),
-  { NULL, NULL, NULL, 0 }
-};
-
-/** Allocate and return a list of recognized GETINFO options. */
-static char *
-list_getinfo_options(void)
-{
-  int i;
-  smartlist_t *lines = smartlist_new();
-  char *ans;
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    if (!getinfo_items[i].desc)
-      continue;
-
-    smartlist_add_asprintf(lines, "%s%s -- %s\n",
-                 getinfo_items[i].varname,
-                 getinfo_items[i].is_prefix ? "*" : "",
-                 getinfo_items[i].desc);
-  }
-  smartlist_sort_strings(lines);
-
-  ans = smartlist_join_strings(lines, "", 0, NULL);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-
-  return ans;
-}
-
-/** Lookup the 'getinfo' entry <b>question</b>, and return
- * the answer in <b>*answer</b> (or NULL if key not recognized).
- * Return 0 if success or unrecognized, or -1 if recognized but
- * internal error. */
-static int
-handle_getinfo_helper(control_connection_t *control_conn,
-                      const char *question, char **answer,
-                      const char **err_out)
-{
-  int i;
-  *answer = NULL; /* unrecognized key by default */
-
-  for (i = 0; getinfo_items[i].varname; ++i) {
-    int match;
-    if (getinfo_items[i].is_prefix)
-      match = !strcmpstart(question, getinfo_items[i].varname);
-    else
-      match = !strcmp(question, getinfo_items[i].varname);
-    if (match) {
-      tor_assert(getinfo_items[i].fn);
-      return getinfo_items[i].fn(control_conn, question, answer, err_out);
-    }
-  }
-
-  return 0; /* unrecognized */
-}
-
-/** Called when we receive a GETINFO command.  Try to fetch all requested
- * information, and reply with information or error message. */
-static int
-handle_control_getinfo(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *questions = smartlist_new();
-  smartlist_t *answers = smartlist_new();
-  smartlist_t *unrecognized = smartlist_new();
-  char *ans = NULL;
-  int i;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  smartlist_split_string(questions, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
-    const char *errmsg = NULL;
-
-    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
-      if (!errmsg)
-        errmsg = "Internal error";
-      connection_printf_to_buf(conn, "551 %s\r\n", errmsg);
-      goto done;
-    }
-    if (!ans) {
-      if (errmsg) /* use provided error message */
-        smartlist_add_strdup(unrecognized, errmsg);
-      else /* use default error message */
-        smartlist_add_asprintf(unrecognized, "Unrecognized key \"%s\"", q);
-    } else {
-      smartlist_add_strdup(answers, q);
-      smartlist_add(answers, ans);
-    }
-  } SMARTLIST_FOREACH_END(q);
-
-  if (smartlist_len(unrecognized)) {
-    /* control-spec section 2.3, mid-reply '-' or end of reply ' ' */
-    for (i=0; i < smartlist_len(unrecognized)-1; ++i)
-      connection_printf_to_buf(conn,
-                               "552-%s\r\n",
-                               (char *)smartlist_get(unrecognized, i));
-
-    connection_printf_to_buf(conn,
-                             "552 %s\r\n",
-                             (char *)smartlist_get(unrecognized, i));
-    goto done;
-  }
-
-  for (i = 0; i < smartlist_len(answers); i += 2) {
-    char *k = smartlist_get(answers, i);
-    char *v = smartlist_get(answers, i+1);
-    if (!strchr(v, '\n') && !strchr(v, '\r')) {
-      connection_printf_to_buf(conn, "250-%s=", k);
-      connection_write_str_to_buf(v, conn);
-      connection_write_str_to_buf("\r\n", conn);
-    } else {
-      char *esc = NULL;
-      size_t esc_len;
-      esc_len = write_escaped_data(v, strlen(v), &esc);
-      connection_printf_to_buf(conn, "250+%s=\r\n", k);
-      connection_buf_add(esc, esc_len, TO_CONN(conn));
-      tor_free(esc);
-    }
-  }
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  SMARTLIST_FOREACH(answers, char *, cp, tor_free(cp));
-  smartlist_free(answers);
-  SMARTLIST_FOREACH(questions, char *, cp, tor_free(cp));
-  smartlist_free(questions);
-  SMARTLIST_FOREACH(unrecognized, char *, cp, tor_free(cp));
-  smartlist_free(unrecognized);
-
-  return 0;
-}
-
-/** Given a string, convert it to a circuit purpose. */
-static uint8_t
-circuit_purpose_from_string(const char *string)
-{
-  if (!strcasecmpstart(string, "purpose="))
-    string += strlen("purpose=");
-
-  if (!strcasecmp(string, "general"))
-    return CIRCUIT_PURPOSE_C_GENERAL;
-  else if (!strcasecmp(string, "controller"))
-    return CIRCUIT_PURPOSE_CONTROLLER;
-  else
-    return CIRCUIT_PURPOSE_UNKNOWN;
-}
-
-/** Return a newly allocated smartlist containing the arguments to the command
- * waiting in <b>body</b>. If there are fewer than <b>min_args</b> arguments,
- * or if <b>max_args</b> is nonnegative and there are more than
- * <b>max_args</b> arguments, send a 512 error to the controller, using
- * <b>command</b> as the command name in the error message. */
-static smartlist_t *
-getargs_helper(const char *command, control_connection_t *conn,
-               const char *body, int min_args, int max_args)
-{
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(args) < min_args) {
-    connection_printf_to_buf(conn, "512 Missing argument to %s\r\n",command);
-    goto err;
-  } else if (max_args >= 0 && smartlist_len(args) > max_args) {
-    connection_printf_to_buf(conn, "512 Too many arguments to %s\r\n",command);
-    goto err;
-  }
-  return args;
- err:
-  SMARTLIST_FOREACH(args, char *, s, tor_free(s));
-  smartlist_free(args);
-  return NULL;
-}
-
-/** Helper.  Return the first element of <b>sl</b> at index <b>start_at</b> or
- * higher that starts with <b>prefix</b>, case-insensitive.  Return NULL if no
- * such element exists. */
-static const char *
-find_element_starting_with(smartlist_t *sl, int start_at, const char *prefix)
-{
-  int i;
-  for (i = start_at; i < smartlist_len(sl); ++i) {
-    const char *elt = smartlist_get(sl, i);
-    if (!strcasecmpstart(elt, prefix))
-      return elt;
-  }
-  return NULL;
-}
-
-/** Helper.  Return true iff s is an argument that we should treat as a
- * key-value pair. */
-static int
-is_keyval_pair(const char *s)
-{
-  /* An argument is a key-value pair if it has an =, and it isn't of the form
-   * $fingeprint=name */
-  return strchr(s, '=') && s[0] != '$';
-}
-
-/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
- * circuit, and report success or failure. */
-static int
-handle_control_extendcircuit(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  smartlist_t *router_nicknames=NULL, *nodes=NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
-  smartlist_t *args;
-  (void) len;
-
-  router_nicknames = smartlist_new();
-
-  args = getargs_helper("EXTENDCIRCUIT", conn, body, 1, -1);
-  if (!args)
-    goto done;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,0));
-
-  if (zero_circ) {
-    const char *purp = find_element_starting_with(args, 1, "PURPOSE=");
-
-    if (purp) {
-      intended_purpose = circuit_purpose_from_string(purp);
-      if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-        SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-        smartlist_free(args);
-        goto done;
-      }
-    }
-
-    if ((smartlist_len(args) == 1) ||
-        (smartlist_len(args) >= 2 && is_keyval_pair(smartlist_get(args, 1)))) {
-      // "EXTENDCIRCUIT 0" || EXTENDCIRCUIT 0 foo=bar"
-      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
-      if (!circ) {
-        connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      } else {
-        connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                  (unsigned long)circ->global_identifier);
-      }
-      SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-      smartlist_free(args);
-      goto done;
-    }
-    // "EXTENDCIRCUIT 0 router1,router2" ||
-    // "EXTENDCIRCUIT 0 router1,router2 PURPOSE=foo"
-  }
-
-  if (!zero_circ && !(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  if (smartlist_len(args) < 2) {
-    connection_printf_to_buf(conn,
-                             "512 syntax error: not enough arguments.\r\n");
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-    goto done;
-  }
-
-  smartlist_split_string(router_nicknames, smartlist_get(args,1), ",", 0, 0);
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-
-  nodes = smartlist_new();
-  int first_node = zero_circ;
-  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
-    const node_t *node = node_get_by_nickname(n, 0);
-    if (!node) {
-      connection_printf_to_buf(conn, "552 No such router \"%s\"\r\n", n);
-      goto done;
-    }
-    if (!node_has_preferred_descriptor(node, first_node)) {
-      connection_printf_to_buf(conn, "552 No descriptor for \"%s\"\r\n", n);
-      goto done;
-    }
-    smartlist_add(nodes, (void*)node);
-    first_node = 0;
-  } SMARTLIST_FOREACH_END(n);
-  if (!smartlist_len(nodes)) {
-    connection_write_str_to_buf("512 No router names provided\r\n", conn);
-    goto done;
-  }
-
-  if (zero_circ) {
-    /* start a new circuit */
-    circ = origin_circuit_init(intended_purpose, 0);
-  }
-
-  /* now circ refers to something that is ready to be extended */
-  first_node = zero_circ;
-  SMARTLIST_FOREACH(nodes, const node_t *, node,
-  {
-    extend_info_t *info = extend_info_from_node(node, first_node);
-    if (!info) {
-      tor_assert_nonfatal(first_node);
-      log_warn(LD_CONTROL,
-               "controller tried to connect to a node that lacks a suitable "
-               "descriptor, or which doesn't have any "
-               "addresses that are allowed by the firewall configuration; "
-               "circuit marked for closing.");
-      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-    circuit_append_new_exit(circ, info);
-    if (circ->build_state->desired_path_len > 1) {
-      circ->build_state->onehop_tunnel = 0;
-    }
-    extend_info_free(info);
-    first_node = 0;
-  });
-
-  /* now that we've populated the cpath, start extending */
-  if (zero_circ) {
-    int err_reason = 0;
-    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
-      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-      connection_write_str_to_buf("551 Couldn't start circuit\r\n", conn);
-      goto done;
-    }
-  } else {
-    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
-        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
-      int err_reason = 0;
-      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
-      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
-        log_info(LD_CONTROL,
-                 "send_next_onion_skin failed; circuit marked for closing.");
-        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
-        connection_write_str_to_buf("551 Couldn't send onion skin\r\n", conn);
-        goto done;
-      }
-    }
-  }
-
-  connection_printf_to_buf(conn, "250 EXTENDED %lu\r\n",
-                             (unsigned long)circ->global_identifier);
-  if (zero_circ) /* send a 'launched' event, for completeness */
-    control_event_circuit_status(circ, CIRC_EVENT_LAUNCHED, 0);
- done:
-  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
-  smartlist_free(router_nicknames);
-  smartlist_free(nodes);
-  return 0;
-}
-
-/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
- * circuit and it's a valid purpose, change it. */
-static int
-handle_control_setcircuitpurpose(control_connection_t *conn,
-                                 uint32_t len, const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  uint8_t new_purpose;
-  smartlist_t *args;
-  (void) len; /* body is NUL-terminated, so it's safe to ignore the length. */
-
-  args = getargs_helper("SETCIRCUITPURPOSE", conn, body, 2, -1);
-  if (!args)
-    goto done;
-
-  if (!(circ = get_circ(smartlist_get(args,0)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-    goto done;
-  }
-
-  {
-    const char *purp = find_element_starting_with(args,1,"PURPOSE=");
-    if (!purp) {
-      connection_write_str_to_buf("552 No purpose given\r\n", conn);
-      goto done;
-    }
-    new_purpose = circuit_purpose_from_string(purp);
-    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
-      connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n", purp);
-      goto done;
-    }
-  }
-
-  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
-  connection_write_str_to_buf("250 OK\r\n", conn);
-
- done:
-  if (args) {
-    SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-    smartlist_free(args);
-  }
-  return 0;
-}
-
-/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
- * stream, and report success or failure. */
-static int
-handle_control_attachstream(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  origin_circuit_t *circ = NULL;
-  int zero_circ;
-  smartlist_t *args;
-  crypt_path_t *cpath=NULL;
-  int hop=0, hop_line_ok=1;
-  (void) len;
-
-  args = getargs_helper("ATTACHSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  zero_circ = !strcmp("0", (char*)smartlist_get(args,1));
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else if (!zero_circ && !(circ = get_circ(smartlist_get(args, 1)))) {
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 1));
-  } else if (circ) {
-    const char *hopstring = find_element_starting_with(args,2,"HOP=");
-    if (hopstring) {
-      hopstring += strlen("HOP=");
-      hop = (int) tor_parse_ulong(hopstring, 10, 0, INT_MAX,
-                                  &hop_line_ok, NULL);
-      if (!hop_line_ok) { /* broken hop line */
-        connection_printf_to_buf(conn, "552 Bad value hop=%s\r\n", hopstring);
-      }
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn || (!zero_circ && !circ) || !hop_line_ok)
-    return 0;
-
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
-      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
-    connection_write_str_to_buf(
-                    "555 Connection is not managed by controller.\r\n",
-                    conn);
-    return 0;
-  }
-
-  /* Do we need to detach it first? */
-  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
-    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
-    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
-    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
-    /* Un-mark it as ending, since we're going to reuse it. */
-    edge_conn->edge_has_sent_end = 0;
-    edge_conn->end_reason = 0;
-    if (tmpcirc)
-      circuit_detach_stream(tmpcirc, edge_conn);
-    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
-    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
-  }
-
-  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
-    connection_write_str_to_buf(
-                    "551 Can't attach stream to non-open origin circuit\r\n",
-                    conn);
-    return 0;
-  }
-  /* Is this a single hop circuit? */
-  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
-    connection_write_str_to_buf(
-               "551 Can't attach stream to this one-hop circuit.\r\n", conn);
-    return 0;
-  }
-
-  if (circ && hop>0) {
-    /* find this hop in the circuit, and set cpath */
-    cpath = circuit_get_cpath_hop(circ, hop);
-    if (!cpath) {
-      connection_printf_to_buf(conn,
-                               "551 Circuit doesn't have %d hops.\r\n", hop);
-      return 0;
-    }
-  }
-  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
-    connection_write_str_to_buf("551 Unable to attach stream\r\n", conn);
-    return 0;
-  }
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
- * descriptor, and report success or failure. */
-static int
-handle_control_postdescriptor(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  char *desc;
-  const char *msg=NULL;
-  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
-  int cache = 0; /* eventually, we may switch this to 1 */
-
-  const char *cp = memchr(body, '\n', len);
-
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  ++cp;
-
-  char *cmdline = tor_memdup_nulterm(body, cp-body);
-  smartlist_t *args = smartlist_new();
-  smartlist_split_string(args, cmdline, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, char *, option) {
-    if (!strcasecmpstart(option, "purpose=")) {
-      option += strlen("purpose=");
-      purpose = router_purpose_from_string(option);
-      if (purpose == ROUTER_PURPOSE_UNKNOWN) {
-        connection_printf_to_buf(conn, "552 Unknown purpose \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else if (!strcasecmpstart(option, "cache=")) {
-      option += strlen("cache=");
-      if (!strcasecmp(option, "no"))
-        cache = 0;
-      else if (!strcasecmp(option, "yes"))
-        cache = 1;
-      else {
-        connection_printf_to_buf(conn, "552 Unknown cache request \"%s\"\r\n",
-                                 option);
-        goto done;
-      }
-    } else { /* unrecognized argument? */
-      connection_printf_to_buf(conn,
-        "512 Unexpected argument \"%s\" to postdescriptor\r\n", option);
-      goto done;
-    }
-  } SMARTLIST_FOREACH_END(option);
-
-  read_escaped_data(cp, len-(cp-body), &desc);
-
-  switch (router_load_single_router(desc, purpose, cache, &msg)) {
-  case -1:
-    if (!msg) msg = "Could not parse descriptor";
-    connection_printf_to_buf(conn, "554 %s\r\n", msg);
-    break;
-  case 0:
-    if (!msg) msg = "Descriptor not added";
-    connection_printf_to_buf(conn, "251 %s\r\n",msg);
-    break;
-  case 1:
-    send_control_done(conn);
-    break;
-  }
-
-  tor_free(desc);
- done:
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  tor_free(cmdline);
-  return 0;
-}
-
-/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
- * address of the named AP stream, and report success or failure. */
-static int
-handle_control_redirectstream(control_connection_t *conn, uint32_t len,
-                              const char *body)
-{
-  entry_connection_t *ap_conn = NULL;
-  char *new_addr = NULL;
-  uint16_t new_port = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("REDIRECTSTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
-           || !ap_conn->socks_request) {
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  } else {
-    int ok = 1;
-    if (smartlist_len(args) > 2) { /* they included a port too */
-      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
-                                            10, 1, 65535, &ok, NULL);
-    }
-    if (!ok) {
-      connection_printf_to_buf(conn, "512 Cannot parse port \"%s\"\r\n",
-                               (char*)smartlist_get(args, 2));
-    } else {
-      new_addr = tor_strdup(smartlist_get(args, 1));
-    }
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!new_addr)
-    return 0;
-
-  strlcpy(ap_conn->socks_request->address, new_addr,
-          sizeof(ap_conn->socks_request->address));
-  if (new_port)
-    ap_conn->socks_request->port = new_port;
-  tor_free(new_addr);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSESTREAM command; try to close the named stream
- * and report success or failure. */
-static int
-handle_control_closestream(control_connection_t *conn, uint32_t len,
-                           const char *body)
-{
-  entry_connection_t *ap_conn=NULL;
-  uint8_t reason=0;
-  smartlist_t *args;
-  int ok;
-  (void) len;
-
-  args = getargs_helper("CLOSESTREAM", conn, body, 2, -1);
-  if (!args)
-    return 0;
-
-  else if (!(ap_conn = get_stream(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown stream \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
-                                       &ok, NULL);
-    if (!ok) {
-      connection_printf_to_buf(conn, "552 Unrecognized reason \"%s\"\r\n",
-                               (char*)smartlist_get(args, 1));
-      ap_conn = NULL;
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!ap_conn)
-    return 0;
-
-  connection_mark_unattached_ap(ap_conn, reason);
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
- * and report success or failure. */
-static int
-handle_control_closecircuit(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  origin_circuit_t *circ = NULL;
-  int safe = 0;
-  smartlist_t *args;
-  (void) len;
-
-  args = getargs_helper("CLOSECIRCUIT", conn, body, 1, -1);
-  if (!args)
-    return 0;
-
-  if (!(circ=get_circ(smartlist_get(args, 0))))
-    connection_printf_to_buf(conn, "552 Unknown circuit \"%s\"\r\n",
-                             (char*)smartlist_get(args, 0));
-  else {
-    int i;
-    for (i=1; i < smartlist_len(args); ++i) {
-      if (!strcasecmp(smartlist_get(args, i), "IfUnused"))
-        safe = 1;
-      else
-        log_info(LD_CONTROL, "Skipping unknown option %s",
-                 (char*)smartlist_get(args,i));
-    }
-  }
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  if (!circ)
-    return 0;
-
-  if (!safe || !circ->p_streams) {
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
-  }
-
-  send_control_done(conn);
-  return 0;
-}
-
-/** Called when we get a RESOLVE command: start trying to resolve
- * the listed addresses. */
-static int
-handle_control_resolve(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  smartlist_t *args, *failed;
-  int is_reverse = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-
-  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
-    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
-             "isn't listening for ADDRMAP events.  It probably won't see "
-             "the answer.");
-  }
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  {
-    const char *modearg = find_element_starting_with(args, 0, "mode=");
-    if (modearg && !strcasecmp(modearg, "mode=reverse"))
-      is_reverse = 1;
-  }
-  failed = smartlist_new();
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      if (!is_keyval_pair(arg)) {
-          if (dnsserv_launch_request(arg, is_reverse, conn)<0)
-            smartlist_add(failed, (char*)arg);
-      }
-  });
-
-  send_control_done(conn);
-  SMARTLIST_FOREACH(failed, const char *, arg, {
-      control_event_address_mapped(arg, arg, time(NULL),
-                                   "internal", 0);
-  });
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  smartlist_free(failed);
-  return 0;
-}
-
-/** Called when we get a PROTOCOLINFO command: send back a reply. */
-static int
-handle_control_protocolinfo(control_connection_t *conn, uint32_t len,
-                            const char *body)
-{
-  const char *bad_arg = NULL;
-  smartlist_t *args;
-  (void)len;
-
-  conn->have_sent_protocolinfo = 1;
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH(args, const char *, arg, {
-      int ok;
-      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
-      if (!ok) {
-        bad_arg = arg;
-        break;
-      }
-    });
-  if (bad_arg) {
-    connection_printf_to_buf(conn, "513 No such version %s\r\n",
-                             escaped(bad_arg));
-    /* Don't tolerate bad arguments when not authenticated. */
-    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
-      connection_mark_for_close(TO_CONN(conn));
-    goto done;
-  } else {
-    const or_options_t *options = get_options();
-    int cookies = options->CookieAuthentication;
-    char *cfile = get_controller_cookie_file_name();
-    char *abs_cfile;
-    char *esc_cfile;
-    char *methods;
-    abs_cfile = make_path_absolute(cfile);
-    esc_cfile = esc_for_log(abs_cfile);
-    {
-      int passwd = (options->HashedControlPassword != NULL ||
-                    options->HashedControlSessionPassword != NULL);
-      smartlist_t *mlist = smartlist_new();
-      if (cookies) {
-        smartlist_add(mlist, (char*)"COOKIE");
-        smartlist_add(mlist, (char*)"SAFECOOKIE");
-      }
-      if (passwd)
-        smartlist_add(mlist, (char*)"HASHEDPASSWORD");
-      if (!cookies && !passwd)
-        smartlist_add(mlist, (char*)"NULL");
-      methods = smartlist_join_strings(mlist, ",", 0, NULL);
-      smartlist_free(mlist);
-    }
-
-    connection_printf_to_buf(conn,
-                             "250-PROTOCOLINFO 1\r\n"
-                             "250-AUTH METHODS=%s%s%s\r\n"
-                             "250-VERSION Tor=%s\r\n"
-                             "250 OK\r\n",
-                             methods,
-                             cookies?" COOKIEFILE=":"",
-                             cookies?esc_cfile:"",
-                             escaped(VERSION));
-    tor_free(methods);
-    tor_free(cfile);
-    tor_free(abs_cfile);
-    tor_free(esc_cfile);
-  }
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Called when we get an AUTHCHALLENGE command. */
-static int
-handle_control_authchallenge(control_connection_t *conn, uint32_t len,
-                             const char *body)
-{
-  const char *cp = body;
-  char *client_nonce;
-  size_t client_nonce_len;
-  char server_hash[DIGEST256_LEN];
-  char server_hash_encoded[HEX_DIGEST256_LEN+1];
-  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
-  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
-
-  cp += strspn(cp, " \t\n\r");
-  if (!strcasecmpstart(cp, "SAFECOOKIE")) {
-    cp += strlen("SAFECOOKIE");
-  } else {
-    connection_write_str_to_buf("513 AUTHCHALLENGE only supports SAFECOOKIE "
-                                "authentication\r\n", conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  if (!authentication_cookie_is_set) {
-    connection_write_str_to_buf("515 Cookie authentication is disabled\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    return -1;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp == '"') {
-    const char *newcp =
-      decode_escaped_string(cp, len - (cp - body),
-                            &client_nonce, &client_nonce_len);
-    if (newcp == NULL) {
-      connection_write_str_to_buf("513 Invalid quoted client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      return -1;
-    }
-    cp = newcp;
-  } else {
-    size_t client_nonce_encoded_len = strspn(cp, "0123456789ABCDEFabcdef");
-
-    client_nonce_len = client_nonce_encoded_len / 2;
-    client_nonce = tor_malloc_zero(client_nonce_len);
-
-    if (base16_decode(client_nonce, client_nonce_len,
-                      cp, client_nonce_encoded_len)
-                      != (int) client_nonce_len) {
-      connection_write_str_to_buf("513 Invalid base16 client nonce\r\n",
-                                  conn);
-      connection_mark_for_close(TO_CONN(conn));
-      tor_free(client_nonce);
-      return -1;
-    }
-
-    cp += client_nonce_encoded_len;
-  }
-
-  cp += strspn(cp, " \t\n\r");
-  if (*cp != '\0' ||
-      cp != body + len) {
-    connection_write_str_to_buf("513 Junk at end of AUTHCHALLENGE command\r\n",
-                                conn);
-    connection_mark_for_close(TO_CONN(conn));
-    tor_free(client_nonce);
-    return -1;
-  }
-  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-  /* Now compute and send the server-to-controller response, and the
-   * server's nonce. */
-  tor_assert(authentication_cookie != NULL);
-
-  {
-    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
-                      client_nonce_len +
-                      SAFECOOKIE_SERVER_NONCE_LEN);
-    char *tmp = tor_malloc_zero(tmp_len);
-    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
-    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
-    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
-           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
-
-    crypto_hmac_sha256(server_hash,
-                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
-                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    crypto_hmac_sha256(client_hash,
-                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
-                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
-                       tmp,
-                       tmp_len);
-
-    conn->safecookie_client_hash = client_hash;
-
-    tor_free(tmp);
-  }
-
-  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
-                server_hash, sizeof(server_hash));
-  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
-                server_nonce, sizeof(server_nonce));
-
-  connection_printf_to_buf(conn,
-                           "250 AUTHCHALLENGE SERVERHASH=%s "
-                           "SERVERNONCE=%s\r\n",
-                           server_hash_encoded,
-                           server_nonce_encoded);
-
-  tor_free(client_nonce);
-  return 0;
-}
-
-/** Called when we get a USEFEATURE command: parse the feature list, and
- * set up the control_connection's options properly. */
-static int
-handle_control_usefeature(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  int bad = 0;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmp(arg, "VERBOSE_NAMES"))
-        ;
-      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
-        ;
-      else {
-        connection_printf_to_buf(conn, "552 Unrecognized feature \"%s\"\r\n",
-                                 arg);
-        bad = 1;
-        break;
-      }
-  } SMARTLIST_FOREACH_END(arg);
-
-  if (!bad) {
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the DROPGUARDS command. */
-static int
-handle_control_dropguards(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = smartlist_new();
-  smartlist_split_string(args, body, " ",
-                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-
-  static int have_warned = 0;
-  if (! have_warned) {
-    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
-             "the risks before using it. It may be removed in a future "
-             "version of Tor.");
-    have_warned = 1;
-  }
-
-  if (smartlist_len(args)) {
-    connection_printf_to_buf(conn, "512 Too many arguments to DROPGUARDS\r\n");
-  } else {
-    remove_all_entry_guards();
-    send_control_done(conn);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  return 0;
-}
-
-/** Implementation for the HSFETCH command. */
-static int
-handle_control_hsfetch(control_connection_t *conn, uint32_t len,
-                       const char *body)
-{
-  int i;
-  char digest[DIGEST_LEN], *hsaddress = NULL, *arg1 = NULL, *desc_id = NULL;
-  smartlist_t *args = NULL, *hsdirs = NULL;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  static const char *hsfetch_command = "HSFETCH";
-  static const char *v2_str = "v2-";
-  const size_t v2_str_len = strlen(v2_str);
-  rend_data_t *rend_query = NULL;
-
-  /* Make sure we have at least one argument, the HSAddress. */
-  args = getargs_helper(hsfetch_command, conn, body, 1, -1);
-  if (!args) {
-    goto exit;
-  }
-
-  /* Extract the first argument (either HSAddress or DescID). */
-  arg1 = smartlist_get(args, 0);
-  /* Test if it's an HS address without the .onion part. */
-  if (rend_valid_v2_service_id(arg1)) {
-    hsaddress = arg1;
-  } else if (strcmpstart(arg1, v2_str) == 0 &&
-             rend_valid_descriptor_id(arg1 + v2_str_len) &&
-             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
-                           REND_DESC_ID_V2_LEN_BASE32) == 0) {
-    /* We have a well formed version 2 descriptor ID. Keep the decoded value
-     * of the id. */
-    desc_id = digest;
-  } else {
-    connection_printf_to_buf(conn, "513 Invalid argument \"%s\"\r\n",
-                             arg1);
-    goto done;
-  }
-
-  static const char *opt_server = "SERVER=";
-
-  /* Skip first argument because it's the HSAddress or DescID. */
-  for (i = 1; i < smartlist_len(args); ++i) {
-    const char *arg = smartlist_get(args, i);
-    const node_t *node;
-
-    if (!strcasecmpstart(arg, opt_server)) {
-      const char *server;
-
-      server = arg + strlen(opt_server);
-      node = node_get_by_hex_id(server, 0);
-      if (!node) {
-        connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                 server);
-        goto done;
-      }
-      if (!hsdirs) {
-        /* Stores routerstatus_t object for each specified server. */
-        hsdirs = smartlist_new();
-      }
-      /* Valid server, add it to our local list. */
-      smartlist_add(hsdirs, node->rs);
-    } else {
-      connection_printf_to_buf(conn, "513 Unexpected argument \"%s\"\r\n",
-                               arg);
-      goto done;
-    }
-  }
-
-  rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
-                                       REND_NO_AUTH);
-  if (rend_query == NULL) {
-    connection_printf_to_buf(conn, "551 Error creating the HS query\r\n");
-    goto done;
-  }
-
-  /* Using a descriptor ID, we force the user to provide at least one
-   * hsdir server using the SERVER= option. */
-  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
-      connection_printf_to_buf(conn, "512 %s option is required\r\n",
-                               opt_server);
-      goto done;
-  }
-
-  /* We are about to trigger HSDir fetch so send the OK now because after
-   * that 650 event(s) are possible so better to have the 250 OK before them
-   * to avoid out of order replies. */
-  send_control_done(conn);
-
-  /* Trigger the fetch using the built rend query and possibly a list of HS
-   * directory to use. This function ignores the client cache thus this will
-   * always send a fetch command. */
-  rend_client_fetch_v2_desc(rend_query, hsdirs);
-
- done:
-  SMARTLIST_FOREACH(args, char *, cp, tor_free(cp));
-  smartlist_free(args);
-  /* Contains data pointer that we don't own thus no cleanup. */
-  smartlist_free(hsdirs);
-  rend_data_free(rend_query);
- exit:
-  return 0;
-}
-
-/** Implementation for the HSPOST command. */
-static int
-handle_control_hspost(control_connection_t *conn,
-                      uint32_t len,
-                      const char *body)
-{
-  static const char *opt_server = "SERVER=";
-  static const char *opt_hsaddress = "HSADDRESS=";
-  smartlist_t *hs_dirs = NULL;
-  const char *encoded_desc = body;
-  size_t encoded_desc_len = len;
-  const char *onion_address = NULL;
-
-  char *cp = memchr(body, '\n', len);
-  if (cp == NULL) {
-    connection_printf_to_buf(conn, "251 Empty body\r\n");
-    return 0;
-  }
-  char *argline = tor_strndup(body, cp-body);
-
-  smartlist_t *args = smartlist_new();
-
-  /* If any SERVER= or HSADDRESS= options were specified, try to parse
-   * the options line. */
-  if (!strcasecmpstart(argline, opt_server) ||
-      !strcasecmpstart(argline, opt_hsaddress)) {
-    /* encoded_desc begins after a newline character */
-    cp = cp + 1;
-    encoded_desc = cp;
-    encoded_desc_len = len-(cp-body);
-
-    smartlist_split_string(args, argline, " ",
-                           SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
-    SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
-      if (!strcasecmpstart(arg, opt_server)) {
-        const char *server = arg + strlen(opt_server);
-        const node_t *node = node_get_by_hex_id(server, 0);
-
-        if (!node || !node->rs) {
-          connection_printf_to_buf(conn, "552 Server \"%s\" not found\r\n",
-                                   server);
-          goto done;
-        }
-        /* Valid server, add it to our local list. */
-        if (!hs_dirs)
-          hs_dirs = smartlist_new();
-        smartlist_add(hs_dirs, node->rs);
-      } else if (!strcasecmpstart(arg, opt_hsaddress)) {
-        const char *address = arg + strlen(opt_hsaddress);
-        if (!hs_address_is_valid(address)) {
-          connection_printf_to_buf(conn, "512 Malformed onion address\r\n");
-          goto done;
-        }
-        onion_address = address;
-      } else {
-        connection_printf_to_buf(conn, "512 Unexpected argument \"%s\"\r\n",
-                                 arg);
-        goto done;
-      }
-    } SMARTLIST_FOREACH_END(arg);
-  }
-
-  /* Handle the v3 case. */
-  if (onion_address) {
-    char *desc_str = NULL;
-    read_escaped_data(encoded_desc, encoded_desc_len, &desc_str);
-    if (hs_control_hspost_command(desc_str, onion_address, hs_dirs) < 0) {
-      connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-    } else {
-      send_control_done(conn);
-    }
-    tor_free(desc_str);
-    goto done;
-  }
-
-  /* From this point on, it is only v2. */
-
-  /* Read the dot encoded descriptor, and parse it. */
-  rend_encoded_v2_service_descriptor_t *desc =
-      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
-  read_escaped_data(encoded_desc, encoded_desc_len, &desc->desc_str);
-
-  rend_service_descriptor_t *parsed = NULL;
-  char *intro_content = NULL;
-  size_t intro_size;
-  size_t encoded_size;
-  const char *next_desc;
-  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
-                                        &intro_size, &encoded_size,
-                                        &next_desc, desc->desc_str, 1)) {
-    /* Post the descriptor. */
-    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
-    if (!rend_get_service_id(parsed->pk, serviceid)) {
-      smartlist_t *descs = smartlist_new();
-      smartlist_add(descs, desc);
-
-      /* We are about to trigger HS descriptor upload so send the OK now
-       * because after that 650 event(s) are possible so better to have the
-       * 250 OK before them to avoid out of order replies. */
-      send_control_done(conn);
-
-      /* Trigger the descriptor upload */
-      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
-      smartlist_free(descs);
-    }
-
-    rend_service_descriptor_free(parsed);
-  } else {
-    connection_printf_to_buf(conn, "554 Invalid descriptor\r\n");
-  }
-
-  tor_free(intro_content);
-  rend_encoded_v2_service_descriptor_free(desc);
- done:
-  tor_free(argline);
-  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
-  SMARTLIST_FOREACH(args, char *, arg, tor_free(arg));
-  smartlist_free(args);
-  return 0;
-}
-
-/* Helper function for ADD_ONION that adds an ephemeral service depending on
- * the given hs_version.
- *
- * The secret key in pk depends on the hs_version. The ownership of the key
- * used in pk is given to the HS subsystem so the caller must stop accessing
- * it after.
- *
- * The port_cfgs is a list of service port. Ownership transferred to service.
- * The max_streams refers to the MaxStreams= key.
- * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
- * The auth_type is the authentication type of the clients in auth_clients.
- * The ownership of that list is transferred to the service.
- *
- * On success (RSAE_OKAY), the address_out points to a newly allocated string
- * containing the onion address without the .onion part. On error, address_out
- * is untouched. */
-static hs_service_add_ephemeral_status_t
-add_onion_helper_add_service(int hs_version,
-                             add_onion_secret_key_t *pk,
-                             smartlist_t *port_cfgs, int max_streams,
-                             int max_streams_close_circuit, int auth_type,
-                             smartlist_t *auth_clients, char **address_out)
-{
-  hs_service_add_ephemeral_status_t ret;
-
-  tor_assert(pk);
-  tor_assert(port_cfgs);
-  tor_assert(address_out);
-
-  switch (hs_version) {
-  case HS_VERSION_TWO:
-    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
-                                     max_streams_close_circuit, auth_type,
-                                     auth_clients, address_out);
-    break;
-  case HS_VERSION_THREE:
-    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
-                                   max_streams_close_circuit, address_out);
-    break;
-  default:
-    tor_assert_unreached();
-  }
-
-  return ret;
-}
-
-/** Called when we get a ADD_ONION command; parse the body, and set up
- * the new ephemeral Onion Service. */
-static int
-handle_control_add_onion(control_connection_t *conn,
-                         uint32_t len,
-                         const char *body)
-{
-  smartlist_t *args;
-  int arg_len;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("ADD_ONION", conn, body, 2, -1);
-  if (!args)
-    return 0;
-  arg_len = smartlist_len(args);
-
-  /* Parse all of the arguments that do not involve handling cryptographic
-   * material first, since there's no reason to touch that at all if any of
-   * the other arguments are malformed.
-   */
-  smartlist_t *port_cfgs = smartlist_new();
-  smartlist_t *auth_clients = NULL;
-  smartlist_t *auth_created_clients = NULL;
-  int discard_pk = 0;
-  int detach = 0;
-  int max_streams = 0;
-  int max_streams_close_circuit = 0;
-  rend_auth_type_t auth_type = REND_NO_AUTH;
-  /* Default to adding an anonymous hidden service if no flag is given */
-  int non_anonymous = 0;
-  for (int i = 1; i < arg_len; i++) {
-    static const char *port_prefix = "Port=";
-    static const char *flags_prefix = "Flags=";
-    static const char *max_s_prefix = "MaxStreams=";
-    static const char *auth_prefix = "ClientAuth=";
-
-    const char *arg = smartlist_get(args, (int)i);
-    if (!strcasecmpstart(arg, port_prefix)) {
-      /* "Port=VIRTPORT[,TARGET]". */
-      const char *port_str = arg + strlen(port_prefix);
-
-      rend_service_port_config_t *cfg =
-          rend_service_parse_port_config(port_str, ",", NULL);
-      if (!cfg) {
-        connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-        goto out;
-      }
-      smartlist_add(port_cfgs, cfg);
-    } else if (!strcasecmpstart(arg, max_s_prefix)) {
-      /* "MaxStreams=[0..65535]". */
-      const char *max_s_str = arg + strlen(max_s_prefix);
-      int ok = 0;
-      max_streams = (int)tor_parse_long(max_s_str, 10, 0, 65535, &ok, NULL);
-      if (!ok) {
-        connection_printf_to_buf(conn, "512 Invalid MaxStreams\r\n");
-        goto out;
-      }
-    } else if (!strcasecmpstart(arg, flags_prefix)) {
-      /* "Flags=Flag[,Flag]", where Flag can be:
-       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
-       *                   the response.
-       *   * 'Detach' - Do not tie this onion service to any particular control
-       *                connection.
-       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
-       *                                exceeded.
-       *   * 'BasicAuth' - Client authorization using the 'basic' method.
-       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
-       *                      flag is present, tor must be in non-anonymous
-       *                      hidden service mode. If this flag is absent,
-       *                      tor must be in anonymous hidden service mode.
-       */
-      static const char *discard_flag = "DiscardPK";
-      static const char *detach_flag = "Detach";
-      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
-      static const char *basicauth_flag = "BasicAuth";
-      static const char *non_anonymous_flag = "NonAnonymous";
-
-      smartlist_t *flags = smartlist_new();
-      int bad = 0;
-
-      smartlist_split_string(flags, arg + strlen(flags_prefix), ",",
-                             SPLIT_IGNORE_BLANK, 0);
-      if (smartlist_len(flags) < 1) {
-        connection_printf_to_buf(conn, "512 Invalid 'Flags' argument\r\n");
-        bad = 1;
-      }
-      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
-      {
-        if (!strcasecmp(flag, discard_flag)) {
-          discard_pk = 1;
-        } else if (!strcasecmp(flag, detach_flag)) {
-          detach = 1;
-        } else if (!strcasecmp(flag, max_s_close_flag)) {
-          max_streams_close_circuit = 1;
-        } else if (!strcasecmp(flag, basicauth_flag)) {
-          auth_type = REND_BASIC_AUTH;
-        } else if (!strcasecmp(flag, non_anonymous_flag)) {
-          non_anonymous = 1;
-        } else {
-          connection_printf_to_buf(conn,
-                                   "512 Invalid 'Flags' argument: %s\r\n",
-                                   escaped(flag));
-          bad = 1;
-          break;
-        }
-      } SMARTLIST_FOREACH_END(flag);
-      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
-      smartlist_free(flags);
-      if (bad)
-        goto out;
-    } else if (!strcasecmpstart(arg, auth_prefix)) {
-      char *err_msg = NULL;
-      int created = 0;
-      rend_authorized_client_t *client =
-        add_onion_helper_clientauth(arg + strlen(auth_prefix),
-                                    &created, &err_msg);
-      if (!client) {
-        if (err_msg) {
-          connection_write_str_to_buf(err_msg, conn);
-          tor_free(err_msg);
-        }
-        goto out;
-      }
-
-      if (auth_clients != NULL) {
-        int bad = 0;
-        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
-          if (strcmp(ac->client_name, client->client_name) == 0) {
-            bad = 1;
-            break;
-          }
-        } SMARTLIST_FOREACH_END(ac);
-        if (bad) {
-          connection_printf_to_buf(conn,
-                                   "512 Duplicate name in ClientAuth\r\n");
-          rend_authorized_client_free(client);
-          goto out;
-        }
-      } else {
-        auth_clients = smartlist_new();
-        auth_created_clients = smartlist_new();
-      }
-      smartlist_add(auth_clients, client);
-      if (created) {
-        smartlist_add(auth_created_clients, client);
-      }
-    } else {
-      connection_printf_to_buf(conn, "513 Invalid argument\r\n");
-      goto out;
-    }
-  }
-  if (smartlist_len(port_cfgs) == 0) {
-    connection_printf_to_buf(conn, "512 Missing 'Port' argument\r\n");
-    goto out;
-  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
-    connection_printf_to_buf(conn, "512 No auth type specified\r\n");
-    goto out;
-  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
-    connection_printf_to_buf(conn, "512 No auth clients specified\r\n");
-    goto out;
-  } else if ((auth_type == REND_BASIC_AUTH &&
-              smartlist_len(auth_clients) > 512) ||
-             (auth_type == REND_STEALTH_AUTH &&
-              smartlist_len(auth_clients) > 16)) {
-    connection_printf_to_buf(conn, "512 Too many auth clients\r\n");
-    goto out;
-  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
-                                                              get_options())) {
-    /* If we failed, and the non-anonymous flag is set, Tor must be in
-     * anonymous hidden service mode.
-     * The error message changes based on the current Tor config:
-     * 512 Tor is in anonymous hidden service mode
-     * 512 Tor is in non-anonymous hidden service mode
-     * (I've deliberately written them out in full here to aid searchability.)
-     */
-    connection_printf_to_buf(conn, "512 Tor is in %sanonymous hidden service "
-                             "mode\r\n",
-                             non_anonymous ? "" : "non-");
-    goto out;
-  }
-
-  /* Parse the "keytype:keyblob" argument. */
-  int hs_version = 0;
-  add_onion_secret_key_t pk = { NULL };
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-
-  if (add_onion_helper_keyarg(smartlist_get(args, 0), discard_pk,
-                              &key_new_alg, &key_new_blob, &pk, &hs_version,
-                              &err_msg) < 0) {
-    if (err_msg) {
-      connection_write_str_to_buf(err_msg, conn);
-      tor_free(err_msg);
-    }
-    goto out;
-  }
-  tor_assert(!err_msg);
-
-  /* Hidden service version 3 don't have client authentication support so if
-   * ClientAuth was given, send back an error. */
-  if (hs_version == HS_VERSION_THREE && auth_clients) {
-    connection_printf_to_buf(conn, "513 ClientAuth not supported\r\n");
-    goto out;
-  }
-
-  /* Create the HS, using private key pk, client authentication auth_type,
-   * the list of auth_clients, and port config port_cfg.
-   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
-   * regardless of success/failure.
-   */
-  char *service_id = NULL;
-  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
-                                         max_streams,
-                                         max_streams_close_circuit, auth_type,
-                                         auth_clients, &service_id);
-  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
-  auth_clients = NULL; /* so is auth_clients */
-  switch (ret) {
-  case RSAE_OKAY:
-  {
-    if (detach) {
-      if (!detached_onion_services)
-        detached_onion_services = smartlist_new();
-      smartlist_add(detached_onion_services, service_id);
-    } else {
-      if (!conn->ephemeral_onion_services)
-        conn->ephemeral_onion_services = smartlist_new();
-      smartlist_add(conn->ephemeral_onion_services, service_id);
-    }
-
-    tor_assert(service_id);
-    connection_printf_to_buf(conn, "250-ServiceID=%s\r\n", service_id);
-    if (key_new_alg) {
-      tor_assert(key_new_blob);
-      connection_printf_to_buf(conn, "250-PrivateKey=%s:%s\r\n",
-                               key_new_alg, key_new_blob);
-    }
-    if (auth_created_clients) {
-      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
-        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
-                                                auth_type);
-        tor_assert(encoded);
-        connection_printf_to_buf(conn, "250-ClientAuth=%s:%s\r\n",
-                                 ac->client_name, encoded);
-        memwipe(encoded, 0, strlen(encoded));
-        tor_free(encoded);
-      });
-    }
-
-    connection_printf_to_buf(conn, "250 OK\r\n");
-    break;
-  }
-  case RSAE_BADPRIVKEY:
-    connection_printf_to_buf(conn, "551 Failed to generate onion address\r\n");
-    break;
-  case RSAE_ADDREXISTS:
-    connection_printf_to_buf(conn, "550 Onion address collision\r\n");
-    break;
-  case RSAE_BADVIRTPORT:
-    connection_printf_to_buf(conn, "512 Invalid VIRTPORT/TARGET\r\n");
-    break;
-  case RSAE_BADAUTH:
-    connection_printf_to_buf(conn, "512 Invalid client authorization\r\n");
-    break;
-  case RSAE_INTERNAL: FALLTHROUGH;
-  default:
-    connection_printf_to_buf(conn, "551 Failed to add Onion Service\r\n");
-  }
-  if (key_new_blob) {
-    memwipe(key_new_blob, 0, strlen(key_new_blob));
-    tor_free(key_new_blob);
-  }
-
- out:
-  if (port_cfgs) {
-    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
-                      rend_service_port_config_free(p));
-    smartlist_free(port_cfgs);
-  }
-
-  if (auth_clients) {
-    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
-                      rend_authorized_client_free(ac));
-    smartlist_free(auth_clients);
-  }
-  if (auth_created_clients) {
-    // Do not free entries; they are the same as auth_clients
-    smartlist_free(auth_created_clients);
-  }
-
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
-/** Helper function to handle parsing the KeyType:KeyBlob argument to the
- * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
- * and the private key not discarded, the algorithm and serialized private key,
- * or NULL and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned key_new_blob and err_msg.
- *
- * Note: The error messages returned are deliberately vague to avoid echoing
- * key material.
- */
-STATIC int
-add_onion_helper_keyarg(const char *arg, int discard_pk,
-                        const char **key_new_alg_out, char **key_new_blob_out,
-                        add_onion_secret_key_t *decoded_key, int *hs_version,
-                        char **err_msg_out)
-{
-  smartlist_t *key_args = smartlist_new();
-  crypto_pk_t *pk = NULL;
-  const char *key_new_alg = NULL;
-  char *key_new_blob = NULL;
-  char *err_msg = NULL;
-  int ret = -1;
-
-  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
-  if (smartlist_len(key_args) != 2) {
-    err_msg = tor_strdup("512 Invalid key type/blob\r\n");
-    goto err;
-  }
-
-  /* The format is "KeyType:KeyBlob". */
-  static const char *key_type_new = "NEW";
-  static const char *key_type_best = "BEST";
-  static const char *key_type_rsa1024 = "RSA1024";
-  static const char *key_type_ed25519_v3 = "ED25519-V3";
-
-  const char *key_type = smartlist_get(key_args, 0);
-  const char *key_blob = smartlist_get(key_args, 1);
-
-  if (!strcasecmp(key_type_rsa1024, key_type)) {
-    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
-    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
-    if (!pk) {
-      err_msg = tor_strdup("512 Failed to decode RSA key\r\n");
-      goto err;
-    }
-    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
-      crypto_pk_free(pk);
-      err_msg = tor_strdup("512 Invalid RSA key size\r\n");
-      goto err;
-    }
-    decoded_key->v2 = pk;
-    *hs_version = HS_VERSION_TWO;
-  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
-    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
-    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
-                      strlen(key_blob)) != sizeof(sk->seckey)) {
-      tor_free(sk);
-      err_msg = tor_strdup("512 Failed to decode ED25519-V3 key\r\n");
-      goto err;
-    }
-    decoded_key->v3 = sk;
-    *hs_version = HS_VERSION_THREE;
-  } else if (!strcasecmp(key_type_new, key_type)) {
-    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
-    if (!strcasecmp(key_type_rsa1024, key_blob) ||
-        !strcasecmp(key_type_best, key_blob)) {
-      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
-      pk = crypto_pk_new();
-      if (crypto_pk_generate_key(pk)) {
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_rsa1024);
-        goto err;
-      }
-      if (!discard_pk) {
-        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
-          crypto_pk_free(pk);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_rsa1024);
-          goto err;
-        }
-        key_new_alg = key_type_rsa1024;
-      }
-      decoded_key->v2 = pk;
-      *hs_version = HS_VERSION_TWO;
-    } else if (!strcasecmp(key_type_ed25519_v3, key_blob)) {
-      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
-      if (ed25519_secret_key_generate(sk, 1) < 0) {
-        tor_free(sk);
-        tor_asprintf(&err_msg, "551 Failed to generate %s key\r\n",
-                     key_type_ed25519_v3);
-        goto err;
-      }
-      if (!discard_pk) {
-        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
-        key_new_blob = tor_malloc_zero(len);
-        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
-                          sizeof(sk->seckey), 0) != (len - 1)) {
-          tor_free(sk);
-          tor_free(key_new_blob);
-          tor_asprintf(&err_msg, "551 Failed to encode %s key\r\n",
-                       key_type_ed25519_v3);
-          goto err;
-        }
-        key_new_alg = key_type_ed25519_v3;
-      }
-      decoded_key->v3 = sk;
-      *hs_version = HS_VERSION_THREE;
-    } else {
-      err_msg = tor_strdup("513 Invalid key type\r\n");
-      goto err;
-    }
-  } else {
-    err_msg = tor_strdup("513 Invalid key type\r\n");
-    goto err;
-  }
-
-  /* Succeeded in loading or generating a private key. */
-  ret = 0;
-
- err:
-  SMARTLIST_FOREACH(key_args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(key_args);
-
-  if (err_msg_out) {
-    *err_msg_out = err_msg;
-  } else {
-    tor_free(err_msg);
-  }
-  *key_new_alg_out = key_new_alg;
-  *key_new_blob_out = key_new_blob;
-
-  return ret;
-}
-
-/** Helper function to handle parsing a ClientAuth argument to the
- * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
- * and an optional control protocol error message on failure.  The
- * caller is responsible for freeing the returned auth_client and err_msg.
- *
- * If 'created' is specified, it will be set to 1 when a new cookie has
- * been generated.
- */
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg)
-{
-  int ok = 0;
-
-  tor_assert(arg);
-  tor_assert(created);
-  tor_assert(err_msg);
-  *err_msg = NULL;
-
-  smartlist_t *auth_args = smartlist_new();
-  rend_authorized_client_t *client =
-    tor_malloc_zero(sizeof(rend_authorized_client_t));
-  smartlist_split_string(auth_args, arg, ":", 0, 0);
-  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
-    *err_msg = tor_strdup("512 Invalid ClientAuth syntax\r\n");
-    goto err;
-  }
-  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
-  if (smartlist_len(auth_args) == 2) {
-    char *decode_err_msg = NULL;
-    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
-                                client->descriptor_cookie,
-                                NULL, &decode_err_msg) < 0) {
-      tor_assert(decode_err_msg);
-      tor_asprintf(err_msg, "512 %s\r\n", decode_err_msg);
-      tor_free(decode_err_msg);
-      goto err;
-    }
-    *created = 0;
-  } else {
-    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
-    *created = 1;
-  }
-
-  if (!rend_valid_client_name(client->client_name)) {
-    *err_msg = tor_strdup("512 Invalid name in ClientAuth\r\n");
-    goto err;
-  }
-
-  ok = 1;
- err:
-  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
-  smartlist_free(auth_args);
-  if (!ok) {
-    rend_authorized_client_free(client);
-    client = NULL;
-  }
-  return client;
-}
-
-/** Called when we get a DEL_ONION command; parse the body, and remove
- * the existing ephemeral Onion Service. */
-static int
-handle_control_del_onion(control_connection_t *conn,
-                          uint32_t len,
-                          const char *body)
-{
-  int hs_version = 0;
-  smartlist_t *args;
-  (void) len; /* body is nul-terminated; it's safe to ignore the length */
-  args = getargs_helper("DEL_ONION", conn, body, 1, 1);
-  if (!args)
-    return 0;
-
-  const char *service_id = smartlist_get(args, 0);
-  if (rend_valid_v2_service_id(service_id)) {
-    hs_version = HS_VERSION_TWO;
-  } else if (hs_address_is_valid(service_id)) {
-    hs_version = HS_VERSION_THREE;
-  } else {
-    connection_printf_to_buf(conn, "512 Malformed Onion Service id\r\n");
-    goto out;
-  }
-
-  /* Determine if the onion service belongs to this particular control
-   * connection, or if it is in the global list of detached services.  If it
-   * is in neither, either the service ID is invalid in some way, or it
-   * explicitly belongs to a different control connection, and an error
-   * should be returned.
-   */
-  smartlist_t *services[2] = {
-    conn->ephemeral_onion_services,
-    detached_onion_services
-  };
-  smartlist_t *onion_services = NULL;
-  int idx = -1;
-  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
-    idx = smartlist_string_pos(services[i], service_id);
-    if (idx != -1) {
-      onion_services = services[i];
-      break;
-    }
-  }
-  if (onion_services == NULL) {
-    connection_printf_to_buf(conn, "552 Unknown Onion Service id\r\n");
-  } else {
-    int ret = -1;
-    switch (hs_version) {
-    case HS_VERSION_TWO:
-      ret = rend_service_del_ephemeral(service_id);
-      break;
-    case HS_VERSION_THREE:
-      ret = hs_service_del_ephemeral(service_id);
-      break;
-    default:
-      /* The ret value will be -1 thus hitting the warning below. This should
-       * never happen because of the check at the start of the function. */
-      break;
-    }
-    if (ret < 0) {
-      /* This should *NEVER* fail, since the service is on either the
-       * per-control connection list, or the global one.
-       */
-      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
-               escaped(service_id));
-      tor_fragile_assert();
-    }
-
-    /* Remove/scrub the service_id from the appropriate list. */
-    char *cp = smartlist_get(onion_services, idx);
-    smartlist_del(onion_services, idx);
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-
-    send_control_done(conn);
-  }
-
- out:
-  SMARTLIST_FOREACH(args, char *, cp, {
-    memwipe(cp, 0, strlen(cp));
-    tor_free(cp);
-  });
-  smartlist_free(args);
-  return 0;
-}
-
 /** Called when <b>conn</b> has no more bytes left on its outbuf. */
 int
 connection_control_finished_flushing(control_connection_t *conn)
@@ -5349,6 +279,44 @@ peek_connection_has_http_command(connection_t *conn)
   return peek_buf_has_http_command(conn->inbuf);
 }
 
+/**
+ * Helper: take a nul-terminated command of given length, and find where the
+ * command starts and the arguments begin.  Separate them, allocate a new
+ * string in <b>current_cmd_out</b> for the command, and return a pointer
+ * to the arguments.
+ **/
+STATIC char *
+control_split_incoming_command(char *incoming_cmd,
+                               size_t *data_len,
+                               char **current_cmd_out)
+{
+  const bool is_multiline = *data_len && incoming_cmd[0] == '+';
+  size_t cmd_len = 0;
+  while (cmd_len < *data_len
+         && !TOR_ISSPACE(incoming_cmd[cmd_len]))
+    ++cmd_len;
+
+  *current_cmd_out = tor_memdup_nulterm(incoming_cmd, cmd_len);
+  char *args = incoming_cmd+cmd_len;
+  tor_assert(*data_len>=cmd_len);
+  *data_len -= cmd_len;
+  if (is_multiline) {
+    // Only match horizontal space: any line after the first is data,
+    // not arguments.
+    while ((*args == '\t' || *args == ' ') && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  } else {
+    while (TOR_ISSPACE(*args) && *data_len) {
+      ++args;
+      --*data_len;
+    }
+  }
+
+  return args;
+}
+
 static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "HTTP/1.0 501 Tor ControlPort is not an HTTP proxy"
   "\r\nContent-Type: text/html; charset=iso-8859-1\r\n\r\n"
@@ -5375,6 +343,60 @@ static const char CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG[] =
   "</body>\n"
   "</html>\n";
 
+/** Return an error on a control connection that tried to use the v0 protocol.
+ */
+static void
+control_send_v0_reject(control_connection_t *conn)
+{
+  size_t body_len;
+  char buf[128];
+  set_uint16(buf+2, htons(0x0000)); /* type == error */
+  set_uint16(buf+4, htons(0x0001)); /* code == internal error */
+  strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
+          "and later; upgrade your controller.",
+          sizeof(buf)-6);
+  body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
+  set_uint16(buf+0, htons(body_len));
+  connection_buf_add(buf, 4+body_len, TO_CONN(conn));
+
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Return an error on a control connection that tried to use HTTP.
+ */
+static void
+control_send_http_reject(control_connection_t *conn)
+{
+  connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
+  log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
+  connection_mark_and_flush(TO_CONN(conn));
+}
+
+/** Check if a control connection has tried to use a known invalid protocol.
+ * If it has, then:
+ *  - send a reject response,
+ *  - log a notice-level message, and
+ *  - return false. */
+static bool
+control_protocol_is_valid(control_connection_t *conn)
+{
+  /* Detect v0 commands and send a "no more v0" message. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_control0_command(TO_CONN(conn))) {
+    control_send_v0_reject(conn);
+    return 0;
+  }
+
+  /* If the user has the HTTP proxy port and the control port confused. */
+  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
+      peek_connection_has_http_command(TO_CONN(conn))) {
+    control_send_http_reject(conn);
+    return 0;
+  }
+
+  return 1;
+}
+
 /** Called when data has arrived on a v1 control connection: Try to fetch
  * commands from conn->inbuf, and execute them.
  */
@@ -5383,7 +405,6 @@ connection_control_process_inbuf(control_connection_t *conn)
 {
   size_t data_len;
   uint32_t cmd_data_len;
-  int cmd_len;
   char *args;
 
   tor_assert(conn);
@@ -5396,30 +417,7 @@ connection_control_process_inbuf(control_connection_t *conn)
     conn->incoming_cmd_cur_len = 0;
   }
 
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_control0_command(TO_CONN(conn))) {
-    /* Detect v0 commands and send a "no more v0" message. */
-    size_t body_len;
-    char buf[128];
-    set_uint16(buf+2, htons(0x0000)); /* type == error */
-    set_uint16(buf+4, htons(0x0001)); /* code == internal error */
-    strlcpy(buf+6, "The v0 control protocol is not supported by Tor 0.1.2.17 "
-            "and later; upgrade your controller.",
-            sizeof(buf)-6);
-    body_len = 2+strlen(buf+6)+2; /* code, msg, nul. */
-    set_uint16(buf+0, htons(body_len));
-    connection_buf_add(buf, 4+body_len, TO_CONN(conn));
-
-    connection_mark_and_flush(TO_CONN(conn));
-    return 0;
-  }
-
-  /* If the user has the HTTP proxy port and the control port confused. */
-  if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      peek_connection_has_http_command(TO_CONN(conn))) {
-    connection_write_str_to_buf(CONTROLPORT_IS_NOT_AN_HTTP_PROXY_MSG, conn);
-    log_notice(LD_CONTROL, "Received HTTP request on ControlPort");
-    connection_mark_and_flush(TO_CONN(conn));
+  if (!control_protocol_is_valid(conn)) {
     return 0;
   }
 
@@ -5438,7 +436,7 @@ connection_control_process_inbuf(control_connection_t *conn)
         return 0;
       else if (r == -1) {
         if (data_len + conn->incoming_cmd_cur_len > MAX_COMMAND_LINE_LENGTH) {
-          connection_write_str_to_buf("500 Line too long.\r\n", conn);
+          control_write_endreply(conn, 500, "Line too long.");
           connection_stop_reading(TO_CONN(conn));
           connection_mark_and_flush(TO_CONN(conn));
         }
@@ -5475,22 +473,15 @@ connection_control_process_inbuf(control_connection_t *conn)
     /* Otherwise, read another line. */
   }
   data_len = conn->incoming_cmd_cur_len;
+
   /* Okay, we now have a command sitting on conn->incoming_cmd. See if we
    * recognize it.
    */
-  cmd_len = 0;
-  while ((size_t)cmd_len < data_len
-         && !TOR_ISSPACE(conn->incoming_cmd[cmd_len]))
-    ++cmd_len;
-
-  conn->incoming_cmd[cmd_len]='\0';
-  args = conn->incoming_cmd+cmd_len+1;
-  tor_assert(data_len>(size_t)cmd_len);
-  data_len -= (cmd_len+1); /* skip the command and NUL we added after it */
-  while (TOR_ISSPACE(*args)) {
-    ++args;
-    --data_len;
-  }
+  tor_free(conn->current_cmd);
+  args = control_split_incoming_command(conn->incoming_cmd, &data_len,
+                                        &conn->current_cmd);
+  if (BUG(!conn->current_cmd))
+    return -1;
 
   /* If the connection is already closing, ignore further commands */
   if (TO_CONN(conn)->marked_for_close) {
@@ -5498,1448 +489,50 @@ connection_control_process_inbuf(control_connection_t *conn)
   }
 
   /* Otherwise, Quit is always valid. */
-  if (!strcasecmp(conn->incoming_cmd, "QUIT")) {
-    connection_write_str_to_buf("250 closing connection\r\n", conn);
+  if (!strcasecmp(conn->current_cmd, "QUIT")) {
+    control_write_endreply(conn, 250, "closing connection");
     connection_mark_and_flush(TO_CONN(conn));
     return 0;
   }
 
   if (conn->base_.state == CONTROL_CONN_STATE_NEEDAUTH &&
-      !is_valid_initial_command(conn, conn->incoming_cmd)) {
-    connection_write_str_to_buf("514 Authentication required.\r\n", conn);
+      !is_valid_initial_command(conn, conn->current_cmd)) {
+    control_write_endreply(conn, 514, "Authentication required.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
   if (data_len >= UINT32_MAX) {
-    connection_write_str_to_buf("500 A 4GB command? Nice try.\r\n", conn);
+    control_write_endreply(conn, 500, "A 4GB command? Nice try.");
     connection_mark_for_close(TO_CONN(conn));
     return 0;
   }
 
-  /* XXXX Why is this not implemented as a table like the GETINFO
-   * items are?  Even handling the plus signs at the beginnings of
-   * commands wouldn't be very hard with proper macros. */
   cmd_data_len = (uint32_t)data_len;
-  if (!strcasecmp(conn->incoming_cmd, "SETCONF")) {
-    if (handle_control_setconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESETCONF")) {
-    if (handle_control_resetconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETCONF")) {
-    if (handle_control_getconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+LOADCONF")) {
-    if (handle_control_loadconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETEVENTS")) {
-    if (handle_control_setevents(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHENTICATE")) {
-    if (handle_control_authenticate(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SAVECONF")) {
-    if (handle_control_saveconf(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SIGNAL")) {
-    if (handle_control_signal(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "TAKEOWNERSHIP")) {
-    if (handle_control_takeownership(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "MAPADDRESS")) {
-    if (handle_control_mapaddress(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "GETINFO")) {
-    if (handle_control_getinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "EXTENDCIRCUIT")) {
-    if (handle_control_extendcircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETCIRCUITPURPOSE")) {
-    if (handle_control_setcircuitpurpose(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "SETROUTERPURPOSE")) {
-    connection_write_str_to_buf("511 SETROUTERPURPOSE is obsolete.\r\n", conn);
-  } else if (!strcasecmp(conn->incoming_cmd, "ATTACHSTREAM")) {
-    if (handle_control_attachstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+POSTDESCRIPTOR")) {
-    if (handle_control_postdescriptor(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "REDIRECTSTREAM")) {
-    if (handle_control_redirectstream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSESTREAM")) {
-    if (handle_control_closestream(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "CLOSECIRCUIT")) {
-    if (handle_control_closecircuit(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "USEFEATURE")) {
-    if (handle_control_usefeature(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "RESOLVE")) {
-    if (handle_control_resolve(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "PROTOCOLINFO")) {
-    if (handle_control_protocolinfo(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "AUTHCHALLENGE")) {
-    if (handle_control_authchallenge(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DROPGUARDS")) {
-    if (handle_control_dropguards(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "HSFETCH")) {
-    if (handle_control_hsfetch(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "+HSPOST")) {
-    if (handle_control_hspost(conn, cmd_data_len, args))
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "ADD_ONION")) {
-    int ret = handle_control_add_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the private key. */
-    if (ret)
-      return -1;
-  } else if (!strcasecmp(conn->incoming_cmd, "DEL_ONION")) {
-    int ret = handle_control_del_onion(conn, cmd_data_len, args);
-    memwipe(args, 0, cmd_data_len); /* Scrub the service id/pk. */
-    if (ret)
-      return -1;
-  } else {
-    connection_printf_to_buf(conn, "510 Unrecognized command \"%s\"\r\n",
-                             conn->incoming_cmd);
-  }
+  if (handle_control_command(conn, cmd_data_len, args) < 0)
+    return -1;
 
   conn->incoming_cmd_cur_len = 0;
   goto again;
 }
 
-/** Something major has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-int
-control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
-                             int reason_code)
-{
-  const char *status;
-  char reasons[64] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
-    return 0;
-  tor_assert(circ);
-
-  switch (tp)
-    {
-    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case CIRC_EVENT_BUILT: status = "BUILT"; break;
-    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
-    case CIRC_EVENT_FAILED: status = "FAILED"; break;
-    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
-    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
-    char unk_reason_buf[16];
-    if (!reason_str) {
-      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
-      reason_str = unk_reason_buf;
-    }
-    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
-    } else {
-      tor_snprintf(reasons, sizeof(reasons),
-                   " REASON=%s", reason_str);
-    }
-  }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS,
-                                "650 CIRC %lu %s%s%s%s\r\n",
-                                (unsigned long)circ->global_identifier,
-                                status, sp,
-                                circdesc,
-                                reasons);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/** Something minor has happened to circuit <b>circ</b>: tell any
- * interested control connections. */
-static int
-control_event_circuit_status_minor(origin_circuit_t *circ,
-                                   circuit_status_minor_event_t e,
-                                   int purpose, const struct timeval *tv)
-{
-  const char *event_desc;
-  char event_tail[160] = "";
-  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
-    return 0;
-  tor_assert(circ);
-
-  switch (e)
-    {
-    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
-      event_desc = "PURPOSE_CHANGED";
-
-      {
-        /* event_tail can currently be up to 68 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "");
-      }
-
-      break;
-    case CIRC_MINOR_EVENT_CANNIBALIZED:
-      event_desc = "CANNIBALIZED";
-
-      {
-        /* event_tail can currently be up to 130 chars long */
-        const char *hs_state_str =
-          circuit_purpose_to_controller_hs_state_string(purpose);
-        const struct timeval *old_timestamp_began = tv;
-        char tbuf[ISO_TIME_USEC_LEN+1];
-        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
-
-        tor_snprintf(event_tail, sizeof(event_tail),
-                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
-                     circuit_purpose_to_controller_string(purpose),
-                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
-                     (hs_state_str != NULL) ? hs_state_str : "",
-                     tbuf);
-      }
-
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
-      tor_fragile_assert();
-      return 0;
-    }
-
-  {
-    char *circdesc = circuit_describe_status_for_controller(circ);
-    const char *sp = strlen(circdesc) ? " " : "";
-    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
-                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
-                       (unsigned long)circ->global_identifier,
-                       event_desc, sp,
-                       circdesc,
-                       event_tail);
-    tor_free(circdesc);
-  }
-
-  return 0;
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
- * interested controllers.
- */
-int
-control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                      int old_purpose)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-                                            old_purpose,
-                                            NULL);
-}
-
-/**
- * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
- * created-time from <b>old_tv_created</b>: tell any interested controllers.
- */
-int
-control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                   int old_purpose,
-                                   const struct timeval *old_tv_created)
-{
-  return control_event_circuit_status_minor(circ,
-                                            CIRC_MINOR_EVENT_CANNIBALIZED,
-                                            old_purpose,
-                                            old_tv_created);
-}
-
-/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
- * <b>buf</b>, determine the address:port combination requested on
- * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
- * failure. */
-static int
-write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
-{
-  char buf2[256];
-  if (conn->chosen_exit_name)
-    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
-      return -1;
-  if (!conn->socks_request)
-    return -1;
-  if (tor_snprintf(buf, len, "%s%s%s:%d",
-               conn->socks_request->address,
-               conn->chosen_exit_name ? buf2 : "",
-               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
-                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
-               conn->socks_request->port)<0)
-    return -1;
-  return 0;
-}
-
-/** Something has happened to the stream associated with AP connection
- * <b>conn</b>: tell any interested control connections. */
-int
-control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
-                            int reason_code)
-{
-  char reason_buf[64];
-  char addrport_buf[64];
-  const char *status;
-  circuit_t *circ;
-  origin_circuit_t *origin_circ = NULL;
-  char buf[256];
-  const char *purpose = "";
-  tor_assert(conn->socks_request);
-
-  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
-    return 0;
-
-  if (tp == STREAM_EVENT_CLOSED &&
-      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
-    return 0;
-
-  write_stream_target_to_buf(conn, buf, sizeof(buf));
-
-  reason_buf[0] = '\0';
-  switch (tp)
-    {
-    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
-    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
-    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
-    case STREAM_EVENT_FAILED: status = "FAILED"; break;
-    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
-    case STREAM_EVENT_NEW: status = "NEW"; break;
-    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
-    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
-    case STREAM_EVENT_REMAP: status = "REMAP"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (reason_code && (tp == STREAM_EVENT_FAILED ||
-                      tp == STREAM_EVENT_CLOSED ||
-                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
-    const char *reason_str = stream_end_reason_to_control_string(reason_code);
-    char *r = NULL;
-    if (!reason_str) {
-      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
-      reason_str = r;
-    }
-    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=END REMOTE_REASON=%s", reason_str);
-    else
-      tor_snprintf(reason_buf, sizeof(reason_buf),
-                   " REASON=%s", reason_str);
-    tor_free(r);
-  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
-    switch (reason_code) {
-    case REMAP_STREAM_SOURCE_CACHE:
-      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
-      break;
-    case REMAP_STREAM_SOURCE_EXIT:
-      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
-      break;
-    default:
-      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
-                   reason_code);
-      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
-      break;
-    }
-  }
-
-  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
-    /*
-     * When the control conn is an AF_UNIX socket and we have no address,
-     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
-     * dnsserv.c.
-     */
-    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
-      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
-                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
-    } else {
-      /*
-       * else leave it blank so control on AF_UNIX doesn't need to make
-       * something up.
-       */
-      addrport_buf[0] = '\0';
-    }
-  } else {
-    addrport_buf[0] = '\0';
-  }
-
-  if (tp == STREAM_EVENT_NEW_RESOLVE) {
-    purpose = " PURPOSE=DNS_REQUEST";
-  } else if (tp == STREAM_EVENT_NEW) {
-    if (conn->use_begindir) {
-      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
-      int linked_dir_purpose = -1;
-      if (linked && linked->type == CONN_TYPE_DIR)
-        linked_dir_purpose = linked->purpose;
-      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
-        purpose = " PURPOSE=DIR_UPLOAD";
-      else
-        purpose = " PURPOSE=DIR_FETCH";
-    } else
-      purpose = " PURPOSE=USER";
-  }
-
-  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
-  if (circ && CIRCUIT_IS_ORIGIN(circ))
-    origin_circ = TO_ORIGIN_CIRCUIT(circ);
-  send_control_event(EVENT_STREAM_STATUS,
-                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s\r\n",
-                     (ENTRY_TO_CONN(conn)->global_identifier),
-                     status,
-                        origin_circ?
-                           (unsigned long)origin_circ->global_identifier : 0ul,
-                        buf, reason_buf, addrport_buf, purpose);
-
-  /* XXX need to specify its intended exit, etc? */
-
-  return 0;
-}
-
-/** Figure out the best name for the target router of an OR connection
- * <b>conn</b>, and write it into the <b>len</b>-character buffer
- * <b>name</b>. */
-static void
-orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
-{
-  const node_t *node = node_get_by_id(conn->identity_digest);
-  if (node) {
-    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
-    node_get_verbose_nickname(node, name);
-  } else if (! tor_digest_is_zero(conn->identity_digest)) {
-    name[0] = '$';
-    base16_encode(name+1, len-1, conn->identity_digest,
-                  DIGEST_LEN);
-  } else {
-    tor_snprintf(name, len, "%s:%d",
-                 conn->base_.address, conn->base_.port);
-  }
-}
-
-/** Called when the status of an OR connection <b>conn</b> changes: tell any
- * interested control connections. <b>tp</b> is the new status for the
- * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
- * may be the reason why.
- */
-int
-control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
-                             int reason)
-{
-  int ncircs = 0;
-  const char *status;
-  char name[128];
-  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
-
-  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
-    return 0;
-
-  switch (tp)
-    {
-    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
-    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
-    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
-    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
-    case OR_CONN_EVENT_NEW: status = "NEW"; break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
-      return 0;
-    }
-  if (conn->chan) {
-    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
-  } else {
-    ncircs = 0;
-  }
-  ncircs += connection_or_get_num_circuits(conn);
-  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
-    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
-  }
-
-  orconn_target_get_name(name, sizeof(name), conn);
-  send_control_event(EVENT_OR_CONN_STATUS,
-                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
-                              name, status,
-                              reason ? " REASON=" : "",
-                              orconn_end_reason_to_control_string(reason),
-                              ncircs_buf,
-                              (conn->base_.global_identifier));
-
-  return 0;
-}
-
-/**
- * Print out STREAM_BW event for a single conn
- */
-int
-control_event_stream_bandwidth(edge_connection_t *edge_conn)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    if (!edge_conn->n_read && !edge_conn->n_written)
-      return 0;
-
-    tor_gettimeofday(&now);
-    format_iso_time_nospace_usec(tbuf, &now);
-    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                       (edge_conn->base_.global_identifier),
-                       (unsigned long)edge_conn->n_read,
-                       (unsigned long)edge_conn->n_written,
-                       tbuf);
-
-    edge_conn->n_written = edge_conn->n_read = 0;
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth streams have used. */
-int
-control_event_stream_bandwidth_used(void)
-{
-  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
-    smartlist_t *conns = get_connection_array();
-    edge_connection_t *edge_conn;
-    struct timeval now;
-    char tbuf[ISO_TIME_USEC_LEN+1];
-
-    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
-    {
-        if (conn->type != CONN_TYPE_AP)
-          continue;
-        edge_conn = TO_EDGE_CONN(conn);
-        if (!edge_conn->n_read && !edge_conn->n_written)
-          continue;
-
-        tor_gettimeofday(&now);
-        format_iso_time_nospace_usec(tbuf, &now);
-        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
-                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
-                           (edge_conn->base_.global_identifier),
-                           (unsigned long)edge_conn->n_read,
-                           (unsigned long)edge_conn->n_written,
-                           tbuf);
-
-        edge_conn->n_written = edge_conn->n_read = 0;
-    }
-    SMARTLIST_FOREACH_END(conn);
-  }
-
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control connections
- * how much bandwidth origin circuits have used. */
-int
-control_event_circ_bandwidth_used(void)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!CIRCUIT_IS_ORIGIN(circ))
-      continue;
-
-    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
-  }
-  SMARTLIST_FOREACH_END(circ);
-
-  return 0;
-}
-
-/**
- * Emit a CIRC_BW event line for a specific circuit.
- *
- * This function sets the values it emits to 0, and does not emit
- * an event if there is no new data to report since the last call.
- *
- * Therefore, it may be called at any frequency.
- */
-int
-control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
-{
-  struct timeval now;
-  char tbuf[ISO_TIME_USEC_LEN+1];
-
-  tor_assert(ocirc);
-
-  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
-    return 0;
-
-  /* n_read_circ_bw and n_written_circ_bw are always updated
-   * when there is any new cell on a circuit, and set to 0 after
-   * the event, below.
-   *
-   * Therefore, checking them is sufficient to determine if there
-   * is new data to report. */
-  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
-    return 0;
-
-  tor_gettimeofday(&now);
-  format_iso_time_nospace_usec(tbuf, &now);
-  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
-                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
-                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
-                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
-                     ocirc->global_identifier,
-                     (unsigned long)ocirc->n_read_circ_bw,
-                     (unsigned long)ocirc->n_written_circ_bw,
-                     tbuf,
-                     (unsigned long)ocirc->n_delivered_read_circ_bw,
-                     (unsigned long)ocirc->n_overhead_read_circ_bw,
-                     (unsigned long)ocirc->n_delivered_written_circ_bw,
-                     (unsigned long)ocirc->n_overhead_written_circ_bw);
-  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
-  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
-  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
-
-  return 0;
-}
-
-/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
-  * bandwidth counters. */
-int
-control_event_conn_bandwidth(connection_t *conn)
-{
-  const char *conn_type_str;
-  if (!get_options()->TestingEnableConnBwEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
-    return 0;
-  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
-    return 0;
-  switch (conn->type) {
-    case CONN_TYPE_OR:
-      conn_type_str = "OR";
-      break;
-    case CONN_TYPE_DIR:
-      conn_type_str = "DIR";
-      break;
-    case CONN_TYPE_EXIT:
-      conn_type_str = "EXIT";
-      break;
-    default:
-      return 0;
-  }
-  send_control_event(EVENT_CONN_BW,
-                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
-                     "READ=%lu WRITTEN=%lu\r\n",
-                     (conn->global_identifier),
-                     conn_type_str,
-                     (unsigned long)conn->n_read_conn_bw,
-                     (unsigned long)conn->n_written_conn_bw);
-  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
-  return 0;
-}
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth connections have used. */
-int
-control_event_conn_bandwidth_used(void)
-{
-  if (get_options()->TestingEnableConnBwEvent &&
-      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
-    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
-                      control_event_conn_bandwidth(conn));
-  }
-  return 0;
-}
-
-/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
- * cells, removed cells, and waiting times by cell command and direction.
- * Store results in <b>cell_stats</b>.  Free cell statistics of the
- * circuit afterwards. */
-void
-sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
-{
-  memset(cell_stats, 0, sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
-                          const testing_cell_stats_entry_t *, ent) {
-    tor_assert(ent->command <= CELL_COMMAND_MAX_);
-    if (!ent->removed && !ent->exitward) {
-      cell_stats->added_cells_appward[ent->command] += 1;
-    } else if (!ent->removed && ent->exitward) {
-      cell_stats->added_cells_exitward[ent->command] += 1;
-    } else if (!ent->exitward) {
-      cell_stats->removed_cells_appward[ent->command] += 1;
-      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
-    } else {
-      cell_stats->removed_cells_exitward[ent->command] += 1;
-      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
-    }
-  } SMARTLIST_FOREACH_END(ent);
-  circuit_clear_testing_cell_stats(circ);
-}
-
-/** Helper: append a cell statistics string to <code>event_parts</code>,
- * prefixed with <code>key</code>=.  Statistics consist of comma-separated
- * key:value pairs with lower-case command strings as keys and cell
- * numbers or total waiting times as values.  A key:value pair is included
- * if the entry in <code>include_if_non_zero</code> is not zero, but with
- * the (possibly zero) entry from <code>number_to_include</code>.  Both
- * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
- * entry in <code>include_if_non_zero</code> is positive, no string will
- * be added to <code>event_parts</code>. */
-void
-append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
-                             const uint64_t *include_if_non_zero,
-                             const uint64_t *number_to_include)
-{
-  smartlist_t *key_value_strings = smartlist_new();
-  int i;
-  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
-    if (include_if_non_zero[i] > 0) {
-      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
-                             cell_command_to_string(i),
-                             (number_to_include[i]));
-    }
-  }
-  if (smartlist_len(key_value_strings) > 0) {
-    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
-    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
-    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
-    tor_free(joined);
-  }
-  smartlist_free(key_value_strings);
-}
-
-/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
- * CELL_STATS event and write result string to <b>event_string</b>. */
-void
-format_cell_stats(char **event_string, circuit_t *circ,
-                  cell_stats_t *cell_stats)
-{
-  smartlist_t *event_parts = smartlist_new();
-  if (CIRCUIT_IS_ORIGIN(circ)) {
-    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "ID=%lu",
-                 (unsigned long)ocirc->global_identifier);
-  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
-    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
-    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
-                 (unsigned long)or_circ->p_circ_id);
-    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
-                 (or_circ->p_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "InboundAdded",
-                                 cell_stats->added_cells_appward,
-                                 cell_stats->added_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundRemoved",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->removed_cells_appward);
-    append_cell_stats_by_command(event_parts, "InboundTime",
-                                 cell_stats->removed_cells_appward,
-                                 cell_stats->total_time_appward);
-  }
-  if (circ->n_chan) {
-    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
-                     (unsigned long)circ->n_circ_id);
-    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
-                 (circ->n_chan->global_identifier));
-    append_cell_stats_by_command(event_parts, "OutboundAdded",
-                                 cell_stats->added_cells_exitward,
-                                 cell_stats->added_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundRemoved",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->removed_cells_exitward);
-    append_cell_stats_by_command(event_parts, "OutboundTime",
-                                 cell_stats->removed_cells_exitward,
-                                 cell_stats->total_time_exitward);
-  }
-  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
-  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
-  smartlist_free(event_parts);
-}
-
-/** A second or more has elapsed: tell any interested control connection
- * how many cells have been processed for a given circuit. */
-int
-control_event_circuit_cell_stats(void)
-{
-  cell_stats_t *cell_stats;
-  char *event_string;
-  if (!get_options()->TestingEnableCellStatsEvent ||
-      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
-    return 0;
-  cell_stats = tor_malloc(sizeof(cell_stats_t));
-  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
-    if (!circ->testing_cell_stats)
-      continue;
-    sum_up_cell_stats_by_command(circ, cell_stats);
-    format_cell_stats(&event_string, circ, cell_stats);
-    send_control_event(EVENT_CELL_STATS,
-                       "650 CELL_STATS %s\r\n", event_string);
-    tor_free(event_string);
-  }
-  SMARTLIST_FOREACH_END(circ);
-  tor_free(cell_stats);
-  return 0;
-}
-
-/* about 5 minutes worth. */
-#define N_BW_EVENTS_TO_CACHE 300
-/* Index into cached_bw_events to next write. */
-static int next_measurement_idx = 0;
-/* number of entries set in n_measurements */
-static int n_measurements = 0;
-static struct cached_bw_event_s {
-  uint32_t n_read;
-  uint32_t n_written;
-} cached_bw_events[N_BW_EVENTS_TO_CACHE];
-
-/** A second or more has elapsed: tell any interested control
- * connections how much bandwidth we used. */
-int
-control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
-{
-  cached_bw_events[next_measurement_idx].n_read = n_read;
-  cached_bw_events[next_measurement_idx].n_written = n_written;
-  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
-    next_measurement_idx = 0;
-  if (n_measurements < N_BW_EVENTS_TO_CACHE)
-    ++n_measurements;
-
-  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
-    send_control_event(EVENT_BANDWIDTH_USED,
-                       "650 BW %lu %lu\r\n",
-                       (unsigned long)n_read,
-                       (unsigned long)n_written);
-  }
-
-  return 0;
-}
-
-STATIC char *
-get_bw_samples(void)
-{
-  int i;
-  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
-    % N_BW_EVENTS_TO_CACHE;
-  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-
-  smartlist_t *elements = smartlist_new();
-
-  for (i = 0; i < n_measurements; ++i) {
-    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
-    const struct cached_bw_event_s *bwe = &cached_bw_events[idx];
-
-    smartlist_add_asprintf(elements, "%u,%u",
-                           (unsigned)bwe->n_read,
-                           (unsigned)bwe->n_written);
-
-    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
-  }
-
-  char *result = smartlist_join_strings(elements, " ", 0, NULL);
-
-  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
-  smartlist_free(elements);
-
-  return result;
-}
-
-/** Called when we are sending a log message to the controllers: suspend
- * sending further log messages to the controllers until we're done.  Used by
- * CONN_LOG_PROTECT. */
-void
-disable_control_logging(void)
-{
-  ++disable_log_messages;
-}
-
-/** We're done sending a log message to the controllers: re-enable controller
- * logging.  Used by CONN_LOG_PROTECT. */
-void
-enable_control_logging(void)
-{
-  if (--disable_log_messages < 0)
-    tor_assert(0);
-}
-
-/** We got a log message: tell any interested control connections. */
-void
-control_event_logmsg(int severity, uint32_t domain, const char *msg)
-{
-  int event;
-
-  /* Don't even think of trying to add stuff to a buffer from a cpuworker
-   * thread. (See #25987 for plan to fix.) */
-  if (! in_main_thread())
-    return;
-
-  if (disable_log_messages)
-    return;
-
-  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
-      severity <= LOG_NOTICE) {
-    char *esc = esc_for_log(msg);
-    ++disable_log_messages;
-    control_event_general_status(severity, "BUG REASON=%s", esc);
-    --disable_log_messages;
-    tor_free(esc);
-  }
-
-  event = log_severity_to_event(severity);
-  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
-    char *b = NULL;
-    const char *s;
-    if (strchr(msg, '\n')) {
-      char *cp;
-      b = tor_strdup(msg);
-      for (cp = b; *cp; ++cp)
-        if (*cp == '\r' || *cp == '\n')
-          *cp = ' ';
-    }
-    switch (severity) {
-      case LOG_DEBUG: s = "DEBUG"; break;
-      case LOG_INFO: s = "INFO"; break;
-      case LOG_NOTICE: s = "NOTICE"; break;
-      case LOG_WARN: s = "WARN"; break;
-      case LOG_ERR: s = "ERR"; break;
-      default: s = "UnknownLogSeverity"; break;
-    }
-    ++disable_log_messages;
-    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
-    if (severity == LOG_ERR) {
-      /* Force a flush, since we may be about to die horribly */
-      queued_events_flush_all(1);
-    }
-    --disable_log_messages;
-    tor_free(b);
-  }
-}
-
-/**
- * Logging callback: called when there is a queued pending log callback.
- */
-void
-control_event_logmsg_pending(void)
-{
-  if (! in_main_thread()) {
-    /* We can't handle this case yet, since we're using a
-     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
-     * use a different mechanism instead: see #25987.
-     **/
-    return;
-  }
-  tor_assert(flush_queued_events_event);
-  mainloop_event_activate(flush_queued_events_event);
-}
-
-/** Called whenever we receive new router descriptors: tell any
- * interested control connections.  <b>routers</b> is a list of
- * routerinfo_t's.
- */
-int
-control_event_descriptors_changed(smartlist_t *routers)
-{
-  char *msg;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
-    return 0;
-
-  {
-    smartlist_t *names = smartlist_new();
-    char *ids;
-    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
-        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
-        router_get_verbose_nickname(b, ri);
-        smartlist_add(names, b);
-      });
-    ids = smartlist_join_strings(names, " ", 0, NULL);
-    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
-    send_control_event_string(EVENT_NEW_DESC,  msg);
-    tor_free(ids);
-    tor_free(msg);
-    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
-    smartlist_free(names);
-  }
-  return 0;
-}
-
-/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
- * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
- * <b>error</b> is non-NULL, it is an error code describing the failure
- * mode of the mapping.
- */
-int
-control_event_address_mapped(const char *from, const char *to, time_t expires,
-                             const char *error, const int cached)
-{
-  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
-    return 0;
-
-  if (expires < 3 || expires == TIME_MAX)
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s NEVER %s%s"
-                                "CACHED=\"%s\"\r\n",
-                                  from, to, error?error:"", error?" ":"",
-                                cached?"YES":"NO");
-  else {
-    char buf[ISO_TIME_LEN+1];
-    char buf2[ISO_TIME_LEN+1];
-    format_local_iso_time(buf,expires);
-    format_iso_time(buf2,expires);
-    send_control_event(EVENT_ADDRMAP,
-                                "650 ADDRMAP %s %s \"%s\""
-                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
-                                from, to, buf,
-                                error?error:"", error?" ":"",
-                                buf2, cached?"YES":"NO");
-  }
-
-  return 0;
-}
-
 /** Cached liveness for network liveness events and GETINFO
  */
 
 static int network_is_live = 0;
 
-static int
+int
 get_cached_network_liveness(void)
 {
   return network_is_live;
 }
 
-static void
+void
 set_cached_network_liveness(int liveness)
 {
   network_is_live = liveness;
 }
 
-/** The network liveness has changed; this is called from circuitstats.c
- * whenever we receive a cell, or when timeout expires and we assume the
- * network is down. */
-int
-control_event_network_liveness_update(int liveness)
-{
-  if (liveness > 0) {
-    if (get_cached_network_liveness() <= 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(1);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS UP\r\n");
-    }
-    /* else was already live, no-op */
-  } else {
-    if (get_cached_network_liveness() > 0) {
-      /* Update cached liveness */
-      set_cached_network_liveness(0);
-      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
-      send_control_event_string(EVENT_NETWORK_LIVENESS,
-                                "650 NETWORK_LIVENESS DOWN\r\n");
-    }
-    /* else was already dead, no-op */
-  }
-
-  return 0;
-}
-
-/** Helper function for NS-style events. Constructs and sends an event
- * of type <b>event</b> with string <b>event_string</b> out of the set of
- * networkstatuses <b>statuses</b>. Currently it is used for NS events
- * and NEWCONSENSUS events. */
-static int
-control_event_networkstatus_changed_helper(smartlist_t *statuses,
-                                           uint16_t event,
-                                           const char *event_string)
-{
-  smartlist_t *strs;
-  char *s, *esc = NULL;
-  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
-    return 0;
-
-  strs = smartlist_new();
-  smartlist_add_strdup(strs, "650+");
-  smartlist_add_strdup(strs, event_string);
-  smartlist_add_strdup(strs, "\r\n");
-  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
-    {
-      s = networkstatus_getinfo_helper_single(rs);
-      if (!s) continue;
-      smartlist_add(strs, s);
-    });
-
-  s = smartlist_join_strings(strs, "", 0, NULL);
-  write_escaped_data(s, strlen(s), &esc);
-  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
-  smartlist_free(strs);
-  tor_free(s);
-  send_control_event_string(event,  esc);
-  send_control_event_string(event,
-                            "650 OK\r\n");
-
-  tor_free(esc);
-  return 0;
-}
-
-/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
- * an NS event to any controller that cares. */
-int
-control_event_networkstatus_changed(smartlist_t *statuses)
-{
-  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
-}
-
-/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
- * event consisting of an NS-style line for each relay in the consensus. */
-int
-control_event_newconsensus(const networkstatus_t *consensus)
-{
-  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
-    return 0;
-  return control_event_networkstatus_changed_helper(
-           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
-}
-
-/** Called when we compute a new circuitbuildtimeout */
-int
-control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                               const char *args)
-{
-  const char *type_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
-    return 0;
-
-  switch (type) {
-    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
-      type_string = "COMPUTED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESET:
-      type_string = "RESET";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
-      type_string = "SUSPENDED";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_DISCARD:
-      type_string = "DISCARD";
-      break;
-    case BUILDTIMEOUT_SET_EVENT_RESUME:
-      type_string = "RESUME";
-      break;
-    default:
-      type_string = "UNKNOWN";
-      break;
-  }
-
-  send_control_event(EVENT_BUILDTIMEOUT_SET,
-                     "650 BUILDTIMEOUT_SET %s %s\r\n",
-                     type_string, args);
-
-  return 0;
-}
-
-/** Called when a signal has been processed from signal_callback */
-int
-control_event_signal(uintptr_t signal_num)
-{
-  const char *signal_string = NULL;
-
-  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
-    return 0;
-
-  switch (signal_num) {
-    case SIGHUP:
-      signal_string = "RELOAD";
-      break;
-    case SIGUSR1:
-      signal_string = "DUMP";
-      break;
-    case SIGUSR2:
-      signal_string = "DEBUG";
-      break;
-    case SIGNEWNYM:
-      signal_string = "NEWNYM";
-      break;
-    case SIGCLEARDNSCACHE:
-      signal_string = "CLEARDNSCACHE";
-      break;
-    case SIGHEARTBEAT:
-      signal_string = "HEARTBEAT";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
-               (unsigned long)signal_num);
-      return -1;
-  }
-
-  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
-                     signal_string);
-  return 0;
-}
-
-/** Called when a single local_routerstatus_t has changed: Sends an NS event
- * to any controller that cares. */
-int
-control_event_networkstatus_changed_single(const routerstatus_t *rs)
-{
-  smartlist_t *statuses;
-  int r;
-
-  if (!EVENT_IS_INTERESTING(EVENT_NS))
-    return 0;
-
-  statuses = smartlist_new();
-  smartlist_add(statuses, (void*)rs);
-  r = control_event_networkstatus_changed(statuses);
-  smartlist_free(statuses);
-  return r;
-}
-
-/** Our own router descriptor has changed; tell any controllers that care.
- */
-int
-control_event_my_descriptor_changed(void)
-{
-  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
-  return 0;
-}
-
-/** Helper: sends a status event where <b>type</b> is one of
- * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
- * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
- * string corresponding to <b>args</b>. */
-static int
-control_event_status(int type, int severity, const char *format, va_list args)
-{
-  char *user_buf = NULL;
-  char format_buf[160];
-  const char *status, *sev;
-
-  switch (type) {
-    case EVENT_STATUS_GENERAL:
-      status = "STATUS_GENERAL";
-      break;
-    case EVENT_STATUS_CLIENT:
-      status = "STATUS_CLIENT";
-      break;
-    case EVENT_STATUS_SERVER:
-      status = "STATUS_SERVER";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status type %d", type);
-      return -1;
-  }
-  switch (severity) {
-    case LOG_NOTICE:
-      sev = "NOTICE";
-      break;
-    case LOG_WARN:
-      sev = "WARN";
-      break;
-    case LOG_ERR:
-      sev = "ERR";
-      break;
-    default:
-      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
-      return -1;
-  }
-  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
-                   status, sev)<0) {
-    log_warn(LD_BUG, "Format string too long.");
-    return -1;
-  }
-  tor_vasprintf(&user_buf, format, args);
-
-  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
-  tor_free(user_buf);
-  return 0;
-}
-
-#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
-  int r;                                                        \
-  do {                                                          \
-    va_list ap;                                                 \
-    if (!EVENT_IS_INTERESTING(event))                           \
-      return 0;                                                 \
-                                                                \
-    va_start(ap, format);                                       \
-    r = control_event_status((event), (sev), format, ap);       \
-    va_end(ap);                                                 \
-  } while (0)
-
-/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_general_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_general_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_client_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_client_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
- * by formatting the arguments using the printf-style <b>format</b>. */
-int
-control_event_server_status(int severity, const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
-  return r;
-}
-
-/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
- * controller(s) immediately. */
-int
-control_event_server_error(const char *format, ...)
-{
-  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
-  /* Force a flush, since we may be about to die horribly */
-  queued_events_flush_all(1);
-  return r;
-}
-
-/** Called when the status of an entry guard with the given <b>nickname</b>
- * and identity <b>digest</b> has changed to <b>status</b>: tells any
- * controllers that care. */
-int
-control_event_guard(const char *nickname, const char *digest,
-                    const char *status)
-{
-  char hbuf[HEX_DIGEST_LEN+1];
-  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
-  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
-    return 0;
-
-  {
-    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
-    const node_t *node = node_get_by_id(digest);
-    if (node) {
-      node_get_verbose_nickname(node, buf);
-    } else {
-      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
-    }
-    send_control_event(EVENT_GUARD,
-                       "650 GUARD ENTRY %s %s\r\n", buf, status);
-  }
-  return 0;
-}
-
-/** Called when a configuration option changes. This is generally triggered
- * by SETCONF requests and RELOAD/SIGHUP signals. The <b>elements</b> is
- * a smartlist_t containing (key, value, ...) pairs in sequence.
- * <b>value</b> can be NULL. */
-int
-control_event_conf_changed(const smartlist_t *elements)
-{
-  int i;
-  char *result;
-  smartlist_t *lines;
-  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) ||
-      smartlist_len(elements) == 0) {
-    return 0;
-  }
-  lines = smartlist_new();
-  for (i = 0; i < smartlist_len(elements); i += 2) {
-    char *k = smartlist_get(elements, i);
-    char *v = smartlist_get(elements, i+1);
-    if (v == NULL) {
-      smartlist_add_asprintf(lines, "650-%s", k);
-    } else {
-      smartlist_add_asprintf(lines, "650-%s=%s", k, v);
-    }
-  }
-  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
-  send_control_event(EVENT_CONF_CHANGED,
-    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
-  tor_free(result);
-  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
-  smartlist_free(lines);
-  return 0;
-}
-
-/** Helper: Return a newly allocated string containing a path to the
- * file where we store our authentication cookie. */
-char *
-get_controller_cookie_file_name(void)
-{
-  const or_options_t *options = get_options();
-  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
-    return tor_strdup(options->CookieAuthFile);
-  } else {
-    return get_datadir_fname("control_auth_cookie");
-  }
-}
-
-/* Initialize the cookie-based authentication system of the
- * ControlPort. If <b>enabled</b> is 0, then disable the cookie
- * authentication system.  */
-int
-init_control_cookie_authentication(int enabled)
-{
-  char *fname = NULL;
-  int retval;
-
-  if (!enabled) {
-    authentication_cookie_is_set = 0;
-    return 0;
-  }
-
-  fname = get_controller_cookie_file_name();
-  retval = init_cookie_authentication(fname, "", /* no header */
-                                      AUTHENTICATION_COOKIE_LEN,
-                                   get_options()->CookieAuthFileGroupReadable,
-                                      &authentication_cookie,
-                                      &authentication_cookie_is_set);
-  tor_free(fname);
-  return retval;
-}
-
 /** A copy of the process specifier of Tor's owning controller, or
  * NULL if this Tor instance is not currently owned by a process. */
 static char *owning_controller_process_spec = NULL;
@@ -7008,895 +601,12 @@ monitor_owning_controller_process(const char *process_spec)
   }
 }
 
-/** Convert the name of a bootstrapping phase <b>s</b> into strings
- * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
-static int
-bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
-                           const char **summary)
-{
-  switch (s) {
-    case BOOTSTRAP_STATUS_UNDEF:
-      *tag = "undef";
-      *summary = "Undefined";
-      break;
-    case BOOTSTRAP_STATUS_STARTING:
-      *tag = "starting";
-      *summary = "Starting";
-      break;
-    case BOOTSTRAP_STATUS_CONN_DIR:
-      *tag = "conn_dir";
-      *summary = "Connecting to directory server";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE:
-      *tag = "status_handshake";
-      *summary = "Finishing handshake";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_DIR:
-      *tag = "handshake_dir";
-      *summary = "Finishing handshake with directory server";
-      break;
-    case BOOTSTRAP_STATUS_ONEHOP_CREATE:
-      *tag = "onehop_create";
-      *summary = "Establishing an encrypted directory connection";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_STATUS:
-      *tag = "requesting_status";
-      *summary = "Asking for networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_STATUS:
-      *tag = "loading_status";
-      *summary = "Loading networkstatus consensus";
-      break;
-    case BOOTSTRAP_STATUS_LOADING_KEYS:
-      *tag = "loading_keys";
-      *summary = "Loading authority key certs";
-      break;
-    case BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS:
-      *tag = "requesting_descriptors";
-      /* XXXX this appears to incorrectly report internal on most loads */
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Asking for relay descriptors for internal paths" :
-        "Asking for relay descriptors";
-      break;
-    /* If we're sure there are no exits in the consensus,
-     * inform the controller by adding "internal"
-     * to the status summaries.
-     * (We only check this while loading descriptors,
-     * so we may not know in the earlier stages.)
-     * But if there are exits, we can't be sure whether
-     * we're creating internal or exit paths/circuits.
-     * XXXX Or should be use different tags or statuses
-     * for internal and exit/all? */
-    case BOOTSTRAP_STATUS_LOADING_DESCRIPTORS:
-      *tag = "loading_descriptors";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Loading relay descriptors for internal paths" :
-        "Loading relay descriptors";
-      break;
-    case BOOTSTRAP_STATUS_CONN_OR:
-      *tag = "conn_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Connecting to the Tor network internally" :
-        "Connecting to the Tor network";
-      break;
-    case BOOTSTRAP_STATUS_HANDSHAKE_OR:
-      *tag = "handshake_or";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Finishing handshake with first hop of internal circuit" :
-        "Finishing handshake with first hop";
-      break;
-    case BOOTSTRAP_STATUS_CIRCUIT_CREATE:
-      *tag = "circuit_create";
-      *summary = router_have_consensus_path() == CONSENSUS_PATH_INTERNAL ?
-        "Establishing an internal Tor circuit" :
-        "Establishing a Tor circuit";
-      break;
-    case BOOTSTRAP_STATUS_DONE:
-      *tag = "done";
-      *summary = "Done";
-      break;
-    default:
-//      log_warn(LD_BUG, "Unrecognized bootstrap status code %d", s);
-      *tag = *summary = "unknown";
-      return -1;
-  }
-  return 0;
-}
-
-/** What percentage through the bootstrap process are we? We remember
- * this so we can avoid sending redundant bootstrap status events, and
- * so we can guess context for the bootstrap messages which are
- * ambiguous. It starts at 'undef', but gets set to 'starting' while
- * Tor initializes. */
-static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_percent, but only takes on the enumerated values in
- * bootstrap_status_t.
- */
-static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** As bootstrap_percent, but holds the bootstrapping level at which we last
- * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
- * to avoid flooding the log with a new message every time we get a few more
- * microdescriptors */
-static int notice_bootstrap_percent = 0;
-
-/** How many problems have we had getting to the next bootstrapping phase?
- * These include failure to establish a connection to a Tor relay,
- * failures to finish the TLS handshake, failures to validate the
- * consensus document, etc. */
-static int bootstrap_problems = 0;
-
-/** We only tell the controller once we've hit a threshold of problems
- * for the current phase. */
-#define BOOTSTRAP_PROBLEM_THRESHOLD 10
-
-/** When our bootstrapping progress level changes, but our bootstrapping
- * status has not advanced, we only log at NOTICE when we have made at least
- * this much progress.
- */
-#define BOOTSTRAP_PCT_INCREMENT 5
-
-/** Do the actual logging and notifications for
- * control_event_bootstrap().  Doesn't change any state beyond that.
- */
-static void
-control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
-                             int progress)
-{
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *tag, *summary;
-
-  bootstrap_status_to_string(status, &tag, &summary);
-  /* Locally reset status if there's incremental progress */
-  if (progress)
-    status = progress;
-
-  tor_log(loglevel, LD_CONTROL,
-          "Bootstrapped %d%%: %s", status, summary);
-  tor_snprintf(buf, sizeof(buf),
-               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
-               status, tag, summary);
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "NOTICE %s", buf);
-  control_event_client_status(LOG_NOTICE, "%s", buf);
-}
-
-/** Called when Tor has made progress at bootstrapping its directory
- * information and initial circuits.
- *
- * <b>status</b> is the new status, that is, what task we will be doing
- * next. <b>progress</b> is zero if we just started this task, else it
- * represents progress on the task.
- */
-void
-control_event_bootstrap(bootstrap_status_t status, int progress)
-{
-  int loglevel = LOG_NOTICE;
-
-  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  /* special case for handshaking status, since our TLS handshaking code
-   * can't distinguish what the connection is going to be for. */
-  if (status == BOOTSTRAP_STATUS_HANDSHAKE) {
-    if (bootstrap_percent < BOOTSTRAP_STATUS_CONN_OR) {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_DIR;
-    } else {
-      status = BOOTSTRAP_STATUS_HANDSHAKE_OR;
-    }
-  }
-
-  if (status <= bootstrap_percent) {
-    /* If there's no new progress, return early. */
-    if (!progress || progress <= bootstrap_percent)
-      return;
-    /* Log at INFO if not enough progress happened. */
-    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
-      loglevel = LOG_INFO;
-  }
-
-  control_event_bootstrap_core(loglevel, status, progress);
-
-  if (status > bootstrap_percent) {
-    bootstrap_phase = status; /* new milestone reached */
-    bootstrap_percent = status;
-  }
-  if (progress > bootstrap_percent) {
-    /* incremental progress within a milestone */
-    bootstrap_percent = progress;
-    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
-  }
-  if (loglevel == LOG_NOTICE &&
-      bootstrap_percent > notice_bootstrap_percent) {
-    /* Remember that we gave a notice at this level. */
-    notice_bootstrap_percent = bootstrap_percent;
-  }
-}
-
-/** Flag whether we've opened an OR_CONN yet  */
-static int bootstrap_first_orconn = 0;
-
-/** Like bootstrap_phase, but for (possibly deferred) directory progress */
-static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-
-/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
-static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-
-/** Defer directory info bootstrap events until we have successfully
- * completed our first connection to a router.  */
-void
-control_event_boot_dir(bootstrap_status_t status, int progress)
-{
-  if (status > bootstrap_dir_progress) {
-    bootstrap_dir_progress = status;
-    bootstrap_dir_phase = status;
-  }
-  if (progress && progress >= bootstrap_dir_progress) {
-    bootstrap_dir_progress = progress;
-  }
-
-  /* Don't report unless we have successfully opened at least one OR_CONN */
-  if (!bootstrap_first_orconn)
-    return;
-
-  control_event_bootstrap(status, progress);
-}
-
-/** Set a flag to allow reporting of directory bootstrap progress.
- * (Code that reports completion of an OR_CONN calls this.)  Also,
- * report directory progress so far. */
-void
-control_event_boot_first_orconn(void)
-{
-  bootstrap_first_orconn = 1;
-  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a human-readable hint
- * as to why, and <b>reason</b> provides a controller-facing short
- * tag.  <b>conn</b> is the connection that caused this problem and
- * can be NULL if a connection cannot be easily identified.
- */
-void
-control_event_bootstrap_problem(const char *warn, const char *reason,
-                                const connection_t *conn, int dowarn)
-{
-  int status = bootstrap_percent;
-  const char *tag = "", *summary = "";
-  char buf[BOOTSTRAP_MSG_LEN];
-  const char *recommendation = "ignore";
-  int severity;
-  char *or_id = NULL, *hostaddr = NULL;
-  or_connection_t *or_conn = NULL;
-
-  /* bootstrap_percent must not be in "undefined" state here. */
-  tor_assert(status >= 0);
-
-  if (bootstrap_percent == 100)
-    return; /* already bootstrapped; nothing to be done here. */
-
-  bootstrap_problems++;
-
-  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
-    dowarn = 1;
-
-  /* Don't warn about our bootstrapping status if we are hibernating or
-   * shutting down. */
-  if (we_are_hibernating())
-    dowarn = 0;
-
-  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
-
-  severity = dowarn ? LOG_WARN : LOG_INFO;
-
-  if (dowarn)
-    recommendation = "warn";
-
-  if (conn && conn->type == CONN_TYPE_OR) {
-    /* XXX TO_OR_CONN can't deal with const */
-    or_conn = TO_OR_CONN((connection_t *)conn);
-    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
-  } else {
-    or_id = tor_strdup("?");
-  }
-
-  if (conn)
-    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
-  else
-    hostaddr = tor_strdup("?");
-
-  log_fn(severity,
-         LD_CONTROL, "Problem bootstrapping. Stuck at %d%%: %s. (%s; %s; "
-         "count %d; recommendation %s; host %s at %s)",
-         status, summary, warn, reason,
-         bootstrap_problems, recommendation,
-         or_id, hostaddr);
-
-  connection_or_report_broken_states(severity, LD_HANDSHAKE);
-
-  tor_snprintf(buf, sizeof(buf),
-      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
-      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
-      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
-      recommendation,
-      or_id, hostaddr);
-
-  tor_snprintf(last_sent_bootstrap_message,
-               sizeof(last_sent_bootstrap_message),
-               "WARN %s", buf);
-  control_event_client_status(LOG_WARN, "%s", buf);
-
-  tor_free(hostaddr);
-  tor_free(or_id);
-}
-
-/** Called when Tor has failed to make bootstrapping progress in a way
- * that indicates a problem. <b>warn</b> gives a hint as to why, and
- * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
- * is the connection that caused this problem.
- */
-MOCK_IMPL(void,
-control_event_bootstrap_prob_or, (const char *warn, int reason,
-                                  or_connection_t *or_conn))
-{
-  int dowarn = 0;
-
-  if (or_conn->have_noted_bootstrap_problem)
-    return;
-
-  or_conn->have_noted_bootstrap_problem = 1;
-
-  if (reason == END_OR_CONN_REASON_NO_ROUTE)
-    dowarn = 1;
-
-  /* If we are using bridges and all our OR connections are now
-     closed, it means that we totally failed to connect to our
-     bridges. Throw a warning. */
-  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
-    dowarn = 1;
-
-  control_event_bootstrap_problem(warn,
-                                  orconn_end_reason_to_control_string(reason),
-                                  TO_CONN(or_conn), dowarn);
-}
-
-/** We just generated a new summary of which countries we've seen clients
- * from recently. Send a copy to the controller in case it wants to
- * display it for the user. */
-void
-control_event_clients_seen(const char *controller_str)
-{
-  send_control_event(EVENT_CLIENTS_SEEN,
-    "650 CLIENTS_SEEN %s\r\n", controller_str);
-}
-
-/** A new pluggable transport called <b>transport_name</b> was
- *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
- *  "server" or "client" depending on the mode of the pluggable
- *  transport.
- *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
- */
-void
-control_event_transport_launched(const char *mode, const char *transport_name,
-                                 tor_addr_t *addr, uint16_t port)
-{
-  send_control_event(EVENT_TRANSPORT_LAUNCHED,
-                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
-                     mode, transport_name, fmt_addr(addr), port);
-}
-
-/** Convert rendezvous auth type to string for HS_DESC control events
- */
-const char *
-rend_auth_type_to_string(rend_auth_type_t auth_type)
-{
-  const char *str;
-
-  switch (auth_type) {
-    case REND_NO_AUTH:
-      str = "NO_AUTH";
-      break;
-    case REND_BASIC_AUTH:
-      str = "BASIC_AUTH";
-      break;
-    case REND_STEALTH_AUTH:
-      str = "STEALTH_AUTH";
-      break;
-    default:
-      str = "UNKNOWN";
-  }
-
-  return str;
-}
-
-/** Return a longname the node whose identity is <b>id_digest</b>. If
- * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
- * returned instead.
- *
- * This function is not thread-safe.  Each call to this function invalidates
- * previous values returned by this function.
- */
-MOCK_IMPL(const char *,
-node_describe_longname_by_id,(const char *id_digest))
-{
-  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
-  node_get_verbose_nickname_by_id(id_digest, longname);
-  return longname;
-}
-
-/** Return either the onion address if the given pointer is a non empty
- * string else the unknown string. */
-static const char *
-rend_hsaddress_str_or_unknown(const char *onion_address)
-{
-  static const char *str_unknown = "UNKNOWN";
-  const char *str_ret = str_unknown;
-
-  /* No valid pointer, unknown it is. */
-  if (!onion_address) {
-    goto end;
-  }
-  /* Empty onion address thus we don't know, unknown it is. */
-  if (onion_address[0] == '\0') {
-    goto end;
-  }
-  /* All checks are good so return the given onion address. */
-  str_ret = onion_address;
-
- end:
-  return str_ret;
-}
-
-/** send HS_DESC requested event.
- *
- * <b>rend_query</b> is used to fetch requested onion address and auth type.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id_base32</b> is the ID of requested hs descriptor.
- * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
- */
-void
-control_event_hs_descriptor_requested(const char *onion_address,
-                                      rend_auth_type_t auth_type,
-                                      const char *id_digest,
-                                      const char *desc_id,
-                                      const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** For an HS descriptor query <b>rend_data</b>, using the
- * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
- * which descriptor ID in the query is the right one.
- *
- * Return a pointer of the binary descriptor ID found in the query's object
- * or NULL if not found. */
-static const char *
-get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
-{
-  int replica;
-  const char *desc_id = NULL;
-  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
-
-  /* Possible if the fetch was done using a descriptor ID. This means that
-   * the HSFETCH command was used. */
-  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
-    desc_id = rend_data_v2->desc_id_fetch;
-    goto end;
-  }
-
-  /* Without a directory fingerprint at this stage, we can't do much. */
-  if (hsdir_fp == NULL) {
-     goto end;
-  }
-
-  /* OK, we have an onion address so now let's find which descriptor ID
-   * is the one associated with the HSDir fingerprint. */
-  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
-       replica++) {
-    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
-
-    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
-      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
-        /* Found it! This descriptor ID is the right one. */
-        desc_id = digest;
-        goto end;
-      }
-    } SMARTLIST_FOREACH_END(fingerprint);
-  }
-
- end:
-  return desc_id;
-}
-
-/** send HS_DESC CREATED event when a local service generates a descriptor.
- *
- * <b>onion_address</b> is service address.
- * <b>desc_id</b> is the descriptor ID.
- * <b>replica</b> is the the descriptor replica number. If it is negative, it
- * is ignored.
- */
-void
-control_event_hs_descriptor_created(const char *onion_address,
-                                    const char *desc_id,
-                                    int replica)
-{
-  char *replica_field = NULL;
-
-  if (BUG(!onion_address || !desc_id)) {
-    return;
-  }
-
-  if (replica >= 0) {
-    tor_asprintf(&replica_field, " REPLICA=%d", replica);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
-                     onion_address, desc_id,
-                     replica_field ? replica_field : "");
-  tor_free(replica_field);
-}
-
-/** send HS_DESC upload event.
- *
- * <b>onion_address</b> is service address.
- * <b>hs_dir</b> is the description of contacting hs directory.
- * <b>desc_id</b> is the ID of requested hs descriptor.
- */
-void
-control_event_hs_descriptor_upload(const char *onion_address,
-                                   const char *id_digest,
-                                   const char *desc_id,
-                                   const char *hsdir_index)
-{
-  char *hsdir_index_field = NULL;
-
-  if (BUG(!onion_address || !id_digest || !desc_id)) {
-    return;
-  }
-
-  if (hsdir_index) {
-    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
-                     onion_address,
-                     node_describe_longname_by_id(id_digest),
-                     desc_id,
-                     hsdir_index_field ? hsdir_index_field : "");
-  tor_free(hsdir_index_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hsv2_descriptor_received
- * control_event_hsv2_descriptor_failed
- * control_event_hsv3_descriptor_failed
- *
- * So do not call this function directly.
- */
-static void
-event_hs_descriptor_receive_end(const char *action,
-                                const char *onion_address,
-                                const char *desc_id,
-                                rend_auth_type_t auth_type,
-                                const char *hsdir_id_digest,
-                                const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !onion_address)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s %s %s%s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     rend_auth_type_to_string(auth_type),
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     desc_id ? desc_id : "",
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC event after got response from hs directory.
- *
- * NOTE: this is an internal function used by following functions:
- * control_event_hs_descriptor_uploaded
- * control_event_hs_descriptor_upload_failed
- *
- * So do not call this function directly.
- */
-void
-control_event_hs_descriptor_upload_end(const char *action,
-                                       const char *onion_address,
-                                       const char *id_digest,
-                                       const char *reason)
-{
-  char *reason_field = NULL;
-
-  if (BUG(!action || !id_digest)) {
-    return;
-  }
-
-  if (reason) {
-    tor_asprintf(&reason_field, " REASON=%s", reason);
-  }
-
-  send_control_event(EVENT_HS_DESC,
-                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
-                     action,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     node_describe_longname_by_id(id_digest),
-                     reason_field ? reason_field : "");
-
-  tor_free(reason_field);
-}
-
-/** send HS_DESC RECEIVED event
- *
- * called when we successfully received a hidden service descriptor.
- */
-void
-control_event_hsv2_descriptor_received(const char *onion_address,
-                                       const rend_data_t *rend_data,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/* Send HS_DESC RECEIVED event
- *
- * Called when we successfully received a hidden service descriptor. */
-void
-control_event_hsv3_descriptor_received(const char *onion_address,
-                                       const char *desc_id,
-                                       const char *hsdir_id_digest)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, NULL);
-  tor_free(desc_id_field);
-}
-
-/** send HS_DESC UPLOADED event
- *
- * called when we successfully uploaded a hidden service descriptor.
- */
-void
-control_event_hs_descriptor_uploaded(const char *id_digest,
-                                     const char *onion_address)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-
-  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
-                                         id_digest, NULL);
-}
-
-/** Send HS_DESC event to inform controller that query <b>rend_data</b>
- * failed to retrieve hidden service descriptor from directory identified by
- * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
- * add it to REASON= field.
- */
-void
-control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-  const char *desc_id;
-
-  if (BUG(!rend_data)) {
-    return;
-  }
-
-  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
-  if (desc_id != NULL) {
-    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
-    /* Set the descriptor ID digest to base32 so we can send it. */
-    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
-                  DIGEST_LEN);
-    /* Extra whitespace is needed before the value. */
-    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
-  }
-
-  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
-                                  desc_id_field,
-                                  TO_REND_DATA_V2(rend_data)->auth_type,
-                                  hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC event to inform controller that the query to
- * <b>onion_address</b> failed to retrieve hidden service descriptor
- * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
- * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
- * field. */
-void
-control_event_hsv3_descriptor_failed(const char *onion_address,
-                                     const char *desc_id,
-                                     const char *hsdir_id_digest,
-                                     const char *reason)
-{
-  char *desc_id_field = NULL;
-
-  if (BUG(!onion_address || !desc_id || !reason)) {
-    return;
-  }
-
-  /* Because DescriptorID is an optional positional value, we need to add a
-   * whitespace before in order to not be next to the HsDir value. */
-  tor_asprintf(&desc_id_field, " %s", desc_id);
-
-  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
-                                  REND_NO_AUTH, hsdir_id_digest, reason);
-  tor_free(desc_id_field);
-}
-
-/** Send HS_DESC_CONTENT event after completion of a successful fetch
- * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
- * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
- * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
- * not trigger the control event. */
-void
-control_event_hs_descriptor_content(const char *onion_address,
-                                    const char *desc_id,
-                                    const char *hsdir_id_digest,
-                                    const char *content)
-{
-  static const char *event_name = "HS_DESC_CONTENT";
-  char *esc_content = NULL;
-
-  if (!onion_address || !desc_id) {
-    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
-             onion_address, desc_id);
-    return;
-  }
-
-  if (content == NULL) {
-    /* Point it to empty content so it can still be escaped. */
-    content = "";
-  }
-  write_escaped_data(content, strlen(content), &esc_content);
-
-  send_control_event(EVENT_HS_DESC_CONTENT,
-                     "650+%s %s %s %s\r\n%s650 OK\r\n",
-                     event_name,
-                     rend_hsaddress_str_or_unknown(onion_address),
-                     desc_id,
-                     hsdir_id_digest ?
-                        node_describe_longname_by_id(hsdir_id_digest) :
-                        "UNKNOWN",
-                     esc_content);
-  tor_free(esc_content);
-}
-
-/** Send HS_DESC event to inform controller upload of hidden service
- * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
- * is not NULL, add it to REASON= field.
- */
-void
-control_event_hs_descriptor_upload_failed(const char *id_digest,
-                                          const char *onion_address,
-                                          const char *reason)
-{
-  if (BUG(!id_digest)) {
-    return;
-  }
-  control_event_hs_descriptor_upload_end("FAILED", onion_address,
-                                         id_digest, reason);
-}
-
 /** Free any leftover allocated memory of the control.c subsystem. */
 void
 control_free_all(void)
 {
-  smartlist_t *queued_events = NULL;
-
-  stats_prev_n_read = stats_prev_n_written = 0;
-
-  if (authentication_cookie) /* Free the auth cookie */
-    tor_free(authentication_cookie);
-  if (detached_onion_services) { /* Free the detached onion services */
-    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
-    smartlist_free(detached_onion_services);
-  }
-
-  if (queued_control_events_lock) {
-    tor_mutex_acquire(queued_control_events_lock);
-    flush_queued_event_pending = 0;
-    queued_events = queued_control_events;
-    queued_control_events = NULL;
-    tor_mutex_release(queued_control_events_lock);
-  }
-  if (queued_events) {
-    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
-                      queued_event_free(ev));
-    smartlist_free(queued_events);
-  }
-  if (flush_queued_events_event) {
-    mainloop_event_free(flush_queued_events_event);
-    flush_queued_events_event = NULL;
-  }
-  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
-  notice_bootstrap_percent = 0;
-  bootstrap_problems = 0;
-  bootstrap_first_orconn = 0;
-  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
-  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
-  authentication_cookie_is_set = 0;
-  global_event_mask = 0;
-  disable_log_messages = 0;
-  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
-}
-
-#ifdef TOR_UNIT_TESTS
-/* For testing: change the value of global_event_mask */
-void
-control_testing_set_global_event_mask(uint64_t mask)
-{
-  global_event_mask = mask;
+  control_auth_free_all();
+  control_events_free_all();
+  control_cmd_free_all();
+  control_event_bootstrap_reset();
 }
-#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control.h b/src/feature/control/control.h
index a09c1cd11b..7e72b2736b 100644
--- a/src/feature/control/control.h
+++ b/src/feature/control/control.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,76 +12,6 @@
 #ifndef TOR_CONTROL_H
 #define TOR_CONTROL_H
 
-/** Used to indicate the type of a circuit event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum circuit_status_event_t {
-  CIRC_EVENT_LAUNCHED = 0,
-  CIRC_EVENT_BUILT    = 1,
-  CIRC_EVENT_EXTENDED = 2,
-  CIRC_EVENT_FAILED   = 3,
-  CIRC_EVENT_CLOSED   = 4,
-} circuit_status_event_t;
-
-/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
- * The various types are defined in control-spec.txt . */
-typedef enum circuit_status_minor_event_t {
-  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
-  CIRC_MINOR_EVENT_CANNIBALIZED,
-} circuit_status_minor_event_t;
-
-/** Used to indicate the type of a stream event passed to the controller.
- * The various types are defined in control-spec.txt */
-typedef enum stream_status_event_t {
-  STREAM_EVENT_SENT_CONNECT = 0,
-  STREAM_EVENT_SENT_RESOLVE = 1,
-  STREAM_EVENT_SUCCEEDED    = 2,
-  STREAM_EVENT_FAILED       = 3,
-  STREAM_EVENT_CLOSED       = 4,
-  STREAM_EVENT_NEW          = 5,
-  STREAM_EVENT_NEW_RESOLVE  = 6,
-  STREAM_EVENT_FAILED_RETRIABLE = 7,
-  STREAM_EVENT_REMAP        = 8
-} stream_status_event_t;
-
-/** Used to indicate the type of an OR connection event passed to the
- * controller.  The various types are defined in control-spec.txt */
-typedef enum or_conn_status_event_t {
-  OR_CONN_EVENT_LAUNCHED     = 0,
-  OR_CONN_EVENT_CONNECTED    = 1,
-  OR_CONN_EVENT_FAILED       = 2,
-  OR_CONN_EVENT_CLOSED       = 3,
-  OR_CONN_EVENT_NEW          = 4,
-} or_conn_status_event_t;
-
-/** Used to indicate the type of a buildtime event */
-typedef enum buildtimeout_set_event_t {
-  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
-  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
-  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
-  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
-  BUILDTIMEOUT_SET_EVENT_RESUME = 4
-} buildtimeout_set_event_t;
-
-/** Enum describing various stages of bootstrapping, for use with controller
- * bootstrap status events. The values range from 0 to 100. */
-typedef enum {
-  BOOTSTRAP_STATUS_UNDEF=-1,
-  BOOTSTRAP_STATUS_STARTING=0,
-  BOOTSTRAP_STATUS_CONN_DIR=5,
-  BOOTSTRAP_STATUS_HANDSHAKE=-2,
-  BOOTSTRAP_STATUS_HANDSHAKE_DIR=10,
-  BOOTSTRAP_STATUS_ONEHOP_CREATE=15,
-  BOOTSTRAP_STATUS_REQUESTING_STATUS=20,
-  BOOTSTRAP_STATUS_LOADING_STATUS=25,
-  BOOTSTRAP_STATUS_LOADING_KEYS=40,
-  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
-  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
-  BOOTSTRAP_STATUS_CONN_OR=80,
-  BOOTSTRAP_STATUS_HANDSHAKE_OR=85,
-  BOOTSTRAP_STATUS_CIRCUIT_CREATE=90,
-  BOOTSTRAP_STATUS_DONE=100
-} bootstrap_status_t;
-
 control_connection_t *TO_CONTROL_CONN(connection_t *);
 
 #define CONTROL_CONN_STATE_MIN_ 1
@@ -92,18 +22,6 @@ control_connection_t *TO_CONTROL_CONN(connection_t *);
 #define CONTROL_CONN_STATE_NEEDAUTH 2
 #define CONTROL_CONN_STATE_MAX_ 2
 
-/** Reason for remapping an AP connection's address: we have a cached
- * answer. */
-#define REMAP_STREAM_SOURCE_CACHE 1
-/** Reason for remapping an AP connection's address: the exit node told us an
- * answer. */
-#define REMAP_STREAM_SOURCE_EXIT 2
-
-void control_initialize_event_queue(void);
-
-void control_update_global_event_mask(void);
-void control_adjust_event_log_severity(void);
-
 void control_ports_write_to_file(void);
 
 /** Log information about the connection <b>conn</b>, protecting it as with
@@ -124,294 +42,28 @@ void connection_control_closed(control_connection_t *conn);
 
 int connection_control_process_inbuf(control_connection_t *conn);
 
-#define EVENT_NS 0x000F
-int control_event_is_interesting(int event);
-
-void control_per_second_events(void);
-int control_any_per_second_event_enabled(void);
-
-int control_event_circuit_status(origin_circuit_t *circ,
-                                 circuit_status_event_t e, int reason);
-int control_event_circuit_purpose_changed(origin_circuit_t *circ,
-                                          int old_purpose);
-int control_event_circuit_cannibalized(origin_circuit_t *circ,
-                                       int old_purpose,
-                                       const struct timeval *old_tv_created);
-int control_event_stream_status(entry_connection_t *conn,
-                                stream_status_event_t e,
-                                int reason);
-int control_event_or_conn_status(or_connection_t *conn,
-                                 or_conn_status_event_t e, int reason);
-int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
-int control_event_stream_bandwidth(edge_connection_t *edge_conn);
-int control_event_stream_bandwidth_used(void);
-int control_event_circ_bandwidth_used(void);
-int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
-int control_event_conn_bandwidth(connection_t *conn);
-int control_event_conn_bandwidth_used(void);
-int control_event_circuit_cell_stats(void);
-void control_event_logmsg(int severity, uint32_t domain, const char *msg);
-void control_event_logmsg_pending(void);
-int control_event_descriptors_changed(smartlist_t *routers);
-int control_event_address_mapped(const char *from, const char *to,
-                                 time_t expires, const char *error,
-                                 const int cached);
-int control_event_my_descriptor_changed(void);
-int control_event_network_liveness_update(int liveness);
-int control_event_networkstatus_changed(smartlist_t *statuses);
-
-int control_event_newconsensus(const networkstatus_t *consensus);
-int control_event_networkstatus_changed_single(const routerstatus_t *rs);
-int control_event_general_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_client_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-int control_event_server_status(int severity, const char *format, ...)
-  CHECK_PRINTF(2,3);
-
-int control_event_general_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_client_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-int control_event_server_error(const char *format, ...)
-  CHECK_PRINTF(1,2);
-
-int control_event_guard(const char *nickname, const char *digest,
-                        const char *status);
-int control_event_conf_changed(const smartlist_t *elements);
-int control_event_buildtimeout_set(buildtimeout_set_event_t type,
-                                   const char *args);
-int control_event_signal(uintptr_t signal);
-
-int init_control_cookie_authentication(int enabled);
-char *get_controller_cookie_file_name(void);
-struct config_line_t;
-smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
 void disable_control_logging(void);
 void enable_control_logging(void);
 
 void monitor_owning_controller_process(const char *process_spec);
 
-void control_event_bootstrap(bootstrap_status_t status, int progress);
-MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
-                                                 int reason,
-                                                 or_connection_t *or_conn));
-void control_event_boot_dir(bootstrap_status_t status, int progress);
-void control_event_boot_first_orconn(void);
-void control_event_bootstrap_problem(const char *warn, const char *reason,
-                                     const connection_t *conn, int dowarn);
-
-void control_event_clients_seen(const char *controller_str);
-void control_event_transport_launched(const char *mode,
-                                      const char *transport_name,
-                                      tor_addr_t *addr, uint16_t port);
 const char *rend_auth_type_to_string(rend_auth_type_t auth_type);
-MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
-void control_event_hs_descriptor_requested(const char *onion_address,
-                                           rend_auth_type_t auth_type,
-                                           const char *id_digest,
-                                           const char *desc_id,
-                                           const char *hsdir_index);
-void control_event_hs_descriptor_created(const char *onion_address,
-                                         const char *desc_id,
-                                         int replica);
-void control_event_hs_descriptor_upload(const char *onion_address,
-                                        const char *desc_id,
-                                        const char *hs_dir,
-                                        const char *hsdir_index);
-void control_event_hs_descriptor_upload_end(const char *action,
-                                            const char *onion_address,
-                                            const char *hs_dir,
-                                            const char *reason);
-void control_event_hs_descriptor_uploaded(const char *hs_dir,
-                                          const char *onion_address);
-/* Hidden service v2 HS_DESC specific. */
-void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
-                                          const char *id_digest,
-                                          const char *reason);
-void control_event_hsv2_descriptor_received(const char *onion_address,
-                                            const rend_data_t *rend_data,
-                                            const char *id_digest);
-/* Hidden service v3 HS_DESC specific. */
-void control_event_hsv3_descriptor_failed(const char *onion_address,
-                                          const char *desc_id,
-                                          const char *hsdir_id_digest,
-                                          const char *reason);
-void control_event_hsv3_descriptor_received(const char *onion_address,
-                                            const char *desc_id,
-                                            const char *hsdir_id_digest);
-void control_event_hs_descriptor_upload_failed(const char *hs_dir,
-                                               const char *onion_address,
-                                               const char *reason);
-void control_event_hs_descriptor_content(const char *onion_address,
-                                         const char *desc_id,
-                                         const char *hsdir_fp,
-                                         const char *content);
 void control_free_all(void);
 
-#ifdef CONTROL_PRIVATE
-#include "lib/crypt_ops/crypto_ed25519.h"
-
-/* Recognized asynchronous event types.  It's okay to expand this list
- * because it is used both as a list of v0 event types, and as indices
- * into the bitfield to determine which controllers want which events.
- */
-/* This bitfield has no event zero    0x0000 */
-#define EVENT_MIN_                    0x0001
-#define EVENT_CIRCUIT_STATUS          0x0001
-#define EVENT_STREAM_STATUS           0x0002
-#define EVENT_OR_CONN_STATUS          0x0003
-#define EVENT_BANDWIDTH_USED          0x0004
-#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
-#define EVENT_NEW_DESC                0x0006
-#define EVENT_DEBUG_MSG               0x0007
-#define EVENT_INFO_MSG                0x0008
-#define EVENT_NOTICE_MSG              0x0009
-#define EVENT_WARN_MSG                0x000A
-#define EVENT_ERR_MSG                 0x000B
-#define EVENT_ADDRMAP                 0x000C
-/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
-   can reclaim 0x000D. */
-#define EVENT_DESCCHANGED             0x000E
-/* Exposed above */
-// #define EVENT_NS                   0x000F
-#define EVENT_STATUS_CLIENT           0x0010
-#define EVENT_STATUS_SERVER           0x0011
-#define EVENT_STATUS_GENERAL          0x0012
-#define EVENT_GUARD                   0x0013
-#define EVENT_STREAM_BANDWIDTH_USED   0x0014
-#define EVENT_CLIENTS_SEEN            0x0015
-#define EVENT_NEWCONSENSUS            0x0016
-#define EVENT_BUILDTIMEOUT_SET        0x0017
-#define EVENT_GOT_SIGNAL              0x0018
-#define EVENT_CONF_CHANGED            0x0019
-#define EVENT_CONN_BW                 0x001A
-#define EVENT_CELL_STATS              0x001B
-/* UNUSED :                           0x001C */
-#define EVENT_CIRC_BANDWIDTH_USED     0x001D
-#define EVENT_TRANSPORT_LAUNCHED      0x0020
-#define EVENT_HS_DESC                 0x0021
-#define EVENT_HS_DESC_CONTENT         0x0022
-#define EVENT_NETWORK_LIVENESS        0x0023
-#define EVENT_MAX_                    0x0023
+#ifdef CONTROL_MODULE_PRIVATE
+struct signal_name_t {
+  int sig;
+  const char *signal_name;
+};
+extern const struct signal_name_t signal_table[];
+int get_cached_network_liveness(void);
+void set_cached_network_liveness(int liveness);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
 
-/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
-#define EVENT_CAPACITY_               0x0040
-
-/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
- * different structure, as it can only handle a maximum left shift of 1<<63. */
-
-#if EVENT_MAX_ >= EVENT_CAPACITY_
-#error control_connection_t.event_mask has an event greater than its capacity
+#ifdef CONTROL_PRIVATE
+STATIC char *control_split_incoming_command(char *incoming_cmd,
+                                            size_t *data_len,
+                                            char **current_cmd_out);
 #endif
 
-#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
-
-#define EVENT_MASK_NONE_             ((uint64_t)0x0)
-
-#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
-#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
-                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
-                                          - EVENT_MIN_))
-
-#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
-                                      & EVENT_MASK_BELOW_MAX_)
-
-/* Used only by control.c and test.c */
-STATIC size_t write_escaped_data(const char *data, size_t len, char **out);
-STATIC size_t read_escaped_data(const char *data, size_t len, char **out);
-
-#ifdef TOR_UNIT_TESTS
-MOCK_DECL(STATIC void,
-          send_control_event_string,(uint16_t event, const char *msg));
-
-MOCK_DECL(STATIC void,
-          queue_control_event_string,(uint16_t event, char *msg));
-
-void control_testing_set_global_event_mask(uint64_t mask);
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/** Helper structure: temporarily stores cell statistics for a circuit. */
-typedef struct cell_stats_t {
-  /** Number of cells added in app-ward direction by command. */
-  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells added in exit-ward direction by command. */
-  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in app-ward direction by command. */
-  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
-  /** Number of cells removed in exit-ward direction by command. */
-  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in app-ward direction by command. */
-  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
-  /** Total waiting time of cells in exit-ward direction by command. */
-  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
-} cell_stats_t;
-void sum_up_cell_stats_by_command(circuit_t *circ,
-                                  cell_stats_t *cell_stats);
-void append_cell_stats_by_command(smartlist_t *event_parts,
-                                  const char *key,
-                                  const uint64_t *include_if_non_zero,
-                                  const uint64_t *number_to_include);
-void format_cell_stats(char **event_string, circuit_t *circ,
-                       cell_stats_t *cell_stats);
-STATIC char *get_bw_samples(void);
-
-/* ADD_ONION secret key to create an ephemeral service. The command supports
- * multiple versions so this union stores the key and passes it to the HS
- * subsystem depending on the requested version. */
-typedef union add_onion_secret_key_t {
-  /* Hidden service v2 secret key. */
-  crypto_pk_t *v2;
-  /* Hidden service v3 secret key. */
-  ed25519_secret_key_t *v3;
-} add_onion_secret_key_t;
-
-STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
-                                   const char **key_new_alg_out,
-                                   char **key_new_blob_out,
-                                   add_onion_secret_key_t *decoded_key,
-                                   int *hs_version, char **err_msg_out);
-
-STATIC rend_authorized_client_t *
-add_onion_helper_clientauth(const char *arg, int *created, char **err_msg_out);
-
-STATIC int getinfo_helper_onions(
-    control_connection_t *control_conn,
-    const char *question,
-    char **answer,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_networkstatus(
-    const char *flavor,
-    download_status_t **dl_to_emit,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_cert(
-    const char *fp_sk_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_desc(
-    const char *desc_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC void getinfo_helper_downloads_bridge(
-    const char *bridge_req,
-    download_status_t **dl_to_emit,
-    smartlist_t **digest_list,
-    const char **errmsg);
-STATIC int getinfo_helper_downloads(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_dir(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-STATIC int getinfo_helper_current_time(
-    control_connection_t *control_conn,
-    const char *question, char **answer,
-    const char **errmsg);
-
-#endif /* defined(CONTROL_PRIVATE) */
-
 #endif /* !defined(TOR_CONTROL_H) */
diff --git a/src/feature/control/control_auth.c b/src/feature/control/control_auth.c
new file mode 100644
index 0000000000..b60623ab5c
--- /dev/null
+++ b/src/feature/control/control_auth.c
@@ -0,0 +1,441 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.c
+ * \brief Authentication for Tor's control-socket interface.
+ **/
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_proto.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+
+#include "lib/crypt_ops/crypto_s2k.h"
+
+/** If we're using cookie-type authentication, how long should our cookies be?
+ */
+#define AUTHENTICATION_COOKIE_LEN 32
+
+/** If true, we've set authentication_cookie to a secret code and
+ * stored it to disk. */
+static int authentication_cookie_is_set = 0;
+/** If authentication_cookie_is_set, a secret cookie that we've stored to disk
+ * and which we're using to authenticate controllers.  (If the controller can
+ * read it off disk, it has permission to connect.) */
+static uint8_t *authentication_cookie = NULL;
+
+#define SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT \
+  "Tor safe cookie authentication server-to-controller hash"
+#define SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT \
+  "Tor safe cookie authentication controller-to-server hash"
+#define SAFECOOKIE_SERVER_NONCE_LEN DIGEST256_LEN
+
+/** Helper: Return a newly allocated string containing a path to the
+ * file where we store our authentication cookie. */
+char *
+get_controller_cookie_file_name(void)
+{
+  const or_options_t *options = get_options();
+  if (options->CookieAuthFile && strlen(options->CookieAuthFile)) {
+    return tor_strdup(options->CookieAuthFile);
+  } else {
+    return get_datadir_fname("control_auth_cookie");
+  }
+}
+
+/* Initialize the cookie-based authentication system of the
+ * ControlPort. If <b>enabled</b> is 0, then disable the cookie
+ * authentication system.  */
+int
+init_control_cookie_authentication(int enabled)
+{
+  char *fname = NULL;
+  int retval;
+
+  if (!enabled) {
+    authentication_cookie_is_set = 0;
+    return 0;
+  }
+
+  fname = get_controller_cookie_file_name();
+  retval = init_cookie_authentication(fname, "", /* no header */
+                                      AUTHENTICATION_COOKIE_LEN,
+                                   get_options()->CookieAuthFileGroupReadable,
+                                      &authentication_cookie,
+                                      &authentication_cookie_is_set);
+  tor_free(fname);
+  return retval;
+}
+
+/** Decode the hashed, base64'd passwords stored in <b>passwords</b>.
+ * Return a smartlist of acceptable passwords (unterminated strings of
+ * length S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) on success, or NULL on
+ * failure.
+ */
+smartlist_t *
+decode_hashed_passwords(config_line_t *passwords)
+{
+  char decoded[64];
+  config_line_t *cl;
+  smartlist_t *sl = smartlist_new();
+
+  tor_assert(passwords);
+
+  for (cl = passwords; cl; cl = cl->next) {
+    const char *hashed = cl->value;
+
+    if (!strcmpstart(hashed, "16:")) {
+      if (base16_decode(decoded, sizeof(decoded), hashed+3, strlen(hashed+3))
+                        != S2K_RFC2440_SPECIFIER_LEN + DIGEST_LEN
+          || strlen(hashed+3) != (S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN)*2) {
+        goto err;
+      }
+    } else {
+        if (base64_decode(decoded, sizeof(decoded), hashed, strlen(hashed))
+            != S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN) {
+          goto err;
+        }
+    }
+    smartlist_add(sl,
+                  tor_memdup(decoded, S2K_RFC2440_SPECIFIER_LEN+DIGEST_LEN));
+  }
+
+  return sl;
+
+ err:
+  SMARTLIST_FOREACH(sl, char*, cp, tor_free(cp));
+  smartlist_free(sl);
+  return NULL;
+}
+
+const control_cmd_syntax_t authchallenge_syntax = {
+   .min_args = 1,
+   .max_args = 1,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHCHALLENGE command. */
+int
+handle_control_authchallenge(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  char *client_nonce;
+  size_t client_nonce_len;
+  char server_hash[DIGEST256_LEN];
+  char server_hash_encoded[HEX_DIGEST256_LEN+1];
+  char server_nonce[SAFECOOKIE_SERVER_NONCE_LEN];
+  char server_nonce_encoded[(2*SAFECOOKIE_SERVER_NONCE_LEN) + 1];
+
+  if (strcasecmp(smartlist_get(args->args, 0), "SAFECOOKIE")) {
+    control_write_endreply(conn, 513,
+                           "AUTHCHALLENGE only supports SAFECOOKIE "
+                           "authentication");
+    goto fail;
+  }
+  if (!authentication_cookie_is_set) {
+    control_write_endreply(conn, 515, "Cookie authentication is disabled");
+    goto fail;
+  }
+  if (args->kwargs == NULL || args->kwargs->next != NULL) {
+    control_write_endreply(conn, 512,
+                           "Wrong number of arguments for AUTHCHALLENGE");
+    goto fail;
+  }
+  if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHCHALLENGE does not accept keyword "
+                           "arguments.");
+    goto fail;
+  }
+
+  bool contains_quote = strchr(args->raw_body, '\"');
+  if (contains_quote) {
+    /* The nonce was quoted */
+    client_nonce = tor_strdup(args->kwargs->value);
+    client_nonce_len = strlen(client_nonce);
+  } else {
+    /* The nonce was should be in hex. */
+    const char *hex_nonce = args->kwargs->value;
+    client_nonce_len = strlen(hex_nonce) / 2;
+    client_nonce = tor_malloc(client_nonce_len);
+    if (base16_decode(client_nonce, client_nonce_len, hex_nonce,
+                      strlen(hex_nonce)) != (int)client_nonce_len) {
+      control_write_endreply(conn, 513, "Invalid base16 client nonce");
+      tor_free(client_nonce);
+      goto fail;
+    }
+  }
+
+  crypto_rand(server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+  /* Now compute and send the server-to-controller response, and the
+   * server's nonce. */
+  tor_assert(authentication_cookie != NULL);
+
+  {
+    size_t tmp_len = (AUTHENTICATION_COOKIE_LEN +
+                      client_nonce_len +
+                      SAFECOOKIE_SERVER_NONCE_LEN);
+    char *tmp = tor_malloc_zero(tmp_len);
+    char *client_hash = tor_malloc_zero(DIGEST256_LEN);
+    memcpy(tmp, authentication_cookie, AUTHENTICATION_COOKIE_LEN);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN, client_nonce, client_nonce_len);
+    memcpy(tmp + AUTHENTICATION_COOKIE_LEN + client_nonce_len,
+           server_nonce, SAFECOOKIE_SERVER_NONCE_LEN);
+
+    crypto_hmac_sha256(server_hash,
+                       SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT,
+                       strlen(SAFECOOKIE_SERVER_TO_CONTROLLER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    crypto_hmac_sha256(client_hash,
+                       SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT,
+                       strlen(SAFECOOKIE_CONTROLLER_TO_SERVER_CONSTANT),
+                       tmp,
+                       tmp_len);
+
+    conn->safecookie_client_hash = client_hash;
+
+    tor_free(tmp);
+  }
+
+  base16_encode(server_hash_encoded, sizeof(server_hash_encoded),
+                server_hash, sizeof(server_hash));
+  base16_encode(server_nonce_encoded, sizeof(server_nonce_encoded),
+                server_nonce, sizeof(server_nonce));
+
+  control_printf_endreply(conn, 250,
+                          "AUTHCHALLENGE SERVERHASH=%s SERVERNONCE=%s",
+                          server_hash_encoded,
+                          server_nonce_encoded);
+
+  tor_free(client_nonce);
+  return 0;
+ fail:
+  connection_mark_for_close(TO_CONN(conn));
+  return -1;
+}
+
+const control_cmd_syntax_t authenticate_syntax = {
+   .max_args = 0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_KEYS|KV_QUOTED_QSTRING,
+   .store_raw_body=true
+};
+
+/** Called when we get an AUTHENTICATE message.  Check whether the
+ * authentication is valid, and if so, update the connection's state to
+ * OPEN.  Reply with DONE or ERROR.
+ */
+int
+handle_control_authenticate(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  bool used_quoted_string = false;
+  const or_options_t *options = get_options();
+  const char *errstr = "Unknown error";
+  char *password;
+  size_t password_len;
+  int bad_cookie=0, bad_password=0;
+  smartlist_t *sl = NULL;
+
+  if (args->kwargs == NULL) {
+    password = tor_strdup("");
+    password_len = 0;
+  } else if (args->kwargs->next) {
+    control_write_endreply(conn, 512, "Too many arguments to AUTHENTICATE.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strcmp(args->kwargs->key, "")) {
+    control_write_endreply(conn, 512,
+                           "AUTHENTICATE does not accept keyword arguments.");
+    connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  } else if (strchr(args->raw_body, '\"')) {
+    used_quoted_string = true;
+    password = tor_strdup(args->kwargs->value);
+    password_len = strlen(password);
+  } else {
+    const char *hex_passwd = args->kwargs->value;
+    password_len = strlen(hex_passwd) / 2;
+    password = tor_malloc(password_len+1);
+    if (base16_decode(password, password_len+1, hex_passwd, strlen(hex_passwd))
+                      != (int) password_len) {
+      control_write_endreply(conn, 551,
+            "Invalid hexadecimal encoding.  Maybe you tried a plain text "
+            "password?  If so, the standard requires that you put it in "
+            "double quotes.");
+      connection_mark_for_close(TO_CONN(conn));
+      tor_free(password);
+      return 0;
+    }
+  }
+
+  if (conn->safecookie_client_hash != NULL) {
+    /* The controller has chosen safe cookie authentication; the only
+     * acceptable authentication value is the controller-to-server
+     * response. */
+
+    tor_assert(authentication_cookie_is_set);
+
+    if (password_len != DIGEST256_LEN) {
+      log_warn(LD_CONTROL,
+               "Got safe cookie authentication response with wrong length "
+               "(%d)", (int)password_len);
+      errstr = "Wrong length for safe cookie response.";
+      goto err;
+    }
+
+    if (tor_memneq(conn->safecookie_client_hash, password, DIGEST256_LEN)) {
+      log_warn(LD_CONTROL,
+               "Got incorrect safe cookie authentication response");
+      errstr = "Safe cookie response did not match expected value.";
+      goto err;
+    }
+
+    tor_free(conn->safecookie_client_hash);
+    goto ok;
+  }
+
+  if (!options->CookieAuthentication && !options->HashedControlPassword &&
+      !options->HashedControlSessionPassword) {
+    /* if Tor doesn't demand any stronger authentication, then
+     * the controller can get in with anything. */
+    goto ok;
+  }
+
+  if (options->CookieAuthentication) {
+    int also_password = options->HashedControlPassword != NULL ||
+      options->HashedControlSessionPassword != NULL;
+    if (password_len != AUTHENTICATION_COOKIE_LEN) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got authentication cookie with wrong length "
+                 "(%d)", (int)password_len);
+        errstr = "Wrong length on authentication cookie.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else if (tor_memneq(authentication_cookie, password, password_len)) {
+      if (!also_password) {
+        log_warn(LD_CONTROL, "Got mismatched authentication cookie");
+        errstr = "Authentication cookie did not match expected value.";
+        goto err;
+      }
+      bad_cookie = 1;
+    } else {
+      goto ok;
+    }
+  }
+
+  if (options->HashedControlPassword ||
+      options->HashedControlSessionPassword) {
+    int bad = 0;
+    smartlist_t *sl_tmp;
+    char received[DIGEST_LEN];
+    int also_cookie = options->CookieAuthentication;
+    sl = smartlist_new();
+    if (options->HashedControlPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (options->HashedControlSessionPassword) {
+      sl_tmp = decode_hashed_passwords(options->HashedControlSessionPassword);
+      if (!sl_tmp)
+        bad = 1;
+      else {
+        smartlist_add_all(sl, sl_tmp);
+        smartlist_free(sl_tmp);
+      }
+    }
+    if (bad) {
+      if (!also_cookie) {
+        log_warn(LD_BUG,
+                 "Couldn't decode HashedControlPassword: invalid base16");
+        errstr="Couldn't decode HashedControlPassword value in configuration.";
+        goto err;
+      }
+      bad_password = 1;
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+    } else {
+      SMARTLIST_FOREACH(sl, char *, expected,
+      {
+        secret_to_key_rfc2440(received,DIGEST_LEN,
+                              password,password_len,expected);
+        if (tor_memeq(expected + S2K_RFC2440_SPECIFIER_LEN,
+                      received, DIGEST_LEN))
+          goto ok;
+      });
+      SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+      smartlist_free(sl);
+      sl = NULL;
+
+      if (used_quoted_string)
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration";
+      else
+        errstr = "Password did not match HashedControlPassword value from "
+          "configuration. Maybe you tried a plain text password? "
+          "If so, the standard requires that you put it in double quotes.";
+      bad_password = 1;
+      if (!also_cookie)
+        goto err;
+    }
+  }
+
+  /** We only get here if both kinds of authentication failed. */
+  tor_assert(bad_password && bad_cookie);
+  log_warn(LD_CONTROL, "Bad password or authentication cookie on controller.");
+  errstr = "Password did not match HashedControlPassword *or* authentication "
+    "cookie.";
+
+ err:
+  tor_free(password);
+  control_printf_endreply(conn, 515, "Authentication failed: %s", errstr);
+  connection_mark_for_close(TO_CONN(conn));
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+ ok:
+  log_info(LD_CONTROL, "Authenticated control connection ("TOR_SOCKET_T_FORMAT
+           ")", conn->base_.s);
+  send_control_done(conn);
+  conn->base_.state = CONTROL_CONN_STATE_OPEN;
+  tor_free(password);
+  if (sl) { /* clean up */
+    SMARTLIST_FOREACH(sl, char *, str, tor_free(str));
+    smartlist_free(sl);
+  }
+  return 0;
+}
+
+void
+control_auth_free_all(void)
+{
+  if (authentication_cookie) /* Free the auth cookie */
+    tor_free(authentication_cookie);
+  authentication_cookie_is_set = 0;
+}
diff --git a/src/feature/control/control_auth.h b/src/feature/control/control_auth.h
new file mode 100644
index 0000000000..d4c1dd78a7
--- /dev/null
+++ b/src/feature/control/control_auth.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_auth.h
+ * \brief Header file for control_auth.c.
+ **/
+
+#ifndef TOR_CONTROL_AUTH_H
+#define TOR_CONTROL_AUTH_H
+
+struct control_cmd_args_t;
+struct control_cmd_syntax_t;
+
+int init_control_cookie_authentication(int enabled);
+char *get_controller_cookie_file_name(void);
+struct config_line_t;
+smartlist_t *decode_hashed_passwords(struct config_line_t *passwords);
+
+int handle_control_authchallenge(control_connection_t *conn,
+                                 const struct control_cmd_args_t *args);
+int handle_control_authenticate(control_connection_t *conn,
+                                const struct control_cmd_args_t *args);
+void control_auth_free_all(void);
+
+extern const struct control_cmd_syntax_t authchallenge_syntax;
+extern const struct control_cmd_syntax_t authenticate_syntax;
+
+#endif /* !defined(TOR_CONTROL_AUTH_H) */
diff --git a/src/feature/control/control_bootstrap.c b/src/feature/control/control_bootstrap.c
new file mode 100644
index 0000000000..fee7612ba2
--- /dev/null
+++ b/src/feature/control/control_bootstrap.c
@@ -0,0 +1,389 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_bootstrap.c
+ * \brief Provide bootstrap progress events for the control port.
+ */
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
+#include "core/or/connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/reasons.h"
+#include "feature/control/control_events.h"
+#include "feature/hibernate/hibernate.h"
+#include "lib/malloc/malloc.h"
+
+/** A sufficiently large size to record the last bootstrap phase string. */
+#define BOOTSTRAP_MSG_LEN 1024
+
+/** What was the last bootstrap phase message we sent? We keep track
+ * of this so we can respond to getinfo status/bootstrap-phase queries. */
+static char last_sent_bootstrap_message[BOOTSTRAP_MSG_LEN];
+
+/** Table to convert bootstrap statuses to strings.  */
+static const struct {
+  bootstrap_status_t status;
+  const char *tag;
+  const char *summary;
+} boot_to_str_tab[] = {
+  { BOOTSTRAP_STATUS_UNDEF, "undef", "Undefined" },
+  { BOOTSTRAP_STATUS_STARTING, "starting", "Starting" },
+
+  /* Initial connection to any relay */
+
+  { BOOTSTRAP_STATUS_CONN_PT, "conn_pt", "Connecting to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PT, "conn_done_pt",
+    "Connected to pluggable transport" },
+  { BOOTSTRAP_STATUS_CONN_PROXY, "conn_proxy", "Connecting to proxy" },
+  { BOOTSTRAP_STATUS_CONN_DONE_PROXY, "conn_done_proxy",
+    "Connected to proxy" },
+  { BOOTSTRAP_STATUS_CONN, "conn", "Connecting to a relay" },
+  { BOOTSTRAP_STATUS_CONN_DONE, "conn_done", "Connected to a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE, "handshake",
+    "Handshaking with a relay" },
+  { BOOTSTRAP_STATUS_HANDSHAKE_DONE, "handshake_done",
+    "Handshake with a relay done" },
+
+  /* Loading directory info */
+
+  { BOOTSTRAP_STATUS_ONEHOP_CREATE, "onehop_create",
+    "Establishing an encrypted directory connection" },
+  { BOOTSTRAP_STATUS_REQUESTING_STATUS, "requesting_status",
+    "Asking for networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_STATUS, "loading_status",
+    "Loading networkstatus consensus" },
+  { BOOTSTRAP_STATUS_LOADING_KEYS, "loading_keys",
+    "Loading authority key certs" },
+  { BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, "requesting_descriptors",
+    "Asking for relay descriptors" },
+  { BOOTSTRAP_STATUS_LOADING_DESCRIPTORS, "loading_descriptors",
+    "Loading relay descriptors" },
+  { BOOTSTRAP_STATUS_ENOUGH_DIRINFO, "enough_dirinfo",
+    "Loaded enough directory info to build circuits" },
+
+  /* Connecting to a relay for AP circuits */
+
+  { BOOTSTRAP_STATUS_AP_CONN_PT, "ap_conn_pt",
+    "Connecting to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PT, "ap_conn_done_pt",
+    "Connected to pluggable transport to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_PROXY, "ap_conn_proxy",
+    "Connecting to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY, "ap_conn_done_proxy",
+    "Connected to proxy to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN, "ap_conn",
+    "Connecting to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_CONN_DONE, "ap_conn_done",
+    "Connected to a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE, "ap_handshake",
+    "Finishing handshake with a relay to build circuits" },
+  { BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE, "ap_handshake_done",
+    "Handshake finished with a relay to build circuits" },
+
+  /* Creating AP circuits */
+
+  { BOOTSTRAP_STATUS_CIRCUIT_CREATE, "circuit_create",
+    "Establishing a Tor circuit" },
+  { BOOTSTRAP_STATUS_DONE, "done", "Done" },
+};
+#define N_BOOT_TO_STR (sizeof(boot_to_str_tab)/sizeof(boot_to_str_tab[0]))
+
+/** Convert the name of a bootstrapping phase <b>s</b> into strings
+ * <b>tag</b> and <b>summary</b> suitable for display by the controller. */
+static int
+bootstrap_status_to_string(bootstrap_status_t s, const char **tag,
+                           const char **summary)
+{
+  for (size_t i = 0; i < N_BOOT_TO_STR; i++) {
+    if (s == boot_to_str_tab[i].status) {
+      *tag = boot_to_str_tab[i].tag;
+      *summary = boot_to_str_tab[i].summary;
+      return 0;
+    }
+  }
+
+  *tag = *summary = "unknown";
+  return -1;
+}
+
+/** What percentage through the bootstrap process are we? We remember
+ * this so we can avoid sending redundant bootstrap status events, and
+ * so we can guess context for the bootstrap messages which are
+ * ambiguous. It starts at 'undef', but gets set to 'starting' while
+ * Tor initializes. */
+static int bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_percent, but only takes on the enumerated values in
+ * bootstrap_status_t.
+ */
+static int bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** As bootstrap_percent, but holds the bootstrapping level at which we last
+ * logged a NOTICE-level message. We use this, plus BOOTSTRAP_PCT_INCREMENT,
+ * to avoid flooding the log with a new message every time we get a few more
+ * microdescriptors */
+static int notice_bootstrap_percent = 0;
+
+/** How many problems have we had getting to the next bootstrapping phase?
+ * These include failure to establish a connection to a Tor relay,
+ * failures to finish the TLS handshake, failures to validate the
+ * consensus document, etc. */
+static int bootstrap_problems = 0;
+
+/** We only tell the controller once we've hit a threshold of problems
+ * for the current phase. */
+#define BOOTSTRAP_PROBLEM_THRESHOLD 10
+
+/** When our bootstrapping progress level changes, but our bootstrapping
+ * status has not advanced, we only log at NOTICE when we have made at least
+ * this much progress.
+ */
+#define BOOTSTRAP_PCT_INCREMENT 5
+
+/** Do the actual logging and notifications for
+ * control_event_bootstrap().  Doesn't change any state beyond that.
+ */
+static void
+control_event_bootstrap_core(int loglevel, bootstrap_status_t status,
+                             int progress)
+{
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *tag, *summary;
+
+  bootstrap_status_to_string(status, &tag, &summary);
+  /* Locally reset status if there's incremental progress */
+  if (progress)
+    status = progress;
+
+  tor_log(loglevel, LD_CONTROL,
+          "Bootstrapped %d%% (%s): %s", status, tag, summary);
+  tor_snprintf(buf, sizeof(buf),
+               "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\"",
+               status, tag, summary);
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "NOTICE %s", buf);
+  control_event_client_status(LOG_NOTICE, "%s", buf);
+}
+
+int
+control_get_bootstrap_percent(void)
+{
+  return bootstrap_percent;
+}
+
+/** Called when Tor has made progress at bootstrapping its directory
+ * information and initial circuits.
+ *
+ * <b>status</b> is the new status, that is, what task we will be doing
+ * next. <b>progress</b> is zero if we just started this task, else it
+ * represents progress on the task.
+ */
+void
+control_event_bootstrap(bootstrap_status_t status, int progress)
+{
+  int loglevel = LOG_NOTICE;
+
+  if (bootstrap_percent == BOOTSTRAP_STATUS_DONE)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  if (status <= bootstrap_percent) {
+    /* If there's no new progress, return early. */
+    if (!progress || progress <= bootstrap_percent)
+      return;
+    /* Log at INFO if not enough progress happened. */
+    if (progress < notice_bootstrap_percent + BOOTSTRAP_PCT_INCREMENT)
+      loglevel = LOG_INFO;
+  }
+
+  control_event_bootstrap_core(loglevel, status, progress);
+
+  if (status > bootstrap_percent) {
+    bootstrap_phase = status; /* new milestone reached */
+    bootstrap_percent = status;
+  }
+  if (progress > bootstrap_percent) {
+    /* incremental progress within a milestone */
+    bootstrap_percent = progress;
+    bootstrap_problems = 0; /* Progress! Reset our problem counter. */
+  }
+  if (loglevel == LOG_NOTICE &&
+      bootstrap_percent > notice_bootstrap_percent) {
+    /* Remember that we gave a notice at this level. */
+    notice_bootstrap_percent = bootstrap_percent;
+  }
+}
+
+/** Flag whether we've opened an OR_CONN yet  */
+static int bootstrap_first_orconn = 0;
+
+/** Like bootstrap_phase, but for (possibly deferred) directory progress */
+static int bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+
+/** Like bootstrap_problems, but for (possibly deferred) directory progress  */
+static int bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+
+/** Defer directory info bootstrap events until we have successfully
+ * completed our first connection to a router.  */
+void
+control_event_boot_dir(bootstrap_status_t status, int progress)
+{
+  if (status > bootstrap_dir_progress) {
+    bootstrap_dir_progress = status;
+    bootstrap_dir_phase = status;
+  }
+  if (progress && progress >= bootstrap_dir_progress) {
+    bootstrap_dir_progress = progress;
+  }
+
+  /* Don't report unless we have successfully opened at least one OR_CONN */
+  if (!bootstrap_first_orconn)
+    return;
+
+  control_event_bootstrap(status, progress);
+}
+
+/** Set a flag to allow reporting of directory bootstrap progress.
+ * (Code that reports completion of an OR_CONN calls this.)  Also,
+ * report directory progress so far. */
+void
+control_event_boot_first_orconn(void)
+{
+  bootstrap_first_orconn = 1;
+  control_event_bootstrap(bootstrap_dir_phase, bootstrap_dir_progress);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a human-readable hint
+ * as to why, and <b>reason</b> provides a controller-facing short
+ * tag.  <b>conn</b> is the connection that caused this problem and
+ * can be NULL if a connection cannot be easily identified.
+ */
+void
+control_event_bootstrap_problem(const char *warn, const char *reason,
+                                const connection_t *conn, int dowarn)
+{
+  int status = bootstrap_percent;
+  const char *tag = "", *summary = "";
+  char buf[BOOTSTRAP_MSG_LEN];
+  const char *recommendation = "ignore";
+  int severity;
+  char *or_id = NULL, *hostaddr = NULL;
+  or_connection_t *or_conn = NULL;
+
+  /* bootstrap_percent must not be in "undefined" state here. */
+  tor_assert(status >= 0);
+
+  if (bootstrap_percent == 100)
+    return; /* already bootstrapped; nothing to be done here. */
+
+  bootstrap_problems++;
+
+  if (bootstrap_problems >= BOOTSTRAP_PROBLEM_THRESHOLD)
+    dowarn = 1;
+
+  /* Don't warn about our bootstrapping status if we are hibernating or
+   * shutting down. */
+  if (we_are_hibernating())
+    dowarn = 0;
+
+  tor_assert(bootstrap_status_to_string(bootstrap_phase, &tag, &summary) == 0);
+
+  severity = dowarn ? LOG_WARN : LOG_INFO;
+
+  if (dowarn)
+    recommendation = "warn";
+
+  if (conn && conn->type == CONN_TYPE_OR) {
+    /* XXX TO_OR_CONN can't deal with const */
+    or_conn = TO_OR_CONN((connection_t *)conn);
+    or_id = tor_strdup(hex_str(or_conn->identity_digest, DIGEST_LEN));
+  } else {
+    or_id = tor_strdup("?");
+  }
+
+  if (conn)
+    tor_asprintf(&hostaddr, "%s:%d", conn->address, (int)conn->port);
+  else
+    hostaddr = tor_strdup("?");
+
+  log_fn(severity,
+         LD_CONTROL, "Problem bootstrapping. Stuck at %d%% (%s): %s. (%s; %s; "
+         "count %d; recommendation %s; host %s at %s)",
+         status, tag, summary, warn, reason,
+         bootstrap_problems, recommendation,
+         or_id, hostaddr);
+
+  connection_or_report_broken_states(severity, LD_HANDSHAKE);
+
+  tor_snprintf(buf, sizeof(buf),
+      "BOOTSTRAP PROGRESS=%d TAG=%s SUMMARY=\"%s\" WARNING=\"%s\" REASON=%s "
+      "COUNT=%d RECOMMENDATION=%s HOSTID=\"%s\" HOSTADDR=\"%s\"",
+      bootstrap_percent, tag, summary, warn, reason, bootstrap_problems,
+      recommendation,
+      or_id, hostaddr);
+
+  tor_snprintf(last_sent_bootstrap_message,
+               sizeof(last_sent_bootstrap_message),
+               "WARN %s", buf);
+  control_event_client_status(LOG_WARN, "%s", buf);
+
+  tor_free(hostaddr);
+  tor_free(or_id);
+}
+
+/** Called when Tor has failed to make bootstrapping progress in a way
+ * that indicates a problem. <b>warn</b> gives a hint as to why, and
+ * <b>reason</b> provides an "or_conn_end_reason" tag.  <b>or_conn</b>
+ * is the connection that caused this problem.
+ */
+MOCK_IMPL(void,
+control_event_bootstrap_prob_or, (const char *warn, int reason,
+                                  or_connection_t *or_conn))
+{
+  int dowarn = 0;
+
+  if (or_conn->have_noted_bootstrap_problem)
+    return;
+
+  or_conn->have_noted_bootstrap_problem = 1;
+
+  if (reason == END_OR_CONN_REASON_NO_ROUTE)
+    dowarn = 1;
+
+  /* If we are using bridges and all our OR connections are now
+     closed, it means that we totally failed to connect to our
+     bridges. Throw a warning. */
+  if (get_options()->UseBridges && !any_other_active_or_conns(or_conn))
+    dowarn = 1;
+
+  control_event_bootstrap_problem(warn,
+                                  orconn_end_reason_to_control_string(reason),
+                                  TO_CONN(or_conn), dowarn);
+}
+
+/** Return a copy of the last sent bootstrap message. */
+char *
+control_event_boot_last_msg(void)
+{
+  return tor_strdup(last_sent_bootstrap_message);
+}
+
+/** Reset bootstrap tracking state. */
+void
+control_event_bootstrap_reset(void)
+{
+  bootstrap_percent = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_phase = BOOTSTRAP_STATUS_UNDEF;
+  notice_bootstrap_percent = 0;
+  bootstrap_problems = 0;
+  bootstrap_first_orconn = 0;
+  bootstrap_dir_progress = BOOTSTRAP_STATUS_UNDEF;
+  bootstrap_dir_phase = BOOTSTRAP_STATUS_UNDEF;
+  memset(last_sent_bootstrap_message, 0, sizeof(last_sent_bootstrap_message));
+}
diff --git a/src/feature/control/control_cmd.c b/src/feature/control/control_cmd.c
new file mode 100644
index 0000000000..eb14f101e7
--- /dev/null
+++ b/src/feature/control/control_cmd.c
@@ -0,0 +1,2422 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.c
+ * \brief Implement various commands for Tor's control-socket interface.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/confmgt/confmgt.h"
+#include "app/main/main.h"
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/connection_edge.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/dnsserv.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_auth.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_hs.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/hs/hs_control.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/rend/rendclient.h"
+#include "feature/rend/rendcommon.h"
+#include "feature/rend/rendparse.h"
+#include "feature/rend/rendservice.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/rend/rend_authorized_client_st.h"
+#include "feature/rend/rend_encoded_v2_service_descriptor_st.h"
+#include "feature/rend/rend_service_descriptor_st.h"
+
+static int control_setconf_helper(control_connection_t *conn,
+                                  const control_cmd_args_t *args,
+                                  int use_defaults);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/**
+ * Release all storage held in <b>args</b>
+ **/
+void
+control_cmd_args_free_(control_cmd_args_t *args)
+{
+  if (! args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, tor_free(c));
+    smartlist_free(args->args);
+  }
+  config_free_lines(args->kwargs);
+  tor_free(args->cmddata);
+
+  tor_free(args);
+}
+
+/** Erase all memory held in <b>args</b>. */
+void
+control_cmd_args_wipe(control_cmd_args_t *args)
+{
+  if (!args)
+    return;
+
+  if (args->args) {
+    SMARTLIST_FOREACH(args->args, char *, c, memwipe(c, 0, strlen(c)));
+  }
+  for (config_line_t *line = args->kwargs; line; line = line->next) {
+    memwipe(line->key, 0, strlen(line->key));
+    memwipe(line->value, 0, strlen(line->value));
+  }
+  if (args->cmddata)
+    memwipe(args->cmddata, 0, args->cmddata_len);
+}
+
+/**
+ * Return true iff any element of the NULL-terminated <b>array</b> matches
+ * <b>kwd</b>. Case-insensitive.
+ **/
+static bool
+string_array_contains_keyword(const char **array, const char *kwd)
+{
+  for (unsigned i = 0; array[i]; ++i) {
+    if (! strcasecmp(array[i], kwd))
+      return true;
+  }
+  return false;
+}
+
+/** Helper for argument parsing: check whether the keyword arguments just
+ * parsed in <b>result</b> were well-formed according to <b>syntax</b>.
+ *
+ * On success, return 0.  On failure, return -1 and set *<b>error_out</b>
+ * to a newly allocated error string.
+ **/
+static int
+kvline_check_keyword_args(const control_cmd_args_t *result,
+                          const control_cmd_syntax_t *syntax,
+                          char **error_out)
+{
+  if (result->kwargs == NULL) {
+    tor_asprintf(error_out, "Cannot parse keyword argument(s)");
+    return -1;
+  }
+
+  if (! syntax->allowed_keywords) {
+    /* All keywords are permitted. */
+    return 0;
+  }
+
+  /* Check for unpermitted arguments */
+  const config_line_t *line;
+  for (line = result->kwargs; line; line = line->next) {
+    if (! string_array_contains_keyword(syntax->allowed_keywords,
+                                        line->key)) {
+      tor_asprintf(error_out, "Unrecognized keyword argument %s",
+                   escaped(line->key));
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Helper: parse the arguments to a command according to <b>syntax</b>.  On
+ * success, set *<b>error_out</b> to NULL and return a newly allocated
+ * control_cmd_args_t.  On failure, set *<b>error_out</b> to newly allocated
+ * error string, and return NULL.
+ **/
+STATIC control_cmd_args_t *
+control_cmd_parse_args(const char *command,
+                       const control_cmd_syntax_t *syntax,
+                       size_t body_len,
+                       const char *body,
+                       char **error_out)
+{
+  *error_out = NULL;
+  control_cmd_args_t *result = tor_malloc_zero(sizeof(control_cmd_args_t));
+  const char *cmdline;
+  char *cmdline_alloc = NULL;
+  tor_assert(syntax->max_args < INT_MAX || syntax->max_args == UINT_MAX);
+
+  result->command = command;
+
+  if (syntax->store_raw_body) {
+    tor_assert(body[body_len] == 0);
+    result->raw_body = body;
+  }
+
+  const char *eol = memchr(body, '\n', body_len);
+  if (syntax->want_cmddata) {
+    if (! eol || (eol+1) == body+body_len) {
+      *error_out = tor_strdup("Empty body");
+      goto err;
+    }
+    cmdline_alloc = tor_memdup_nulterm(body, eol-body);
+    cmdline = cmdline_alloc;
+    ++eol;
+    result->cmddata_len = read_escaped_data(eol, (body+body_len)-eol,
+                                           &result->cmddata);
+  } else {
+    if (eol && (eol+1) != body+body_len) {
+      *error_out = tor_strdup("Unexpected body");
+      goto err;
+    }
+    cmdline = body;
+  }
+
+  result->args = smartlist_new();
+  smartlist_split_string(result->args, cmdline, " ",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK,
+                         (int)(syntax->max_args+1));
+  size_t n_args = smartlist_len(result->args);
+  if (n_args < syntax->min_args) {
+    tor_asprintf(error_out, "Need at least %u argument(s)",
+                 syntax->min_args);
+    goto err;
+  } else if (n_args > syntax->max_args && ! syntax->accept_keywords) {
+    tor_asprintf(error_out, "Cannot accept more than %u argument(s)",
+                 syntax->max_args);
+    goto err;
+  }
+
+  if (n_args > syntax->max_args) {
+    /* We have extra arguments after the positional arguments, and we didn't
+       treat them as an error, so they must count as keyword arguments: Either
+       K=V pairs, or flags, or both. */
+    tor_assert(n_args == syntax->max_args + 1);
+    tor_assert(syntax->accept_keywords);
+    char *remainder = smartlist_pop_last(result->args);
+    result->kwargs = kvline_parse(remainder, syntax->kvline_flags);
+    tor_free(remainder);
+    if (kvline_check_keyword_args(result, syntax, error_out) < 0) {
+      goto err;
+    }
+  }
+
+  tor_assert_nonfatal(*error_out == NULL);
+  goto done;
+ err:
+  tor_assert_nonfatal(*error_out != NULL);
+  control_cmd_args_free(result);
+ done:
+  tor_free(cmdline_alloc);
+  return result;
+}
+
+/**
+ * Return true iff <b>lines</b> contains <b>flags</b> as a no-value
+ * (keyword-only) entry.
+ **/
+static bool
+config_lines_contain_flag(const config_line_t *lines, const char *flag)
+{
+  const config_line_t *line = config_line_find_case(lines, flag);
+  return line && !strcmp(line->value, "");
+}
+
+static const control_cmd_syntax_t setconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a SETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body.*/
+static int
+handle_control_setconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 0);
+}
+
+static const control_cmd_syntax_t resetconf_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+
+/** Called when we receive a RESETCONF message: parse the body and try
+ * to update our configuration.  Reply with a DONE or ERROR message.
+ * Modifies the contents of body. */
+static int
+handle_control_resetconf(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  return control_setconf_helper(conn, args, 1);
+}
+
+static const control_cmd_syntax_t getconf_syntax = {
+  .max_args=UINT_MAX
+};
+
+/** Called when we receive a GETCONF message.  Parse the request, and
+ * reply with a CONFVALUE or an ERROR message */
+static int
+handle_control_getconf(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  const or_options_t *options = get_options();
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    if (!option_is_recognized(q)) {
+      control_reply_add_printf(unrecognized, 552,
+                               "Unrecognized configuration key \"%s\"", q);
+    } else {
+      config_line_t *answer = option_get_assignment(options,q);
+      if (!answer) {
+        const char *name = option_get_canonical_name(q);
+        control_reply_add_one_kv(answers, 250, KV_OMIT_VALS, name, "");
+      }
+
+      while (answer) {
+        config_line_t *next;
+        control_reply_add_one_kv(answers, 250, KV_RAW, answer->key,
+                                 answer->value);
+        next = answer->next;
+        tor_free(answer->key);
+        tor_free(answer->value);
+        tor_free(answer);
+        answer = next;
+      }
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  if (smartlist_len(unrecognized)) {
+    control_write_reply_lines(conn, unrecognized);
+  } else if (smartlist_len(answers)) {
+    control_write_reply_lines(conn, answers);
+  } else {
+    send_control_done(conn);
+  }
+
+  control_reply_free(answers);
+  control_reply_free(unrecognized);
+  return 0;
+}
+
+static const control_cmd_syntax_t loadconf_syntax = {
+  .want_cmddata = true
+};
+
+/** Called when we get a +LOADCONF message. */
+static int
+handle_control_loadconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  setopt_err_t retval;
+  char *errstring = NULL;
+
+  retval = options_init_from_string(NULL, args->cmddata,
+                                    CMD_RUN_TOR, NULL, &errstring);
+
+  if (retval != SETOPT_OK)
+    log_warn(LD_CONTROL,
+             "Controller gave us config file that didn't validate: %s",
+             errstring);
+
+#define SEND_ERRMSG(code, msg)                          \
+  control_printf_endreply(conn, code, msg "%s%s",       \
+                          errstring ? ": " : "",        \
+                          errstring ? errstring : "")
+  switch (retval) {
+  case SETOPT_ERR_PARSE:
+    SEND_ERRMSG(552, "Invalid config file");
+    break;
+  case SETOPT_ERR_TRANSITION:
+    SEND_ERRMSG(553, "Transition not allowed");
+    break;
+  case SETOPT_ERR_SETTING:
+    SEND_ERRMSG(553, "Unable to set option");
+    break;
+  case SETOPT_ERR_MISC:
+  default:
+    SEND_ERRMSG(550, "Unable to load config");
+    break;
+  case SETOPT_OK:
+    send_control_done(conn);
+    break;
+  }
+#undef SEND_ERRMSG
+  tor_free(errstring);
+  return 0;
+}
+
+static const control_cmd_syntax_t setevents_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a SETEVENTS message: update conn->event_mask,
+ * and reply with DONE or ERROR. */
+static int
+handle_control_setevents(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  int event_code;
+  event_mask_t event_mask = 0;
+  const smartlist_t *events = args->args;
+
+  SMARTLIST_FOREACH_BEGIN(events, const char *, ev)
+    {
+      if (!strcasecmp(ev, "EXTENDED") ||
+          !strcasecmp(ev, "AUTHDIR_NEWDESCS")) {
+        log_warn(LD_CONTROL, "The \"%s\" SETEVENTS argument is no longer "
+                 "supported.", ev);
+        continue;
+      } else {
+        int i;
+        event_code = -1;
+
+        for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+          if (!strcasecmp(ev, control_event_table[i].event_name)) {
+            event_code = control_event_table[i].event_code;
+            break;
+          }
+        }
+
+        if (event_code == -1) {
+          control_printf_endreply(conn, 552, "Unrecognized event \"%s\"", ev);
+          return 0;
+        }
+      }
+      event_mask |= (((event_mask_t)1) << event_code);
+    }
+  SMARTLIST_FOREACH_END(ev);
+
+  conn->event_mask = event_mask;
+
+  control_update_global_event_mask();
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t saveconf_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a SAVECONF command. Try to flush the current options to
+ * disk, and report success or failure. */
+static int
+handle_control_saveconf(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  bool force = config_lines_contain_flag(args->kwargs, "FORCE");
+  const or_options_t *options = get_options();
+  if ((!force && options->IncludeUsed) || options_save_current() < 0) {
+    control_write_endreply(conn, 551,
+                           "Unable to write configuration to disk.");
+  } else {
+    send_control_done(conn);
+  }
+  return 0;
+}
+
+static const control_cmd_syntax_t signal_syntax = {
+  .min_args = 1,
+  .max_args = 1,
+};
+
+/** Called when we get a SIGNAL command. React to the provided signal, and
+ * report success or failure. (If the signal results in a shutdown, success
+ * may not be reported.) */
+static int
+handle_control_signal(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  int sig = -1;
+  int i;
+
+  tor_assert(smartlist_len(args->args) == 1);
+  const char *s = smartlist_get(args->args, 0);
+
+  for (i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if (!strcasecmp(s, signal_table[i].signal_name)) {
+      sig = signal_table[i].sig;
+      break;
+    }
+  }
+
+  if (sig < 0)
+    control_printf_endreply(conn, 552, "Unrecognized signal code \"%s\"", s);
+  if (sig < 0)
+    return 0;
+
+  send_control_done(conn);
+  /* Flush the "done" first if the signal might make us shut down. */
+  if (sig == SIGTERM || sig == SIGINT)
+    connection_flush(TO_CONN(conn));
+
+  activate_signal(sig);
+
+  return 0;
+}
+
+static const control_cmd_syntax_t takeownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a TAKEOWNERSHIP command.  Mark this connection
+ * as an owning connection, so that we will exit if the connection
+ * closes. */
+static int
+handle_control_takeownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 1;
+
+  log_info(LD_CONTROL, "Control connection %d has taken ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t dropownership_syntax = {
+  .max_args = UINT_MAX, // This should probably become zero. XXXXX
+};
+
+/** Called when we get a DROPOWNERSHIP command.  Mark this connection
+ * as a non-owning connection, so that we will not exit if the connection
+ * closes. */
+static int
+handle_control_dropownership(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  (void)args;
+
+  conn->is_owning_control_connection = 0;
+
+  log_info(LD_CONTROL, "Control connection %d has dropped ownership of this "
+           "Tor instance.",
+           (int)(conn->base_.s));
+
+  send_control_done(conn);
+  return 0;
+}
+
+/** Given a text circuit <b>id</b>, return the corresponding circuit. */
+static origin_circuit_t *
+get_circ(const char *id)
+{
+  uint32_t n_id;
+  int ok;
+  n_id = (uint32_t) tor_parse_ulong(id, 10, 0, UINT32_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  return circuit_get_by_global_id(n_id);
+}
+
+/** Given a text stream <b>id</b>, return the corresponding AP connection. */
+static entry_connection_t *
+get_stream(const char *id)
+{
+  uint64_t n_id;
+  int ok;
+  connection_t *conn;
+  n_id = tor_parse_uint64(id, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok)
+    return NULL;
+  conn = connection_get_by_global_id(n_id);
+  if (!conn || conn->type != CONN_TYPE_AP || conn->marked_for_close)
+    return NULL;
+  return TO_ENTRY_CONN(conn);
+}
+
+/** Helper for setconf and resetconf. Acts like setconf, except
+ * it passes <b>use_defaults</b> on to options_trial_assign().  Modifies the
+ * contents of body.
+ */
+static int
+control_setconf_helper(control_connection_t *conn,
+                       const control_cmd_args_t *args,
+                       int use_defaults)
+{
+  setopt_err_t opt_err;
+  char *errstring = NULL;
+  const unsigned flags =
+    CAL_CLEAR_FIRST | (use_defaults ? CAL_USE_DEFAULTS : 0);
+
+  // We need a copy here, since confmgt.c wants to canonicalize cases.
+  config_line_t *lines = config_lines_dup(args->kwargs);
+
+  opt_err = options_trial_assign(lines, flags, &errstring);
+  {
+#define SEND_ERRMSG(code, msg)                                  \
+    control_printf_endreply(conn, code, msg ": %s", errstring);
+
+    switch (opt_err) {
+      case SETOPT_ERR_MISC:
+        SEND_ERRMSG(552, "Unrecognized option");
+        break;
+      case SETOPT_ERR_PARSE:
+        SEND_ERRMSG(513, "Unacceptable option value");
+        break;
+      case SETOPT_ERR_TRANSITION:
+        SEND_ERRMSG(553, "Transition not allowed");
+        break;
+      case SETOPT_ERR_SETTING:
+      default:
+        SEND_ERRMSG(553, "Unable to set option");
+        break;
+      case SETOPT_OK:
+        config_free_lines(lines);
+        send_control_done(conn);
+        return 0;
+    }
+#undef SEND_ERRMSG
+    log_warn(LD_CONTROL,
+             "Controller gave us config lines that didn't validate: %s",
+             errstring);
+    config_free_lines(lines);
+    tor_free(errstring);
+    return 0;
+  }
+}
+
+/** Return true iff <b>addr</b> is unusable as a mapaddress target because of
+ * containing funny characters. */
+static int
+address_is_invalid_mapaddress_target(const char *addr)
+{
+  if (!strcmpstart(addr, "*."))
+    return address_is_invalid_destination(addr+2, 1);
+  else
+    return address_is_invalid_destination(addr, 1);
+}
+
+static const control_cmd_syntax_t mapaddress_syntax = {
+  // no positional arguments are expected
+  .max_args=0,
+  // an arbitrary number of K=V entries are supported.
+  .accept_keywords=true,
+};
+
+/** Called when we get a MAPADDRESS command; try to bind all listed addresses,
+ * and report success or failure. */
+static int
+handle_control_mapaddress(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  smartlist_t *reply;
+  char *r;
+  size_t sz;
+
+  reply = smartlist_new();
+  const config_line_t *line;
+  for (line = args->kwargs; line; line = line->next) {
+    const char *from = line->key;
+    const char *to = line->value;
+    {
+      if (address_is_invalid_mapaddress_target(to)) {
+        smartlist_add_asprintf(reply,
+                     "512-syntax error: invalid address '%s'", to);
+        log_warn(LD_CONTROL,
+                 "Skipping invalid argument '%s' in MapAddress msg", to);
+      } else if (!strcmp(from, ".") || !strcmp(from, "0.0.0.0") ||
+                 !strcmp(from, "::")) {
+        const char type =
+          !strcmp(from,".") ? RESOLVED_TYPE_HOSTNAME :
+          (!strcmp(from, "0.0.0.0") ? RESOLVED_TYPE_IPV4 : RESOLVED_TYPE_IPV6);
+        const char *address = addressmap_register_virtual_address(
+                                                     type, tor_strdup(to));
+        if (!address) {
+          smartlist_add_asprintf(reply,
+                   "451-resource exhausted: skipping '%s=%s'", from,to);
+          log_warn(LD_CONTROL,
+                   "Unable to allocate address for '%s' in MapAddress msg",
+                   safe_str_client(to));
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", address, to);
+        }
+      } else {
+        const char *msg;
+        if (addressmap_register_auto(from, to, 1,
+                                     ADDRMAPSRC_CONTROLLER, &msg) < 0) {
+          smartlist_add_asprintf(reply,
+                                 "512-syntax error: invalid address mapping "
+                                 " '%s=%s': %s", from, to, msg);
+          log_warn(LD_CONTROL,
+                   "Skipping invalid argument '%s=%s' in MapAddress msg: %s",
+                   from, to, msg);
+        } else {
+          smartlist_add_asprintf(reply, "250-%s=%s", from, to);
+        }
+      }
+    }
+  }
+
+  if (smartlist_len(reply)) {
+    ((char*)smartlist_get(reply,smartlist_len(reply)-1))[3] = ' ';
+    r = smartlist_join_strings(reply, "\r\n", 1, &sz);
+    connection_buf_add(r, sz, TO_CONN(conn));
+    tor_free(r);
+  } else {
+    control_write_endreply(conn, 512, "syntax error: "
+                           "not enough arguments to mapaddress.");
+  }
+
+  SMARTLIST_FOREACH(reply, char *, cp, tor_free(cp));
+  smartlist_free(reply);
+  return 0;
+}
+
+/** Given a string, convert it to a circuit purpose. */
+static uint8_t
+circuit_purpose_from_string(const char *string)
+{
+  if (!strcasecmpstart(string, "purpose="))
+    string += strlen("purpose=");
+
+  if (!strcasecmp(string, "general"))
+    return CIRCUIT_PURPOSE_C_GENERAL;
+  else if (!strcasecmp(string, "controller"))
+    return CIRCUIT_PURPOSE_CONTROLLER;
+  else
+    return CIRCUIT_PURPOSE_UNKNOWN;
+}
+
+static const control_cmd_syntax_t extendcircuit_syntax = {
+  .min_args=1,
+  .max_args=1, // see note in function
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS
+};
+
+/** Called when we get an EXTENDCIRCUIT message.  Try to extend the listed
+ * circuit, and report success or failure. */
+static int
+handle_control_extendcircuit(control_connection_t *conn,
+                             const control_cmd_args_t *args)
+{
+  smartlist_t *router_nicknames=smartlist_new(), *nodes=NULL;
+  origin_circuit_t *circ = NULL;
+  uint8_t intended_purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  const config_line_t *kwargs = args->kwargs;
+  const char *circ_id = smartlist_get(args->args, 0);
+  const char *path_str = NULL;
+  char *path_str_alloc = NULL;
+
+  /* The syntax for this command is unfortunate. The second argument is
+     optional, and is a comma-separated list long-format fingerprints, which
+     can (historically!) contain an equals sign.
+
+     Here we check the second argument to see if it's a path, and if so we
+     remove it from the kwargs list and put it in path_str.
+  */
+  if (kwargs) {
+    const config_line_t *arg1 = kwargs;
+    if (!strcmp(arg1->value, "")) {
+      path_str = arg1->key;
+      kwargs = kwargs->next;
+    } else if (arg1->key[0] == '$') {
+      tor_asprintf(&path_str_alloc, "%s=%s", arg1->key, arg1->value);
+      path_str = path_str_alloc;
+      kwargs = kwargs->next;
+    }
+  }
+
+  const config_line_t *purpose_line = config_line_find_case(kwargs, "PURPOSE");
+  bool zero_circ = !strcmp("0", circ_id);
+
+  if (purpose_line) {
+    intended_purpose = circuit_purpose_from_string(purpose_line->value);
+    if (intended_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purpose_line->value);
+      goto done;
+    }
+  }
+
+  if (zero_circ) {
+    if (!path_str) {
+      // "EXTENDCIRCUIT 0" with no path.
+      circ = circuit_launch(intended_purpose, CIRCLAUNCH_NEED_CAPACITY);
+      if (!circ) {
+        control_write_endreply(conn, 551, "Couldn't start circuit");
+      } else {
+        control_printf_endreply(conn, 250, "EXTENDED %lu",
+                                (unsigned long)circ->global_identifier);
+      }
+      goto done;
+    }
+  }
+
+  if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  if (!path_str) {
+    control_write_endreply(conn, 512, "syntax error: path required.");
+    goto done;
+  }
+
+  smartlist_split_string(router_nicknames, path_str, ",", 0, 0);
+
+  nodes = smartlist_new();
+  bool first_node = zero_circ;
+  SMARTLIST_FOREACH_BEGIN(router_nicknames, const char *, n) {
+    const node_t *node = node_get_by_nickname(n, 0);
+    if (!node) {
+      control_printf_endreply(conn, 552, "No such router \"%s\"", n);
+      goto done;
+    }
+    if (!node_has_preferred_descriptor(node, first_node)) {
+      control_printf_endreply(conn, 552, "No descriptor for \"%s\"", n);
+      goto done;
+    }
+    smartlist_add(nodes, (void*)node);
+    first_node = false;
+  } SMARTLIST_FOREACH_END(n);
+
+  if (!smartlist_len(nodes)) {
+    control_write_endreply(conn, 512, "No router names provided");
+    goto done;
+  }
+
+  if (zero_circ) {
+    /* start a new circuit */
+    circ = origin_circuit_init(intended_purpose, 0);
+  }
+
+  /* now circ refers to something that is ready to be extended */
+  first_node = zero_circ;
+  SMARTLIST_FOREACH(nodes, const node_t *, node,
+  {
+    extend_info_t *info = extend_info_from_node(node, first_node);
+    if (!info) {
+      tor_assert_nonfatal(first_node);
+      log_warn(LD_CONTROL,
+               "controller tried to connect to a node that lacks a suitable "
+               "descriptor, or which doesn't have any "
+               "addresses that are allowed by the firewall configuration; "
+               "circuit marked for closing.");
+      circuit_mark_for_close(TO_CIRCUIT(circ), -END_CIRC_REASON_CONNECTFAILED);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+    circuit_append_new_exit(circ, info);
+    if (circ->build_state->desired_path_len > 1) {
+      circ->build_state->onehop_tunnel = 0;
+    }
+    extend_info_free(info);
+    first_node = 0;
+  });
+
+  /* now that we've populated the cpath, start extending */
+  if (zero_circ) {
+    int err_reason = 0;
+    if ((err_reason = circuit_handle_first_hop(circ)) < 0) {
+      circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+      control_write_endreply(conn, 551, "Couldn't start circuit");
+      goto done;
+    }
+  } else {
+    if (circ->base_.state == CIRCUIT_STATE_OPEN ||
+        circ->base_.state == CIRCUIT_STATE_GUARD_WAIT) {
+      int err_reason = 0;
+      circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_BUILDING);
+      if ((err_reason = circuit_send_next_onion_skin(circ)) < 0) {
+        log_info(LD_CONTROL,
+                 "send_next_onion_skin failed; circuit marked for closing.");
+        circuit_mark_for_close(TO_CIRCUIT(circ), -err_reason);
+        control_write_endreply(conn, 551, "Couldn't send onion skin");
+        goto done;
+      }
+    }
+  }
+
+  control_printf_endreply(conn, 250, "EXTENDED %lu",
+                          (unsigned long)circ->global_identifier);
+  if (zero_circ) /* send a 'launched' event, for completeness */
+    circuit_event_status(circ, CIRC_EVENT_LAUNCHED, 0);
+ done:
+  SMARTLIST_FOREACH(router_nicknames, char *, n, tor_free(n));
+  smartlist_free(router_nicknames);
+  smartlist_free(nodes);
+  tor_free(path_str_alloc);
+  return 0;
+}
+
+static const control_cmd_syntax_t setcircuitpurpose_syntax = {
+  .max_args=1,
+  .accept_keywords=true,
+};
+
+/** Called when we get a SETCIRCUITPURPOSE message. If we can find the
+ * circuit and it's a valid purpose, change it. */
+static int
+handle_control_setcircuitpurpose(control_connection_t *conn,
+                                 const control_cmd_args_t *args)
+{
+  origin_circuit_t *circ = NULL;
+  uint8_t new_purpose;
+  const char *circ_id = smartlist_get(args->args,0);
+
+  if (!(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    goto done;
+  }
+
+  {
+    const config_line_t *purp = config_line_find_case(args->kwargs, "PURPOSE");
+    if (!purp) {
+      control_write_endreply(conn, 552, "No purpose given");
+      goto done;
+    }
+    new_purpose = circuit_purpose_from_string(purp->value);
+    if (new_purpose == CIRCUIT_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              purp->value);
+      goto done;
+    }
+  }
+
+  circuit_change_purpose(TO_CIRCUIT(circ), new_purpose);
+  send_control_done(conn);
+
+ done:
+  return 0;
+}
+
+static const char *attachstream_keywords[] = {
+  "HOP", NULL
+};
+static const control_cmd_syntax_t attachstream_syntax = {
+  .min_args=2, .max_args=2,
+  .accept_keywords=true,
+  .allowed_keywords=attachstream_keywords
+};
+
+/** Called when we get an ATTACHSTREAM message.  Try to attach the requested
+ * stream, and report success or failure. */
+static int
+handle_control_attachstream(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  entry_connection_t *ap_conn = NULL;
+  origin_circuit_t *circ = NULL;
+  crypt_path_t *cpath=NULL;
+  int hop=0, hop_line_ok=1;
+  const char *stream_id = smartlist_get(args->args, 0);
+  const char *circ_id = smartlist_get(args->args, 1);
+  int zero_circ = !strcmp(circ_id, "0");
+  const config_line_t *hoparg = config_line_find_case(args->kwargs, "HOP");
+
+  if (!(ap_conn = get_stream(stream_id))) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"", stream_id);
+    return 0;
+  } else if (!zero_circ && !(circ = get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  } else if (circ) {
+    if (hoparg) {
+      hop = (int) tor_parse_ulong(hoparg->value, 10, 0, INT_MAX,
+                                  &hop_line_ok, NULL);
+      if (!hop_line_ok) { /* broken hop line */
+        control_printf_endreply(conn, 552, "Bad value hop=%s",
+                                hoparg->value);
+        return 0;
+      }
+    }
+  }
+
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONNECT_WAIT &&
+      ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_RESOLVE_WAIT) {
+    control_write_endreply(conn, 555,
+                           "Connection is not managed by controller.");
+    return 0;
+  }
+
+  /* Do we need to detach it first? */
+  if (ENTRY_TO_CONN(ap_conn)->state != AP_CONN_STATE_CONTROLLER_WAIT) {
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(ap_conn);
+    circuit_t *tmpcirc = circuit_get_by_edge_conn(edge_conn);
+    connection_edge_end(edge_conn, END_STREAM_REASON_TIMEOUT);
+    /* Un-mark it as ending, since we're going to reuse it. */
+    edge_conn->edge_has_sent_end = 0;
+    edge_conn->end_reason = 0;
+    if (tmpcirc)
+      circuit_detach_stream(tmpcirc, edge_conn);
+    CONNECTION_AP_EXPECT_NONPENDING(ap_conn);
+    TO_CONN(edge_conn)->state = AP_CONN_STATE_CONTROLLER_WAIT;
+  }
+
+  if (circ && (circ->base_.state != CIRCUIT_STATE_OPEN)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to non-open origin circuit");
+    return 0;
+  }
+  /* Is this a single hop circuit? */
+  if (circ && (circuit_get_cpath_len(circ)<2 || hop==1)) {
+    control_write_endreply(conn, 551,
+                           "Can't attach stream to this one-hop circuit.");
+    return 0;
+  }
+
+  if (circ && hop>0) {
+    /* find this hop in the circuit, and set cpath */
+    cpath = circuit_get_cpath_hop(circ, hop);
+    if (!cpath) {
+      control_printf_endreply(conn, 551, "Circuit doesn't have %d hops.", hop);
+      return 0;
+    }
+  }
+  if (connection_ap_handshake_rewrite_and_attach(ap_conn, circ, cpath) < 0) {
+    control_write_endreply(conn, 551, "Unable to attach stream");
+    return 0;
+  }
+  send_control_done(conn);
+  return 0;
+}
+
+static const char *postdescriptor_keywords[] = {
+  "cache", "purpose", NULL,
+};
+
+static const control_cmd_syntax_t postdescriptor_syntax = {
+  .max_args = 0,
+  .accept_keywords = true,
+  .allowed_keywords = postdescriptor_keywords,
+  .want_cmddata = true,
+};
+
+/** Called when we get a POSTDESCRIPTOR message.  Try to learn the provided
+ * descriptor, and report success or failure. */
+static int
+handle_control_postdescriptor(control_connection_t *conn,
+                              const control_cmd_args_t *args)
+{
+  const char *msg=NULL;
+  uint8_t purpose = ROUTER_PURPOSE_GENERAL;
+  int cache = 0; /* eventually, we may switch this to 1 */
+  const config_line_t *line;
+
+  line = config_line_find_case(args->kwargs, "purpose");
+  if (line) {
+    purpose = router_purpose_from_string(line->value);
+    if (purpose == ROUTER_PURPOSE_UNKNOWN) {
+      control_printf_endreply(conn, 552, "Unknown purpose \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+  line = config_line_find_case(args->kwargs, "cache");
+  if (line) {
+    if (!strcasecmp(line->value, "no"))
+      cache = 0;
+    else if (!strcasecmp(line->value, "yes"))
+      cache = 1;
+    else {
+      control_printf_endreply(conn, 552, "Unknown cache request \"%s\"",
+                              line->value);
+      goto done;
+    }
+  }
+
+  switch (router_load_single_router(args->cmddata, purpose, cache, &msg)) {
+  case -1:
+    if (!msg) msg = "Could not parse descriptor";
+    control_write_endreply(conn, 554, msg);
+    break;
+  case 0:
+    if (!msg) msg = "Descriptor not added";
+    control_write_endreply(conn, 251, msg);
+    break;
+  case 1:
+    send_control_done(conn);
+    break;
+  }
+
+ done:
+  return 0;
+}
+
+static const control_cmd_syntax_t redirectstream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, // XXX should be 3.
+};
+
+/** Called when we receive a REDIRECTSTERAM command.  Try to change the target
+ * address of the named AP stream, and report success or failure. */
+static int
+handle_control_redirectstream(control_connection_t *conn,
+                              const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn = NULL;
+  char *new_addr = NULL;
+  uint16_t new_port = 0;
+  const smartlist_t *args = cmd_args->args;
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0)))
+           || !ap_conn->socks_request) {
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  } else {
+    int ok = 1;
+    if (smartlist_len(args) > 2) { /* they included a port too */
+      new_port = (uint16_t) tor_parse_ulong(smartlist_get(args, 2),
+                                            10, 1, 65535, &ok, NULL);
+    }
+    if (!ok) {
+      control_printf_endreply(conn, 512, "Cannot parse port \"%s\"",
+                              (char*)smartlist_get(args, 2));
+    } else {
+      new_addr = tor_strdup(smartlist_get(args, 1));
+    }
+  }
+
+  if (!new_addr)
+    return 0;
+
+  strlcpy(ap_conn->socks_request->address, new_addr,
+          sizeof(ap_conn->socks_request->address));
+  if (new_port)
+    ap_conn->socks_request->port = new_port;
+  tor_free(new_addr);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closestream_syntax = {
+  .min_args = 2,
+  .max_args = UINT_MAX, /* XXXX This is the original behavior, but
+                         * maybe we should change the spec. */
+};
+
+/** Called when we get a CLOSESTREAM command; try to close the named stream
+ * and report success or failure. */
+static int
+handle_control_closestream(control_connection_t *conn,
+                           const control_cmd_args_t *cmd_args)
+{
+  entry_connection_t *ap_conn=NULL;
+  uint8_t reason=0;
+  int ok;
+  const smartlist_t *args = cmd_args->args;
+
+  tor_assert(smartlist_len(args) >= 2);
+
+  if (!(ap_conn = get_stream(smartlist_get(args, 0))))
+    control_printf_endreply(conn, 552, "Unknown stream \"%s\"",
+                            (char*)smartlist_get(args, 0));
+  else {
+    reason = (uint8_t) tor_parse_ulong(smartlist_get(args,1), 10, 0, 255,
+                                       &ok, NULL);
+    if (!ok) {
+      control_printf_endreply(conn, 552, "Unrecognized reason \"%s\"",
+                              (char*)smartlist_get(args, 1));
+      ap_conn = NULL;
+    }
+  }
+  if (!ap_conn)
+    return 0;
+
+  connection_mark_unattached_ap(ap_conn, reason);
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t closecircuit_syntax = {
+  .min_args=1, .max_args=1,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+  // XXXX we might want to exclude unrecognized flags, but for now we
+  // XXXX just ignore them for backward compatibility.
+};
+
+/** Called when we get a CLOSECIRCUIT command; try to close the named circuit
+ * and report success or failure. */
+static int
+handle_control_closecircuit(control_connection_t *conn,
+                            const control_cmd_args_t *args)
+{
+  const char *circ_id = smartlist_get(args->args, 0);
+  origin_circuit_t *circ = NULL;
+
+  if (!(circ=get_circ(circ_id))) {
+    control_printf_endreply(conn, 552, "Unknown circuit \"%s\"", circ_id);
+    return 0;
+  }
+
+  bool safe =  config_lines_contain_flag(args->kwargs, "IfUnused");
+
+  if (!safe || !circ->p_streams) {
+    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_REQUESTED);
+  }
+
+  send_control_done(conn);
+  return 0;
+}
+
+static const control_cmd_syntax_t resolve_syntax = {
+  .max_args=0,
+  .accept_keywords=true,
+  .kvline_flags=KV_OMIT_VALS,
+};
+
+/** Called when we get a RESOLVE command: start trying to resolve
+ * the listed addresses. */
+static int
+handle_control_resolve(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  smartlist_t *failed;
+  int is_reverse = 0;
+
+  if (!(conn->event_mask & (((event_mask_t)1)<<EVENT_ADDRMAP))) {
+    log_warn(LD_CONTROL, "Controller asked us to resolve an address, but "
+             "isn't listening for ADDRMAP events.  It probably won't see "
+             "the answer.");
+  }
+
+  {
+    const config_line_t *modearg = config_line_find_case(args->kwargs, "mode");
+    if (modearg && !strcasecmp(modearg->value, "reverse"))
+      is_reverse = 1;
+  }
+  failed = smartlist_new();
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strlen(line->value)) {
+      const char *addr = line->key;
+      if (dnsserv_launch_request(addr, is_reverse, conn)<0)
+        smartlist_add(failed, (char*)addr);
+    } else {
+      // XXXX arguably we should reject unrecognized keyword arguments,
+      // XXXX but the old implementation didn't do that.
+    }
+  }
+
+  send_control_done(conn);
+  SMARTLIST_FOREACH(failed, const char *, arg, {
+      control_event_address_mapped(arg, arg, time(NULL),
+                                   "internal", 0);
+  });
+
+  smartlist_free(failed);
+  return 0;
+}
+
+static const control_cmd_syntax_t protocolinfo_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Return a comma-separated list of authentication methods for
+    handle_control_protocolinfo().  Caller must free this string. */
+static char *
+get_authmethods(const or_options_t *options)
+{
+  int cookies = options->CookieAuthentication;
+  char *methods;
+  int passwd = (options->HashedControlPassword != NULL ||
+                options->HashedControlSessionPassword != NULL);
+  smartlist_t *mlist = smartlist_new();
+
+  if (cookies) {
+    smartlist_add(mlist, (char*)"COOKIE");
+    smartlist_add(mlist, (char*)"SAFECOOKIE");
+  }
+  if (passwd)
+    smartlist_add(mlist, (char*)"HASHEDPASSWORD");
+  if (!cookies && !passwd)
+    smartlist_add(mlist, (char*)"NULL");
+  methods = smartlist_join_strings(mlist, ",", 0, NULL);
+  smartlist_free(mlist);
+
+  return methods;
+}
+
+/** Return escaped cookie filename.  Caller must free this string.
+    Return NULL if cookie authentication is disabled. */
+static char *
+get_esc_cfile(const or_options_t *options)
+{
+  char *cfile = NULL, *abs_cfile = NULL, *esc_cfile = NULL;
+
+  if (!options->CookieAuthentication)
+    return NULL;
+
+  cfile = get_controller_cookie_file_name();
+  abs_cfile = make_path_absolute(cfile);
+  esc_cfile = esc_for_log(abs_cfile);
+  tor_free(cfile);
+  tor_free(abs_cfile);
+  return esc_cfile;
+}
+
+/** Compose the auth methods line of a PROTOCOLINFO reply. */
+static void
+add_authmethods(smartlist_t *reply)
+{
+  const or_options_t *options = get_options();
+  char *methods = get_authmethods(options);
+  char *esc_cfile = get_esc_cfile(options);
+
+  control_reply_add_str(reply, 250, "AUTH");
+  control_reply_append_kv(reply, "METHODS", methods);
+  if (esc_cfile)
+    control_reply_append_kv(reply, "COOKIEFILE", esc_cfile);
+
+  tor_free(methods);
+  tor_free(esc_cfile);
+}
+
+/** Called when we get a PROTOCOLINFO command: send back a reply. */
+static int
+handle_control_protocolinfo(control_connection_t *conn,
+                            const control_cmd_args_t *cmd_args)
+{
+  const char *bad_arg = NULL;
+  const smartlist_t *args = cmd_args->args;
+  smartlist_t *reply = NULL;
+
+  conn->have_sent_protocolinfo = 1;
+
+  SMARTLIST_FOREACH(args, const char *, arg, {
+      int ok;
+      tor_parse_long(arg, 10, 0, LONG_MAX, &ok, NULL);
+      if (!ok) {
+        bad_arg = arg;
+        break;
+      }
+    });
+  if (bad_arg) {
+    control_printf_endreply(conn, 513, "No such version %s",
+                            escaped(bad_arg));
+    /* Don't tolerate bad arguments when not authenticated. */
+    if (!STATE_IS_OPEN(TO_CONN(conn)->state))
+      connection_mark_for_close(TO_CONN(conn));
+    return 0;
+  }
+  reply = smartlist_new();
+  control_reply_add_str(reply, 250, "PROTOCOLINFO 1");
+  add_authmethods(reply);
+  control_reply_add_str(reply, 250, "VERSION");
+  control_reply_append_kv(reply, "Tor", escaped(VERSION));
+  control_reply_add_done(reply);
+
+  control_write_reply_lines(conn, reply);
+  control_reply_free(reply);
+  return 0;
+}
+
+static const control_cmd_syntax_t usefeature_syntax = {
+  .max_args = UINT_MAX
+};
+
+/** Called when we get a USEFEATURE command: parse the feature list, and
+ * set up the control_connection's options properly. */
+static int
+handle_control_usefeature(control_connection_t *conn,
+                          const control_cmd_args_t *cmd_args)
+{
+  const smartlist_t *args = cmd_args->args;
+  int bad = 0;
+  SMARTLIST_FOREACH_BEGIN(args, const char *, arg) {
+      if (!strcasecmp(arg, "VERBOSE_NAMES"))
+        ;
+      else if (!strcasecmp(arg, "EXTENDED_EVENTS"))
+        ;
+      else {
+        control_printf_endreply(conn, 552, "Unrecognized feature \"%s\"",
+                                arg);
+        bad = 1;
+        break;
+      }
+  } SMARTLIST_FOREACH_END(arg);
+
+  if (!bad) {
+    send_control_done(conn);
+  }
+
+  return 0;
+}
+
+static const control_cmd_syntax_t dropguards_syntax = {
+  .max_args = 0,
+};
+
+/** Implementation for the DROPGUARDS command. */
+static int
+handle_control_dropguards(control_connection_t *conn,
+                          const control_cmd_args_t *args)
+{
+  (void) args; /* We don't take arguments. */
+
+  static int have_warned = 0;
+  if (! have_warned) {
+    log_warn(LD_CONTROL, "DROPGUARDS is dangerous; make sure you understand "
+             "the risks before using it. It may be removed in a future "
+             "version of Tor.");
+    have_warned = 1;
+  }
+
+  remove_all_entry_guards();
+  send_control_done(conn);
+
+  return 0;
+}
+
+static const char *hsfetch_keywords[] = {
+  "SERVER", NULL,
+};
+static const control_cmd_syntax_t hsfetch_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = hsfetch_keywords,
+};
+
+/** Implementation for the HSFETCH command. */
+static int
+handle_control_hsfetch(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+
+{
+  char digest[DIGEST_LEN], *desc_id = NULL;
+  smartlist_t *hsdirs = NULL;
+  static const char *v2_str = "v2-";
+  const size_t v2_str_len = strlen(v2_str);
+  rend_data_t *rend_query = NULL;
+  ed25519_public_key_t v3_pk;
+  uint32_t version;
+  const char *hsaddress = NULL;
+
+  /* Extract the first argument (either HSAddress or DescID). */
+  const char *arg1 = smartlist_get(args->args, 0);
+  /* Test if it's an HS address without the .onion part. */
+  if (rend_valid_v2_service_id(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_TWO;
+  } else if (strcmpstart(arg1, v2_str) == 0 &&
+             rend_valid_descriptor_id(arg1 + v2_str_len) &&
+             base32_decode(digest, sizeof(digest), arg1 + v2_str_len,
+                           REND_DESC_ID_V2_LEN_BASE32) ==
+                sizeof(digest)) {
+    /* We have a well formed version 2 descriptor ID. Keep the decoded value
+     * of the id. */
+    desc_id = digest;
+    version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(arg1)) {
+    hsaddress = arg1;
+    version = HS_VERSION_THREE;
+    hs_parse_address(hsaddress, &v3_pk, NULL, NULL);
+  } else {
+    control_printf_endreply(conn, 513, "Invalid argument \"%s\"", arg1);
+    goto done;
+  }
+
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strcasecmp(line->key, "SERVER")) {
+      const char *server = line->value;
+
+      const node_t *node = node_get_by_hex_id(server, 0);
+      if (!node) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found", server);
+        goto done;
+      }
+      if (!hsdirs) {
+        /* Stores routerstatus_t cmddata for each specified server. */
+        hsdirs = smartlist_new();
+      }
+      /* Valid server, add it to our local list. */
+      smartlist_add(hsdirs, node->rs);
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  if (version == HS_VERSION_TWO) {
+    rend_query = rend_data_client_create(hsaddress, desc_id, NULL,
+                                         REND_NO_AUTH);
+    if (rend_query == NULL) {
+      control_write_endreply(conn, 551, "Error creating the HS query");
+      goto done;
+    }
+  }
+
+  /* Using a descriptor ID, we force the user to provide at least one
+   * hsdir server using the SERVER= option. */
+  if (desc_id && (!hsdirs || !smartlist_len(hsdirs))) {
+    control_write_endreply(conn, 512, "SERVER option is required");
+    goto done;
+  }
+
+  /* We are about to trigger HSDir fetch so send the OK now because after
+   * that 650 event(s) are possible so better to have the 250 OK before them
+   * to avoid out of order replies. */
+  send_control_done(conn);
+
+  /* Trigger the fetch using the built rend query and possibly a list of HS
+   * directory to use. This function ignores the client cache thus this will
+   * always send a fetch command. */
+  if (version == HS_VERSION_TWO) {
+    rend_client_fetch_v2_desc(rend_query, hsdirs);
+  } else if (version == HS_VERSION_THREE) {
+    hs_control_hsfetch_command(&v3_pk, hsdirs);
+  }
+
+ done:
+  /* Contains data pointer that we don't own thus no cleanup. */
+  smartlist_free(hsdirs);
+  rend_data_free(rend_query);
+  return 0;
+}
+
+static const char *hspost_keywords[] = {
+  "SERVER", "HSADDRESS", NULL
+};
+static const control_cmd_syntax_t hspost_syntax = {
+  .min_args = 0, .max_args = 0,
+  .accept_keywords = true,
+  .want_cmddata = true,
+  .allowed_keywords = hspost_keywords
+};
+
+/** Implementation for the HSPOST command. */
+static int
+handle_control_hspost(control_connection_t *conn,
+                      const control_cmd_args_t *args)
+{
+  smartlist_t *hs_dirs = NULL;
+  const char *encoded_desc = args->cmddata;
+  size_t encoded_desc_len = args->cmddata_len;
+  const char *onion_address = NULL;
+  const config_line_t *line;
+
+  for (line = args->kwargs; line; line = line->next) {
+    if (!strcasecmpstart(line->key, "SERVER")) {
+      const char *server = line->value;
+      const node_t *node = node_get_by_hex_id(server, 0);
+
+      if (!node || !node->rs) {
+        control_printf_endreply(conn, 552, "Server \"%s\" not found",
+                                server);
+        goto done;
+      }
+      /* Valid server, add it to our local list. */
+      if (!hs_dirs)
+        hs_dirs = smartlist_new();
+      smartlist_add(hs_dirs, node->rs);
+    } else if (!strcasecmpstart(line->key, "HSADDRESS")) {
+      const char *address = line->value;
+      if (!hs_address_is_valid(address)) {
+        control_write_endreply(conn, 512, "Malformed onion address");
+        goto done;
+      }
+      onion_address = address;
+    } else {
+      tor_assert_nonfatal_unreached();
+    }
+  }
+
+  /* Handle the v3 case. */
+  if (onion_address) {
+    if (hs_control_hspost_command(encoded_desc, onion_address, hs_dirs) < 0) {
+      control_write_endreply(conn, 554, "Invalid descriptor");
+    } else {
+      send_control_done(conn);
+    }
+    goto done;
+  }
+
+  /* From this point on, it is only v2. */
+
+  /*  parse it. */
+  rend_encoded_v2_service_descriptor_t *desc =
+      tor_malloc_zero(sizeof(rend_encoded_v2_service_descriptor_t));
+  desc->desc_str = tor_memdup_nulterm(encoded_desc, encoded_desc_len);
+
+  rend_service_descriptor_t *parsed = NULL;
+  char *intro_content = NULL;
+  size_t intro_size;
+  size_t encoded_size;
+  const char *next_desc;
+  if (!rend_parse_v2_service_descriptor(&parsed, desc->desc_id, &intro_content,
+                                        &intro_size, &encoded_size,
+                                        &next_desc, desc->desc_str, 1)) {
+    /* Post the descriptor. */
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    if (!rend_get_service_id(parsed->pk, serviceid)) {
+      smartlist_t *descs = smartlist_new();
+      smartlist_add(descs, desc);
+
+      /* We are about to trigger HS descriptor upload so send the OK now
+       * because after that 650 event(s) are possible so better to have the
+       * 250 OK before them to avoid out of order replies. */
+      send_control_done(conn);
+
+      /* Trigger the descriptor upload */
+      directory_post_to_hs_dir(parsed, descs, hs_dirs, serviceid, 0);
+      smartlist_free(descs);
+    }
+
+    rend_service_descriptor_free(parsed);
+  } else {
+    control_write_endreply(conn, 554, "Invalid descriptor");
+  }
+
+  tor_free(intro_content);
+  rend_encoded_v2_service_descriptor_free(desc);
+ done:
+  smartlist_free(hs_dirs); /* Contents belong to the rend service code. */
+  return 0;
+}
+
+/* Helper function for ADD_ONION that adds an ephemeral service depending on
+ * the given hs_version.
+ *
+ * The secret key in pk depends on the hs_version. The ownership of the key
+ * used in pk is given to the HS subsystem so the caller must stop accessing
+ * it after.
+ *
+ * The port_cfgs is a list of service port. Ownership transferred to service.
+ * The max_streams refers to the MaxStreams= key.
+ * The max_streams_close_circuit refers to the MaxStreamsCloseCircuit key.
+ * The auth_type is the authentication type of the clients in auth_clients.
+ * The ownership of that list is transferred to the service.
+ *
+ * On success (RSAE_OKAY), the address_out points to a newly allocated string
+ * containing the onion address without the .onion part. On error, address_out
+ * is untouched. */
+static hs_service_add_ephemeral_status_t
+add_onion_helper_add_service(int hs_version,
+                             add_onion_secret_key_t *pk,
+                             smartlist_t *port_cfgs, int max_streams,
+                             int max_streams_close_circuit, int auth_type,
+                             smartlist_t *auth_clients, char **address_out)
+{
+  hs_service_add_ephemeral_status_t ret;
+
+  tor_assert(pk);
+  tor_assert(port_cfgs);
+  tor_assert(address_out);
+
+  switch (hs_version) {
+  case HS_VERSION_TWO:
+    ret = rend_service_add_ephemeral(pk->v2, port_cfgs, max_streams,
+                                     max_streams_close_circuit, auth_type,
+                                     auth_clients, address_out);
+    break;
+  case HS_VERSION_THREE:
+    ret = hs_service_add_ephemeral(pk->v3, port_cfgs, max_streams,
+                                   max_streams_close_circuit, address_out);
+    break;
+  default:
+    tor_assert_unreached();
+  }
+
+  return ret;
+}
+
+/** The list of onion services that have been added via ADD_ONION that do not
+ * belong to any particular control connection.
+ */
+static smartlist_t *detached_onion_services = NULL;
+
+/**
+ * Return a list of detached onion services, or NULL if none exist.
+ **/
+smartlist_t *
+get_detached_onion_services(void)
+{
+  return detached_onion_services;
+}
+
+static const char *add_onion_keywords[] = {
+   "Port", "Flags", "MaxStreams", "ClientAuth", NULL
+};
+static const control_cmd_syntax_t add_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+  .accept_keywords = true,
+  .allowed_keywords = add_onion_keywords
+};
+
+/** Called when we get a ADD_ONION command; parse the body, and set up
+ * the new ephemeral Onion Service. */
+static int
+handle_control_add_onion(control_connection_t *conn,
+                         const control_cmd_args_t *args)
+{
+  /* Parse all of the arguments that do not involve handling cryptographic
+   * material first, since there's no reason to touch that at all if any of
+   * the other arguments are malformed.
+   */
+  smartlist_t *port_cfgs = smartlist_new();
+  smartlist_t *auth_clients = NULL;
+  smartlist_t *auth_created_clients = NULL;
+  int discard_pk = 0;
+  int detach = 0;
+  int max_streams = 0;
+  int max_streams_close_circuit = 0;
+  rend_auth_type_t auth_type = REND_NO_AUTH;
+  int non_anonymous = 0;
+  const config_line_t *arg;
+
+  for (arg = args->kwargs; arg; arg = arg->next) {
+    if (!strcasecmp(arg->key, "Port")) {
+      /* "Port=VIRTPORT[,TARGET]". */
+      rend_service_port_config_t *cfg =
+          rend_service_parse_port_config(arg->value, ",", NULL);
+      if (!cfg) {
+        control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+        goto out;
+      }
+      smartlist_add(port_cfgs, cfg);
+    } else if (!strcasecmp(arg->key, "MaxStreams")) {
+      /* "MaxStreams=[0..65535]". */
+      int ok = 0;
+      max_streams = (int)tor_parse_long(arg->value, 10, 0, 65535, &ok, NULL);
+      if (!ok) {
+        control_write_endreply(conn, 512, "Invalid MaxStreams");
+        goto out;
+      }
+    } else if (!strcasecmp(arg->key, "Flags")) {
+      /* "Flags=Flag[,Flag]", where Flag can be:
+       *   * 'DiscardPK' - If tor generates the keypair, do not include it in
+       *                   the response.
+       *   * 'Detach' - Do not tie this onion service to any particular control
+       *                connection.
+       *   * 'MaxStreamsCloseCircuit' - Close the circuit if MaxStreams is
+       *                                exceeded.
+       *   * 'BasicAuth' - Client authorization using the 'basic' method.
+       *   * 'NonAnonymous' - Add a non-anonymous Single Onion Service. If this
+       *                      flag is present, tor must be in non-anonymous
+       *                      hidden service mode. If this flag is absent,
+       *                      tor must be in anonymous hidden service mode.
+       */
+      static const char *discard_flag = "DiscardPK";
+      static const char *detach_flag = "Detach";
+      static const char *max_s_close_flag = "MaxStreamsCloseCircuit";
+      static const char *basicauth_flag = "BasicAuth";
+      static const char *non_anonymous_flag = "NonAnonymous";
+
+      smartlist_t *flags = smartlist_new();
+      int bad = 0;
+
+      smartlist_split_string(flags, arg->value, ",", SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        control_write_endreply(conn, 512, "Invalid 'Flags' argument");
+        bad = 1;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag)
+      {
+        if (!strcasecmp(flag, discard_flag)) {
+          discard_pk = 1;
+        } else if (!strcasecmp(flag, detach_flag)) {
+          detach = 1;
+        } else if (!strcasecmp(flag, max_s_close_flag)) {
+          max_streams_close_circuit = 1;
+        } else if (!strcasecmp(flag, basicauth_flag)) {
+          auth_type = REND_BASIC_AUTH;
+        } else if (!strcasecmp(flag, non_anonymous_flag)) {
+          non_anonymous = 1;
+        } else {
+          control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s",
+                                  escaped(flag));
+          bad = 1;
+          break;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+      SMARTLIST_FOREACH(flags, char *, cp, tor_free(cp));
+      smartlist_free(flags);
+      if (bad)
+        goto out;
+
+    } else if (!strcasecmp(arg->key, "ClientAuth")) {
+      int created = 0;
+      rend_authorized_client_t *client =
+        add_onion_helper_clientauth(arg->value, &created, conn);
+      if (!client) {
+        goto out;
+      }
+
+      if (auth_clients != NULL) {
+        int bad = 0;
+        SMARTLIST_FOREACH_BEGIN(auth_clients, rend_authorized_client_t *, ac) {
+          if (strcmp(ac->client_name, client->client_name) == 0) {
+            bad = 1;
+            break;
+          }
+        } SMARTLIST_FOREACH_END(ac);
+        if (bad) {
+          control_write_endreply(conn, 512, "Duplicate name in ClientAuth");
+          rend_authorized_client_free(client);
+          goto out;
+        }
+      } else {
+        auth_clients = smartlist_new();
+        auth_created_clients = smartlist_new();
+      }
+      smartlist_add(auth_clients, client);
+      if (created) {
+        smartlist_add(auth_created_clients, client);
+      }
+    } else {
+      tor_assert_nonfatal_unreached();
+      goto out;
+    }
+  }
+  if (smartlist_len(port_cfgs) == 0) {
+    control_write_endreply(conn, 512, "Missing 'Port' argument");
+    goto out;
+  } else if (auth_type == REND_NO_AUTH && auth_clients != NULL) {
+    control_write_endreply(conn, 512, "No auth type specified");
+    goto out;
+  } else if (auth_type != REND_NO_AUTH && auth_clients == NULL) {
+    control_write_endreply(conn, 512, "No auth clients specified");
+    goto out;
+  } else if ((auth_type == REND_BASIC_AUTH &&
+              smartlist_len(auth_clients) > 512) ||
+             (auth_type == REND_STEALTH_AUTH &&
+              smartlist_len(auth_clients) > 16)) {
+    control_write_endreply(conn, 512, "Too many auth clients");
+    goto out;
+  } else if (non_anonymous != rend_service_non_anonymous_mode_enabled(
+                                                              get_options())) {
+    /* If we failed, and the non-anonymous flag is set, Tor must be in
+     * anonymous hidden service mode.
+     * The error message changes based on the current Tor config:
+     * 512 Tor is in anonymous hidden service mode
+     * 512 Tor is in non-anonymous hidden service mode
+     * (I've deliberately written them out in full here to aid searchability.)
+     */
+    control_printf_endreply(conn, 512,
+                            "Tor is in %sanonymous hidden service " "mode",
+                            non_anonymous ? "" : "non-");
+    goto out;
+  }
+
+  /* Parse the "keytype:keyblob" argument. */
+  int hs_version = 0;
+  add_onion_secret_key_t pk = { NULL };
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+
+  const char *onionkey = smartlist_get(args->args, 0);
+  if (add_onion_helper_keyarg(onionkey, discard_pk,
+                              &key_new_alg, &key_new_blob, &pk, &hs_version,
+                              conn) < 0) {
+    goto out;
+  }
+
+  /* Hidden service version 3 don't have client authentication support so if
+   * ClientAuth was given, send back an error. */
+  if (hs_version == HS_VERSION_THREE && auth_clients) {
+    control_write_endreply(conn, 513, "ClientAuth not supported");
+    goto out;
+  }
+
+  /* Create the HS, using private key pk, client authentication auth_type,
+   * the list of auth_clients, and port config port_cfg.
+   * rend_service_add_ephemeral() will take ownership of pk and port_cfg,
+   * regardless of success/failure.
+   */
+  char *service_id = NULL;
+  int ret = add_onion_helper_add_service(hs_version, &pk, port_cfgs,
+                                         max_streams,
+                                         max_streams_close_circuit, auth_type,
+                                         auth_clients, &service_id);
+  port_cfgs = NULL; /* port_cfgs is now owned by the rendservice code. */
+  auth_clients = NULL; /* so is auth_clients */
+  switch (ret) {
+  case RSAE_OKAY:
+  {
+    if (detach) {
+      if (!detached_onion_services)
+        detached_onion_services = smartlist_new();
+      smartlist_add(detached_onion_services, service_id);
+    } else {
+      if (!conn->ephemeral_onion_services)
+        conn->ephemeral_onion_services = smartlist_new();
+      smartlist_add(conn->ephemeral_onion_services, service_id);
+    }
+
+    tor_assert(service_id);
+    control_printf_midreply(conn, 250, "ServiceID=%s", service_id);
+    if (key_new_alg) {
+      tor_assert(key_new_blob);
+      control_printf_midreply(conn, 250, "PrivateKey=%s:%s",
+                              key_new_alg, key_new_blob);
+    }
+    if (auth_created_clients) {
+      SMARTLIST_FOREACH(auth_created_clients, rend_authorized_client_t *, ac, {
+        char *encoded = rend_auth_encode_cookie(ac->descriptor_cookie,
+                                                auth_type);
+        tor_assert(encoded);
+        control_printf_midreply(conn, 250, "ClientAuth=%s:%s",
+                                ac->client_name, encoded);
+        memwipe(encoded, 0, strlen(encoded));
+        tor_free(encoded);
+      });
+    }
+
+    send_control_done(conn);
+    break;
+  }
+  case RSAE_BADPRIVKEY:
+    control_write_endreply(conn, 551, "Failed to generate onion address");
+    break;
+  case RSAE_ADDREXISTS:
+    control_write_endreply(conn, 550, "Onion address collision");
+    break;
+  case RSAE_BADVIRTPORT:
+    control_write_endreply(conn, 512, "Invalid VIRTPORT/TARGET");
+    break;
+  case RSAE_BADAUTH:
+    control_write_endreply(conn, 512, "Invalid client authorization");
+    break;
+  case RSAE_INTERNAL: FALLTHROUGH;
+  default:
+    control_write_endreply(conn, 551, "Failed to add Onion Service");
+  }
+  if (key_new_blob) {
+    memwipe(key_new_blob, 0, strlen(key_new_blob));
+    tor_free(key_new_blob);
+  }
+
+ out:
+  if (port_cfgs) {
+    SMARTLIST_FOREACH(port_cfgs, rend_service_port_config_t*, p,
+                      rend_service_port_config_free(p));
+    smartlist_free(port_cfgs);
+  }
+
+  if (auth_clients) {
+    SMARTLIST_FOREACH(auth_clients, rend_authorized_client_t *, ac,
+                      rend_authorized_client_free(ac));
+    smartlist_free(auth_clients);
+  }
+  if (auth_created_clients) {
+    // Do not free entries; they are the same as auth_clients
+    smartlist_free(auth_created_clients);
+  }
+  return 0;
+}
+
+/** Helper function to handle parsing the KeyType:KeyBlob argument to the
+ * ADD_ONION command. Return a new crypto_pk_t and if a new key was generated
+ * and the private key not discarded, the algorithm and serialized private key,
+ * or NULL and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned key_new_blob.
+ *
+ * Note: The error messages returned are deliberately vague to avoid echoing
+ * key material.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC int
+add_onion_helper_keyarg(const char *arg, int discard_pk,
+                        const char **key_new_alg_out, char **key_new_blob_out,
+                        add_onion_secret_key_t *decoded_key, int *hs_version,
+                        control_connection_t *conn)
+{
+  smartlist_t *key_args = smartlist_new();
+  crypto_pk_t *pk = NULL;
+  const char *key_new_alg = NULL;
+  char *key_new_blob = NULL;
+  int ret = -1;
+
+  smartlist_split_string(key_args, arg, ":", SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    control_write_endreply(conn, 512, "Invalid key type/blob");
+    goto err;
+  }
+
+  /* The format is "KeyType:KeyBlob". */
+  static const char *key_type_new = "NEW";
+  static const char *key_type_best = "BEST";
+  static const char *key_type_rsa1024 = "RSA1024";
+  static const char *key_type_ed25519_v3 = "ED25519-V3";
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (!strcasecmp(key_type_rsa1024, key_type)) {
+    /* "RSA:<Base64 Blob>" - Loading a pre-existing RSA1024 key. */
+    pk = crypto_pk_base64_decode_private(key_blob, strlen(key_blob));
+    if (!pk) {
+      control_write_endreply(conn, 512, "Failed to decode RSA key");
+      goto err;
+    }
+    if (crypto_pk_num_bits(pk) != PK_BYTES*8) {
+      crypto_pk_free(pk);
+      control_write_endreply(conn, 512, "Invalid RSA key size");
+      goto err;
+    }
+    decoded_key->v2 = pk;
+    *hs_version = HS_VERSION_TWO;
+  } else if (!strcasecmp(key_type_ed25519_v3, key_type)) {
+    /* parsing of private ed25519 key */
+    /* "ED25519-V3:<Base64 Blob>" - Loading a pre-existing ed25519 key. */
+    ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+    if (base64_decode((char *) sk->seckey, sizeof(sk->seckey), key_blob,
+                      strlen(key_blob)) != sizeof(sk->seckey)) {
+      tor_free(sk);
+      control_write_endreply(conn, 512, "Failed to decode ED25519-V3 key");
+      goto err;
+    }
+    decoded_key->v3 = sk;
+    *hs_version = HS_VERSION_THREE;
+  } else if (!strcasecmp(key_type_new, key_type)) {
+    /* "NEW:<Algorithm>" - Generating a new key, blob as algorithm. */
+    if (!strcasecmp(key_type_rsa1024, key_blob)) {
+      /* "RSA1024", RSA 1024 bit, also currently "BEST" by default. */
+      pk = crypto_pk_new();
+      if (crypto_pk_generate_key(pk)) {
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_rsa1024);
+        goto err;
+      }
+      if (!discard_pk) {
+        if (crypto_pk_base64_encode_private(pk, &key_new_blob)) {
+          crypto_pk_free(pk);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_rsa1024);
+          goto err;
+        }
+        key_new_alg = key_type_rsa1024;
+      }
+      decoded_key->v2 = pk;
+      *hs_version = HS_VERSION_TWO;
+    } else if (!strcasecmp(key_type_ed25519_v3, key_blob) ||
+               !strcasecmp(key_type_best, key_blob)) {
+      /* "ED25519-V3", ed25519 key, also currently "BEST" by default. */
+      ed25519_secret_key_t *sk = tor_malloc_zero(sizeof(*sk));
+      if (ed25519_secret_key_generate(sk, 1) < 0) {
+        tor_free(sk);
+        control_printf_endreply(conn, 551, "Failed to generate %s key",
+                                key_type_ed25519_v3);
+        goto err;
+      }
+      if (!discard_pk) {
+        ssize_t len = base64_encode_size(sizeof(sk->seckey), 0) + 1;
+        key_new_blob = tor_malloc_zero(len);
+        if (base64_encode(key_new_blob, len, (const char *) sk->seckey,
+                          sizeof(sk->seckey), 0) != (len - 1)) {
+          tor_free(sk);
+          tor_free(key_new_blob);
+          control_printf_endreply(conn, 551, "Failed to encode %s key",
+                                  key_type_ed25519_v3);
+          goto err;
+        }
+        key_new_alg = key_type_ed25519_v3;
+      }
+      decoded_key->v3 = sk;
+      *hs_version = HS_VERSION_THREE;
+    } else {
+      control_write_endreply(conn, 513, "Invalid key type");
+      goto err;
+    }
+  } else {
+    control_write_endreply(conn, 513, "Invalid key type");
+    goto err;
+  }
+
+  /* Succeeded in loading or generating a private key. */
+  ret = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, cp, {
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+  });
+  smartlist_free(key_args);
+
+  *key_new_alg_out = key_new_alg;
+  *key_new_blob_out = key_new_blob;
+
+  return ret;
+}
+
+/** Helper function to handle parsing a ClientAuth argument to the
+ * ADD_ONION command.  Return a new rend_authorized_client_t, or NULL
+ * and an optional control protocol error message on failure.  The
+ * caller is responsible for freeing the returned auth_client.
+ *
+ * If 'created' is specified, it will be set to 1 when a new cookie has
+ * been generated.
+ *
+ * Note: conn is only used for writing control replies. For testing
+ * purposes, it can be NULL if control_write_reply() is appropriately
+ * mocked.
+ */
+STATIC rend_authorized_client_t *
+add_onion_helper_clientauth(const char *arg, int *created,
+                            control_connection_t *conn)
+{
+  int ok = 0;
+
+  tor_assert(arg);
+  tor_assert(created);
+
+  smartlist_t *auth_args = smartlist_new();
+  rend_authorized_client_t *client =
+    tor_malloc_zero(sizeof(rend_authorized_client_t));
+  smartlist_split_string(auth_args, arg, ":", 0, 0);
+  if (smartlist_len(auth_args) < 1 || smartlist_len(auth_args) > 2) {
+    control_write_endreply(conn, 512, "Invalid ClientAuth syntax");
+    goto err;
+  }
+  client->client_name = tor_strdup(smartlist_get(auth_args, 0));
+  if (smartlist_len(auth_args) == 2) {
+    char *decode_err_msg = NULL;
+    if (rend_auth_decode_cookie(smartlist_get(auth_args, 1),
+                                client->descriptor_cookie,
+                                NULL, &decode_err_msg) < 0) {
+      tor_assert(decode_err_msg);
+      control_write_endreply(conn, 512, decode_err_msg);
+      tor_free(decode_err_msg);
+      goto err;
+    }
+    *created = 0;
+  } else {
+    crypto_rand((char *) client->descriptor_cookie, REND_DESC_COOKIE_LEN);
+    *created = 1;
+  }
+
+  if (!rend_valid_client_name(client->client_name)) {
+    control_write_endreply(conn, 512, "Invalid name in ClientAuth");
+    goto err;
+  }
+
+  ok = 1;
+ err:
+  SMARTLIST_FOREACH(auth_args, char *, item, tor_free(item));
+  smartlist_free(auth_args);
+  if (!ok) {
+    rend_authorized_client_free(client);
+    client = NULL;
+  }
+  return client;
+}
+
+static const control_cmd_syntax_t del_onion_syntax = {
+  .min_args = 1, .max_args = 1,
+};
+
+/** Called when we get a DEL_ONION command; parse the body, and remove
+ * the existing ephemeral Onion Service. */
+static int
+handle_control_del_onion(control_connection_t *conn,
+                         const control_cmd_args_t *cmd_args)
+{
+  int hs_version = 0;
+  smartlist_t *args = cmd_args->args;
+  tor_assert(smartlist_len(args) == 1);
+
+  const char *service_id = smartlist_get(args, 0);
+  if (rend_valid_v2_service_id(service_id)) {
+    hs_version = HS_VERSION_TWO;
+  } else if (hs_address_is_valid(service_id)) {
+    hs_version = HS_VERSION_THREE;
+  } else {
+    control_write_endreply(conn, 512, "Malformed Onion Service id");
+    goto out;
+  }
+
+  /* Determine if the onion service belongs to this particular control
+   * connection, or if it is in the global list of detached services.  If it
+   * is in neither, either the service ID is invalid in some way, or it
+   * explicitly belongs to a different control connection, and an error
+   * should be returned.
+   */
+  smartlist_t *services[2] = {
+    conn->ephemeral_onion_services,
+    detached_onion_services
+  };
+  smartlist_t *onion_services = NULL;
+  int idx = -1;
+  for (size_t i = 0; i < ARRAY_LENGTH(services); i++) {
+    idx = smartlist_string_pos(services[i], service_id);
+    if (idx != -1) {
+      onion_services = services[i];
+      break;
+    }
+  }
+  if (onion_services == NULL) {
+    control_write_endreply(conn, 552, "Unknown Onion Service id");
+  } else {
+    int ret = -1;
+    switch (hs_version) {
+    case HS_VERSION_TWO:
+      ret = rend_service_del_ephemeral(service_id);
+      break;
+    case HS_VERSION_THREE:
+      ret = hs_service_del_ephemeral(service_id);
+      break;
+    default:
+      /* The ret value will be -1 thus hitting the warning below. This should
+       * never happen because of the check at the start of the function. */
+      break;
+    }
+    if (ret < 0) {
+      /* This should *NEVER* fail, since the service is on either the
+       * per-control connection list, or the global one.
+       */
+      log_warn(LD_BUG, "Failed to remove Onion Service %s.",
+               escaped(service_id));
+      tor_fragile_assert();
+    }
+
+    /* Remove/scrub the service_id from the appropriate list. */
+    char *cp = smartlist_get(onion_services, idx);
+    smartlist_del(onion_services, idx);
+    memwipe(cp, 0, strlen(cp));
+    tor_free(cp);
+
+    send_control_done(conn);
+  }
+
+ out:
+  return 0;
+}
+
+static const control_cmd_syntax_t obsolete_syntax = {
+  .max_args = UINT_MAX
+};
+
+/**
+ * Called when we get an obsolete command: tell the controller that it is
+ * obsolete.
+ */
+static int
+handle_control_obsolete(control_connection_t *conn,
+                        const control_cmd_args_t *args)
+{
+  (void)args;
+  char *command = tor_strdup(conn->current_cmd);
+  tor_strupper(command);
+  control_printf_endreply(conn, 511, "%s is obsolete.", command);
+  tor_free(command);
+  return 0;
+}
+
+/**
+ * Function pointer to a handler function for a controller command.
+ **/
+typedef int (*handler_fn_t) (control_connection_t *conn,
+                             const control_cmd_args_t *args);
+
+/**
+ * Definition for a controller command.
+ */
+typedef struct control_cmd_def_t {
+  /**
+   * The name of the command. If the command is multiline, the name must
+   * begin with "+".  This is not case-sensitive. */
+  const char *name;
+  /**
+   * A function to execute the command.
+   */
+  handler_fn_t handler;
+  /**
+   * Zero or more CMD_FL_* flags, or'd together.
+   */
+  unsigned flags;
+  /**
+   * For parsed command: a syntax description.
+   */
+  const control_cmd_syntax_t *syntax;
+} control_cmd_def_t;
+
+/**
+ * Indicates that the command's arguments are sensitive, and should be
+ * memwiped after use.
+ */
+#define CMD_FL_WIPE (1u<<0)
+
+#ifndef COCCI
+/** Macro: declare a command with a one-line argument, a given set of flags,
+ * and a syntax definition.
+ **/
+#define ONE_LINE(name, flags)                                   \
+  {                                                             \
+    (#name),                                                    \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax,                                           \
+   }
+
+/**
+ * Macro: declare a command with a multi-line argument and a given set of
+ * flags.
+ **/
+#define MULTLINE(name, flags)                                   \
+  { ("+"#name),                                                 \
+      handle_control_ ##name,                                   \
+      flags,                                                    \
+      &name##_syntax                                            \
+  }
+
+/**
+ * Macro: declare an obsolete command. (Obsolete commands give a different
+ * error than non-existent ones.)
+ **/
+#define OBSOLETE(name)                          \
+  { #name,                                      \
+      handle_control_obsolete,                  \
+      0,                                        \
+      &obsolete_syntax,                         \
+  }
+#endif /* !defined(COCCI) */
+
+/**
+ * An array defining all the recognized controller commands.
+ **/
+static const control_cmd_def_t CONTROL_COMMANDS[] =
+{
+  ONE_LINE(setconf, 0),
+  ONE_LINE(resetconf, 0),
+  ONE_LINE(getconf, 0),
+  MULTLINE(loadconf, 0),
+  ONE_LINE(setevents, 0),
+  ONE_LINE(authenticate, CMD_FL_WIPE),
+  ONE_LINE(saveconf, 0),
+  ONE_LINE(signal, 0),
+  ONE_LINE(takeownership, 0),
+  ONE_LINE(dropownership, 0),
+  ONE_LINE(mapaddress, 0),
+  ONE_LINE(getinfo, 0),
+  ONE_LINE(extendcircuit, 0),
+  ONE_LINE(setcircuitpurpose, 0),
+  OBSOLETE(setrouterpurpose),
+  ONE_LINE(attachstream, 0),
+  MULTLINE(postdescriptor, 0),
+  ONE_LINE(redirectstream, 0),
+  ONE_LINE(closestream, 0),
+  ONE_LINE(closecircuit, 0),
+  ONE_LINE(usefeature, 0),
+  ONE_LINE(resolve, 0),
+  ONE_LINE(protocolinfo, 0),
+  ONE_LINE(authchallenge, CMD_FL_WIPE),
+  ONE_LINE(dropguards, 0),
+  ONE_LINE(hsfetch, 0),
+  MULTLINE(hspost, 0),
+  ONE_LINE(add_onion, CMD_FL_WIPE),
+  ONE_LINE(del_onion, CMD_FL_WIPE),
+  ONE_LINE(onion_client_auth_add, CMD_FL_WIPE),
+  ONE_LINE(onion_client_auth_remove, 0),
+  ONE_LINE(onion_client_auth_view, 0),
+};
+
+/**
+ * The number of entries in CONTROL_COMMANDS.
+ **/
+static const size_t N_CONTROL_COMMANDS = ARRAY_LENGTH(CONTROL_COMMANDS);
+
+/**
+ * Run a single control command, as defined by a control_cmd_def_t,
+ * with a given set of arguments.
+ */
+static int
+handle_single_control_command(const control_cmd_def_t *def,
+                              control_connection_t *conn,
+                              uint32_t cmd_data_len,
+                              char *args)
+{
+  int rv = 0;
+
+  control_cmd_args_t *parsed_args;
+  char *err=NULL;
+  tor_assert(def->syntax);
+  parsed_args = control_cmd_parse_args(conn->current_cmd,
+                                       def->syntax,
+                                       cmd_data_len, args,
+                                       &err);
+  if (!parsed_args) {
+    control_printf_endreply(conn, 512, "Bad arguments to %s: %s",
+                            conn->current_cmd, err?err:"");
+    tor_free(err);
+  } else {
+    if (BUG(err))
+      tor_free(err);
+    if (def->handler(conn, parsed_args))
+      rv = 0;
+
+    if (def->flags & CMD_FL_WIPE)
+      control_cmd_args_wipe(parsed_args);
+
+    control_cmd_args_free(parsed_args);
+  }
+
+  if (def->flags & CMD_FL_WIPE)
+    memwipe(args, 0, cmd_data_len);
+
+  return rv;
+}
+
+/**
+ * Run a given controller command, as selected by the current_cmd field of
+ * <b>conn</b>.
+ */
+int
+handle_control_command(control_connection_t *conn,
+                       uint32_t cmd_data_len,
+                       char *args)
+{
+  tor_assert(conn);
+  tor_assert(args);
+  tor_assert(args[cmd_data_len] == '\0');
+
+  for (unsigned i = 0; i < N_CONTROL_COMMANDS; ++i) {
+    const control_cmd_def_t *def = &CONTROL_COMMANDS[i];
+    if (!strcasecmp(conn->current_cmd, def->name)) {
+      return handle_single_control_command(def, conn, cmd_data_len, args);
+    }
+  }
+
+  control_printf_endreply(conn, 510, "Unrecognized command \"%s\"",
+                          conn->current_cmd);
+
+  return 0;
+}
+
+void
+control_cmd_free_all(void)
+{
+  if (detached_onion_services) { /* Free the detached onion services */
+    SMARTLIST_FOREACH(detached_onion_services, char *, cp, tor_free(cp));
+    smartlist_free(detached_onion_services);
+  }
+}
diff --git a/src/feature/control/control_cmd.h b/src/feature/control/control_cmd.h
new file mode 100644
index 0000000000..0ff0f0755f
--- /dev/null
+++ b/src/feature/control/control_cmd.h
@@ -0,0 +1,113 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd.h
+ * \brief Header file for control_cmd.c.
+ **/
+
+#ifndef TOR_CONTROL_CMD_H
+#define TOR_CONTROL_CMD_H
+
+#include "lib/malloc/malloc.h"
+
+int handle_control_command(control_connection_t *conn,
+                           uint32_t cmd_data_len,
+                           char *args);
+void control_cmd_free_all(void);
+
+typedef struct control_cmd_args_t control_cmd_args_t;
+void control_cmd_args_free_(control_cmd_args_t *args);
+void control_cmd_args_wipe(control_cmd_args_t *args);
+
+#define control_cmd_args_free(v) \
+  FREE_AND_NULL(control_cmd_args_t, control_cmd_args_free_, (v))
+
+/**
+ * Definition for the syntax of a controller command, as parsed by
+ * control_cmd_parse_args.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+typedef struct control_cmd_syntax_t {
+  /**
+   * Lowest number of positional arguments that this command accepts.
+   * 0 for "it's okay not to have positional arguments."
+   **/
+  unsigned int min_args;
+  /**
+   * Highest number of positional arguments that this command accepts.
+   * UINT_MAX for no limit.
+   **/
+  unsigned int max_args;
+  /**
+   * If true, we should parse options after the positional arguments
+   * as a set of unordered flags and key=value arguments.
+   *
+   * Requires that max_args is not UINT_MAX.
+   **/
+  bool accept_keywords;
+  /**
+   * If accept_keywords is true, then only the keywords listed in this
+   * (NULL-terminated) array are valid keywords for this command.
+   **/
+  const char **allowed_keywords;
+  /**
+   * If accept_keywords is true, this option is passed to kvline_parse() as
+   * its flags.
+   **/
+  unsigned kvline_flags;
+  /**
+   * True iff this command wants to be followed by a multiline object.
+   **/
+  bool want_cmddata;
+  /**
+   * True iff this command needs access to the raw body of the input.
+   *
+   * This should not be needed for pure commands; it is purely a legacy
+   * option.
+   **/
+  bool store_raw_body;
+} control_cmd_syntax_t;
+
+#ifdef CONTROL_CMD_PRIVATE
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+/* ADD_ONION secret key to create an ephemeral service. The command supports
+ * multiple versions so this union stores the key and passes it to the HS
+ * subsystem depending on the requested version. */
+typedef union add_onion_secret_key_t {
+  /* Hidden service v2 secret key. */
+  crypto_pk_t *v2;
+  /* Hidden service v3 secret key. */
+  ed25519_secret_key_t *v3;
+} add_onion_secret_key_t;
+
+STATIC int add_onion_helper_keyarg(const char *arg, int discard_pk,
+                                   const char **key_new_alg_out,
+                                   char **key_new_blob_out,
+                                   add_onion_secret_key_t *decoded_key,
+                                   int *hs_version,
+                                   control_connection_t *conn);
+
+STATIC rend_authorized_client_t *add_onion_helper_clientauth(const char *arg,
+                                   int *created, control_connection_t *conn);
+
+STATIC control_cmd_args_t *control_cmd_parse_args(
+                                   const char *command,
+                                   const control_cmd_syntax_t *syntax,
+                                   size_t body_len,
+                                   const char *body,
+                                   char **error_out);
+
+#endif /* defined(CONTROL_CMD_PRIVATE) */
+
+#ifdef CONTROL_MODULE_PRIVATE
+smartlist_t * get_detached_onion_services(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_CMD_H) */
diff --git a/src/feature/control/control_cmd_args_st.h b/src/feature/control/control_cmd_args_st.h
new file mode 100644
index 0000000000..e7d064c6fe
--- /dev/null
+++ b/src/feature/control/control_cmd_args_st.h
@@ -0,0 +1,52 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_cmd_args_st.h
+ * \brief Definition for control_cmd_args_t
+ **/
+
+#ifndef TOR_CONTROL_CMD_ST_H
+#define TOR_CONTROL_CMD_ST_H
+
+struct smartlist_t;
+struct config_line_t;
+
+/**
+ * Parsed arguments for a control command.
+ *
+ * WORK IN PROGRESS: This structure is going to get more complex as this
+ * branch goes on.
+ **/
+struct control_cmd_args_t {
+  /**
+   * The command itself, as provided by the controller.  Not owned by this
+   * structure.
+   **/
+  const char *command;
+  /**
+   * Positional arguments to the command.
+   **/
+  struct smartlist_t *args;
+  /**
+   * Keyword arguments to the command.
+   **/
+  struct config_line_t *kwargs;
+  /**
+   * Number of bytes in <b>cmddata</b>; 0 if <b>cmddata</b> is not set.
+   **/
+  size_t cmddata_len;
+  /**
+   * A multiline object passed with this command.
+   **/
+  char *cmddata;
+  /**
+   * If set, a nul-terminated string containing the raw unparsed arguments.
+   **/
+  const char *raw_body;
+};
+
+#endif /* !defined(TOR_CONTROL_CMD_ST_H) */
diff --git a/src/feature/control/control_connection_st.h b/src/feature/control/control_connection_st.h
index 177a916257..9e410324e0 100644
--- a/src/feature/control/control_connection_st.h
+++ b/src/feature/control/control_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file control_connection_st.h
+ * @brief Controller connection structure.
+ **/
+
 #ifndef CONTROL_CONNECTION_ST_H
 #define CONTROL_CONNECTION_ST_H
 
@@ -40,7 +45,8 @@ struct control_connection_t {
   /** A control command that we're reading from the inbuf, but which has not
    * yet arrived completely. */
   char *incoming_cmd;
+  /** The control command that we are currently processing. */
+  char *current_cmd;
 };
 
-#endif
-
+#endif /* !defined(CONTROL_CONNECTION_ST_H) */
diff --git a/src/feature/control/control_events.c b/src/feature/control/control_events.c
new file mode 100644
index 0000000000..916ccea875
--- /dev/null
+++ b/src/feature/control/control_events.c
@@ -0,0 +1,2314 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.c
+ * \brief Implement the event-reporting part of the controller API.
+ **/
+
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/channeltls.h"
+#include "core/or/circuitlist.h"
+#include "core/or/command.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/reasons.h"
+#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/describe.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/control/control_connection_st.h"
+#include "core/or/entry_connection_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "lib/evloop/compat_libevent.h"
+#include "lib/encoding/confline.h"
+
+static void flush_queued_events_cb(mainloop_event_t *event, void *arg);
+static void control_get_bytes_rw_last_sec(uint64_t *r, uint64_t *w);
+
+/** Yield true iff <b>s</b> is the state of a control_connection_t that has
+ * finished authentication and is accepting commands. */
+#define STATE_IS_OPEN(s) ((s) == CONTROL_CONN_STATE_OPEN)
+
+/** An event mask of all the events that any controller is interested in
+ * receiving. */
+static event_mask_t global_event_mask = 0;
+
+/** True iff we have disabled log messages from being sent to the controller */
+static int disable_log_messages = 0;
+
+/** Macro: true if any control connection is interested in events of type
+ * <b>e</b>. */
+#define EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & EVENT_MASK_(e)))
+
+/** Macro: true if any event from the bitfield 'e' is interesting. */
+#define ANY_EVENT_IS_INTERESTING(e) \
+  (!! (global_event_mask & (e)))
+
+static void send_control_event_impl(uint16_t event,
+                                    const char *format, va_list ap)
+  CHECK_PRINTF(2,0);
+static int control_event_status(int type, int severity, const char *format,
+                                va_list args)
+  CHECK_PRINTF(3,0);
+
+static void send_control_event(uint16_t event,
+                               const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+/** Table mapping event values to their names.  Used to implement SETEVENTS
+ * and GETINFO events/names, and to keep they in sync. */
+const struct control_event_t control_event_table[] = {
+  { EVENT_CIRCUIT_STATUS, "CIRC" },
+  { EVENT_CIRCUIT_STATUS_MINOR, "CIRC_MINOR" },
+  { EVENT_STREAM_STATUS, "STREAM" },
+  { EVENT_OR_CONN_STATUS, "ORCONN" },
+  { EVENT_BANDWIDTH_USED, "BW" },
+  { EVENT_DEBUG_MSG, "DEBUG" },
+  { EVENT_INFO_MSG, "INFO" },
+  { EVENT_NOTICE_MSG, "NOTICE" },
+  { EVENT_WARN_MSG, "WARN" },
+  { EVENT_ERR_MSG, "ERR" },
+  { EVENT_NEW_DESC, "NEWDESC" },
+  { EVENT_ADDRMAP, "ADDRMAP" },
+  { EVENT_DESCCHANGED, "DESCCHANGED" },
+  { EVENT_NS, "NS" },
+  { EVENT_STATUS_GENERAL, "STATUS_GENERAL" },
+  { EVENT_STATUS_CLIENT, "STATUS_CLIENT" },
+  { EVENT_STATUS_SERVER, "STATUS_SERVER" },
+  { EVENT_GUARD, "GUARD" },
+  { EVENT_STREAM_BANDWIDTH_USED, "STREAM_BW" },
+  { EVENT_CLIENTS_SEEN, "CLIENTS_SEEN" },
+  { EVENT_NEWCONSENSUS, "NEWCONSENSUS" },
+  { EVENT_BUILDTIMEOUT_SET, "BUILDTIMEOUT_SET" },
+  { EVENT_GOT_SIGNAL, "SIGNAL" },
+  { EVENT_CONF_CHANGED, "CONF_CHANGED"},
+  { EVENT_CONN_BW, "CONN_BW" },
+  { EVENT_CELL_STATS, "CELL_STATS" },
+  { EVENT_CIRC_BANDWIDTH_USED, "CIRC_BW" },
+  { EVENT_TRANSPORT_LAUNCHED, "TRANSPORT_LAUNCHED" },
+  { EVENT_HS_DESC, "HS_DESC" },
+  { EVENT_HS_DESC_CONTENT, "HS_DESC_CONTENT" },
+  { EVENT_NETWORK_LIVENESS, "NETWORK_LIVENESS" },
+  { 0, NULL },
+};
+
+/** Given a log severity, return the corresponding control event code. */
+static inline int
+log_severity_to_event(int severity)
+{
+  switch (severity) {
+    case LOG_DEBUG: return EVENT_DEBUG_MSG;
+    case LOG_INFO: return EVENT_INFO_MSG;
+    case LOG_NOTICE: return EVENT_NOTICE_MSG;
+    case LOG_WARN: return EVENT_WARN_MSG;
+    case LOG_ERR: return EVENT_ERR_MSG;
+    default: return -1;
+  }
+}
+
+/** Helper: clear bandwidth counters of all origin circuits. */
+static void
+clear_circ_bw_fields(void)
+{
+  origin_circuit_t *ocirc;
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+    ocirc = TO_ORIGIN_CIRCUIT(circ);
+    ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+    ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+    ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+  }
+  SMARTLIST_FOREACH_END(circ);
+}
+
+/** Set <b>global_event_mask*</b> to the bitwise OR of each live control
+ * connection's event_mask field. */
+void
+control_update_global_event_mask(void)
+{
+  smartlist_t *conns = get_connection_array();
+  event_mask_t old_mask, new_mask;
+  old_mask = global_event_mask;
+  int any_old_per_sec_events = control_any_per_second_event_enabled();
+
+  global_event_mask = 0;
+  SMARTLIST_FOREACH(conns, connection_t *, _conn,
+  {
+    if (_conn->type == CONN_TYPE_CONTROL &&
+        STATE_IS_OPEN(_conn->state)) {
+      control_connection_t *conn = TO_CONTROL_CONN(_conn);
+      global_event_mask |= conn->event_mask;
+    }
+  });
+
+  new_mask = global_event_mask;
+
+  /* Handle the aftermath.  Set up the log callback to tell us only what
+   * we want to hear...*/
+  control_adjust_event_log_severity();
+
+  /* Macro: true if ev was false before and is true now. */
+#define NEWLY_ENABLED(ev) \
+  (! (old_mask & (ev)) && (new_mask & (ev)))
+
+  /* ...then, if we've started logging stream or circ bw, clear the
+   * appropriate fields. */
+  if (NEWLY_ENABLED(EVENT_STREAM_BANDWIDTH_USED)) {
+    SMARTLIST_FOREACH(conns, connection_t *, conn,
+    {
+      if (conn->type == CONN_TYPE_AP) {
+        edge_connection_t *edge_conn = TO_EDGE_CONN(conn);
+        edge_conn->n_written = edge_conn->n_read = 0;
+      }
+    });
+  }
+  if (NEWLY_ENABLED(EVENT_CIRC_BANDWIDTH_USED)) {
+    clear_circ_bw_fields();
+  }
+  if (NEWLY_ENABLED(EVENT_BANDWIDTH_USED)) {
+    uint64_t r, w;
+    control_get_bytes_rw_last_sec(&r, &w);
+  }
+  if (any_old_per_sec_events != control_any_per_second_event_enabled()) {
+    rescan_periodic_events(get_options());
+  }
+
+#undef NEWLY_ENABLED
+}
+
+/** Given a control event code for a message event, return the corresponding
+ * log severity. */
+static inline int
+event_to_log_severity(int event)
+{
+  switch (event) {
+    case EVENT_DEBUG_MSG: return LOG_DEBUG;
+    case EVENT_INFO_MSG: return LOG_INFO;
+    case EVENT_NOTICE_MSG: return LOG_NOTICE;
+    case EVENT_WARN_MSG: return LOG_WARN;
+    case EVENT_ERR_MSG: return LOG_ERR;
+    default: return -1;
+  }
+}
+
+/** Adjust the log severities that result in control_event_logmsg being called
+ * to match the severity of log messages that any controllers are interested
+ * in. */
+void
+control_adjust_event_log_severity(void)
+{
+  int i;
+  int min_log_event=EVENT_ERR_MSG, max_log_event=EVENT_DEBUG_MSG;
+
+  for (i = EVENT_DEBUG_MSG; i <= EVENT_ERR_MSG; ++i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      min_log_event = i;
+      break;
+    }
+  }
+  for (i = EVENT_ERR_MSG; i >= EVENT_DEBUG_MSG; --i) {
+    if (EVENT_IS_INTERESTING(i)) {
+      max_log_event = i;
+      break;
+    }
+  }
+  if (EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL)) {
+    if (min_log_event > EVENT_NOTICE_MSG)
+      min_log_event = EVENT_NOTICE_MSG;
+    if (max_log_event < EVENT_ERR_MSG)
+      max_log_event = EVENT_ERR_MSG;
+  }
+  if (min_log_event <= max_log_event)
+    change_callback_log_severity(event_to_log_severity(min_log_event),
+                                 event_to_log_severity(max_log_event),
+                                 control_event_logmsg);
+  else
+    change_callback_log_severity(LOG_ERR, LOG_ERR,
+                                 control_event_logmsg);
+}
+
+/** Return true iff the event with code <b>c</b> is being sent to any current
+ * control connection.  This is useful if the amount of work needed to prepare
+ * to call the appropriate control_event_...() function is high.
+ */
+int
+control_event_is_interesting(int event)
+{
+  return EVENT_IS_INTERESTING(event);
+}
+
+/** Return true if any event that needs to fire once a second is enabled. */
+int
+control_any_per_second_event_enabled(void)
+{
+  return ANY_EVENT_IS_INTERESTING(
+      EVENT_MASK_(EVENT_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CELL_STATS) |
+      EVENT_MASK_(EVENT_CIRC_BANDWIDTH_USED) |
+      EVENT_MASK_(EVENT_CONN_BW) |
+      EVENT_MASK_(EVENT_STREAM_BANDWIDTH_USED)
+  );
+}
+
+/* The value of 'get_bytes_read()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_read = 0;
+/* The value of 'get_bytes_written()' the previous time that
+ * control_get_bytes_rw_last_sec() as called. */
+static uint64_t stats_prev_n_written = 0;
+
+/**
+ * Set <b>n_read</b> and <b>n_written</b> to the total number of bytes read
+ * and written by Tor since the last call to this function.
+ *
+ * Call this only from the main thread.
+ */
+static void
+control_get_bytes_rw_last_sec(uint64_t *n_read,
+                              uint64_t *n_written)
+{
+  const uint64_t stats_n_bytes_read = get_bytes_read();
+  const uint64_t stats_n_bytes_written = get_bytes_written();
+
+  *n_read = stats_n_bytes_read - stats_prev_n_read;
+  *n_written = stats_n_bytes_written - stats_prev_n_written;
+  stats_prev_n_read = stats_n_bytes_read;
+  stats_prev_n_written = stats_n_bytes_written;
+}
+
+/**
+ * Run all the controller events (if any) that are scheduled to trigger once
+ * per second.
+ */
+void
+control_per_second_events(void)
+{
+  if (!control_any_per_second_event_enabled())
+    return;
+
+  uint64_t bytes_read, bytes_written;
+  control_get_bytes_rw_last_sec(&bytes_read, &bytes_written);
+  control_event_bandwidth_used((uint32_t)bytes_read,(uint32_t)bytes_written);
+
+  control_event_stream_bandwidth_used();
+  control_event_conn_bandwidth_used();
+  control_event_circ_bandwidth_used();
+  control_event_circuit_cell_stats();
+}
+
+/** Represents an event that's queued to be sent to one or more
+ * controllers. */
+typedef struct queued_event_t {
+  uint16_t event;
+  char *msg;
+} queued_event_t;
+
+/** Pointer to int. If this is greater than 0, we don't allow new events to be
+ * queued. */
+static tor_threadlocal_t block_event_queue_flag;
+
+/** Holds a smartlist of queued_event_t objects that may need to be sent
+ * to one or more controllers */
+static smartlist_t *queued_control_events = NULL;
+
+/** True if the flush_queued_events_event is pending. */
+static int flush_queued_event_pending = 0;
+
+/** Lock to protect the above fields. */
+static tor_mutex_t *queued_control_events_lock = NULL;
+
+/** An event that should fire in order to flush the contents of
+ * queued_control_events. */
+static mainloop_event_t *flush_queued_events_event = NULL;
+
+void
+control_initialize_event_queue(void)
+{
+  if (queued_control_events == NULL) {
+    queued_control_events = smartlist_new();
+  }
+
+  if (flush_queued_events_event == NULL) {
+    struct event_base *b = tor_libevent_get_base();
+    if (b) {
+      flush_queued_events_event =
+        mainloop_event_new(flush_queued_events_cb, NULL);
+      tor_assert(flush_queued_events_event);
+    }
+  }
+
+  if (queued_control_events_lock == NULL) {
+    queued_control_events_lock = tor_mutex_new();
+    tor_threadlocal_init(&block_event_queue_flag);
+  }
+}
+
+static int *
+get_block_event_queue(void)
+{
+  int *val = tor_threadlocal_get(&block_event_queue_flag);
+  if (PREDICT_UNLIKELY(val == NULL)) {
+    val = tor_malloc_zero(sizeof(int));
+    tor_threadlocal_set(&block_event_queue_flag, val);
+  }
+  return val;
+}
+
+/** Helper: inserts an event on the list of events queued to be sent to
+ * one or more controllers, and schedules the events to be flushed if needed.
+ *
+ * This function takes ownership of <b>msg</b>, and may free it.
+ *
+ * We queue these events rather than send them immediately in order to break
+ * the dependency in our callgraph from code that generates events for the
+ * controller, and the network layer at large.  Otherwise, nearly every
+ * interesting part of Tor would potentially call every other interesting part
+ * of Tor.
+ */
+MOCK_IMPL(STATIC void,
+queue_control_event_string,(uint16_t event, char *msg))
+{
+  /* This is redundant with checks done elsewhere, but it's a last-ditch
+   * attempt to avoid queueing something we shouldn't have to queue. */
+  if (PREDICT_UNLIKELY( ! EVENT_IS_INTERESTING(event) )) {
+    tor_free(msg);
+    return;
+  }
+
+  int *block_event_queue = get_block_event_queue();
+  if (*block_event_queue) {
+    tor_free(msg);
+    return;
+  }
+
+  queued_event_t *ev = tor_malloc(sizeof(*ev));
+  ev->event = event;
+  ev->msg = msg;
+
+  /* No queueing an event while queueing an event */
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  tor_assert(queued_control_events);
+  smartlist_add(queued_control_events, ev);
+
+  int activate_event = 0;
+  if (! flush_queued_event_pending && in_main_thread()) {
+    activate_event = 1;
+    flush_queued_event_pending = 1;
+  }
+
+  tor_mutex_release(queued_control_events_lock);
+
+  --*block_event_queue;
+
+  /* We just put an event on the queue; mark the queue to be
+   * flushed.  We only do this from the main thread for now; otherwise,
+   * we'd need to incur locking overhead in Libevent or use a socket.
+   */
+  if (activate_event) {
+    tor_assert(flush_queued_events_event);
+    mainloop_event_activate(flush_queued_events_event);
+  }
+}
+
+#define queued_event_free(ev) \
+  FREE_AND_NULL(queued_event_t, queued_event_free_, (ev))
+
+/** Release all storage held by <b>ev</b>. */
+static void
+queued_event_free_(queued_event_t *ev)
+{
+  if (ev == NULL)
+    return;
+
+  tor_free(ev->msg);
+  tor_free(ev);
+}
+
+/** Send every queued event to every controller that's interested in it,
+ * and remove the events from the queue.  If <b>force</b> is true,
+ * then make all controllers send their data out immediately, since we
+ * may be about to shut down. */
+static void
+queued_events_flush_all(int force)
+{
+  /* Make sure that we get all the pending log events, if there are any. */
+  flush_pending_log_callbacks();
+
+  if (PREDICT_UNLIKELY(queued_control_events == NULL)) {
+    return;
+  }
+  smartlist_t *all_conns = get_connection_array();
+  smartlist_t *controllers = smartlist_new();
+  smartlist_t *queued_events;
+
+  int *block_event_queue = get_block_event_queue();
+  ++*block_event_queue;
+
+  tor_mutex_acquire(queued_control_events_lock);
+  /* No queueing an event while flushing events. */
+  flush_queued_event_pending = 0;
+  queued_events = queued_control_events;
+  queued_control_events = smartlist_new();
+  tor_mutex_release(queued_control_events_lock);
+
+  /* Gather all the controllers that will care... */
+  SMARTLIST_FOREACH_BEGIN(all_conns, connection_t *, conn) {
+    if (conn->type == CONN_TYPE_CONTROL &&
+        !conn->marked_for_close &&
+        conn->state == CONTROL_CONN_STATE_OPEN) {
+      control_connection_t *control_conn = TO_CONTROL_CONN(conn);
+
+      smartlist_add(controllers, control_conn);
+    }
+  } SMARTLIST_FOREACH_END(conn);
+
+  SMARTLIST_FOREACH_BEGIN(queued_events, queued_event_t *, ev) {
+    const event_mask_t bit = ((event_mask_t)1) << ev->event;
+    const size_t msg_len = strlen(ev->msg);
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      if (control_conn->event_mask & bit) {
+        connection_buf_add(ev->msg, msg_len, TO_CONN(control_conn));
+      }
+    } SMARTLIST_FOREACH_END(control_conn);
+
+    queued_event_free(ev);
+  } SMARTLIST_FOREACH_END(ev);
+
+  if (force) {
+    SMARTLIST_FOREACH_BEGIN(controllers, control_connection_t *,
+                            control_conn) {
+      connection_flush(TO_CONN(control_conn));
+    } SMARTLIST_FOREACH_END(control_conn);
+  }
+
+  smartlist_free(queued_events);
+  smartlist_free(controllers);
+
+  --*block_event_queue;
+}
+
+/** Libevent callback: Flushes pending events to controllers that are
+ * interested in them. */
+static void
+flush_queued_events_cb(mainloop_event_t *event, void *arg)
+{
+  (void) event;
+  (void) arg;
+  queued_events_flush_all(0);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is given by <b>msg</b>.
+ *
+ * The EXTENDED_FORMAT and NONEXTENDED_FORMAT flags behave similarly with
+ * respect to the EXTENDED_EVENTS feature. */
+MOCK_IMPL(STATIC void,
+send_control_event_string,(uint16_t event,
+                           const char *msg))
+{
+  tor_assert(event >= EVENT_MIN_ && event <= EVENT_MAX_);
+  queue_control_event_string(event, tor_strdup(msg));
+}
+
+/** Helper for send_control_event and control_event_status:
+ * Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event_impl(uint16_t event,
+                        const char *format, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, format, ap);
+  if (len < 0) {
+    log_warn(LD_BUG, "Unable to format event for controller.");
+    return;
+  }
+
+  queue_control_event_string(event, buf);
+}
+
+/** Send an event to all v1 controllers that are listening for code
+ * <b>event</b>.  The event's body is created by the printf-style format in
+ * <b>format</b>, and other arguments as provided. */
+static void
+send_control_event(uint16_t event,
+                   const char *format, ...)
+{
+  va_list ap;
+  va_start(ap, format);
+  send_control_event_impl(event, format, ap);
+  va_end(ap);
+}
+
+/** Something major has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+int
+control_event_circuit_status(origin_circuit_t *circ, circuit_status_event_t tp,
+                             int reason_code)
+{
+  const char *status;
+  char reasons[64] = "";
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS))
+    return 0;
+  tor_assert(circ);
+
+  switch (tp)
+    {
+    case CIRC_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case CIRC_EVENT_BUILT: status = "BUILT"; break;
+    case CIRC_EVENT_EXTENDED: status = "EXTENDED"; break;
+    case CIRC_EVENT_FAILED: status = "FAILED"; break;
+    case CIRC_EVENT_CLOSED: status = "CLOSED"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  if (tp == CIRC_EVENT_FAILED || tp == CIRC_EVENT_CLOSED) {
+    const char *reason_str = circuit_end_reason_to_control_string(reason_code);
+    char unk_reason_buf[16];
+    if (!reason_str) {
+      tor_snprintf(unk_reason_buf, 16, "UNKNOWN_%d", reason_code);
+      reason_str = unk_reason_buf;
+    }
+    if (reason_code > 0 && reason_code & END_CIRC_REASON_FLAG_REMOTE) {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=DESTROYED REMOTE_REASON=%s", reason_str);
+    } else {
+      tor_snprintf(reasons, sizeof(reasons),
+                   " REASON=%s", reason_str);
+    }
+  }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS,
+                                "650 CIRC %lu %s%s%s%s\r\n",
+                                (unsigned long)circ->global_identifier,
+                                status, sp,
+                                circdesc,
+                                reasons);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/** Something minor has happened to circuit <b>circ</b>: tell any
+ * interested control connections. */
+static int
+control_event_circuit_status_minor(origin_circuit_t *circ,
+                                   circuit_status_minor_event_t e,
+                                   int purpose, const struct timeval *tv)
+{
+  const char *event_desc;
+  char event_tail[160] = "";
+  if (!EVENT_IS_INTERESTING(EVENT_CIRCUIT_STATUS_MINOR))
+    return 0;
+  tor_assert(circ);
+
+  switch (e)
+    {
+    case CIRC_MINOR_EVENT_PURPOSE_CHANGED:
+      event_desc = "PURPOSE_CHANGED";
+
+      {
+        /* event_tail can currently be up to 68 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "");
+      }
+
+      break;
+    case CIRC_MINOR_EVENT_CANNIBALIZED:
+      event_desc = "CANNIBALIZED";
+
+      {
+        /* event_tail can currently be up to 130 chars long */
+        const char *hs_state_str =
+          circuit_purpose_to_controller_hs_state_string(purpose);
+        const struct timeval *old_timestamp_began = tv;
+        char tbuf[ISO_TIME_USEC_LEN+1];
+        format_iso_time_nospace_usec(tbuf, old_timestamp_began);
+
+        tor_snprintf(event_tail, sizeof(event_tail),
+                     " OLD_PURPOSE=%s%s%s OLD_TIME_CREATED=%s",
+                     circuit_purpose_to_controller_string(purpose),
+                     (hs_state_str != NULL) ? " OLD_HS_STATE=" : "",
+                     (hs_state_str != NULL) ? hs_state_str : "",
+                     tbuf);
+      }
+
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)e);
+      tor_fragile_assert();
+      return 0;
+    }
+
+  {
+    char *circdesc = circuit_describe_status_for_controller(circ);
+    const char *sp = strlen(circdesc) ? " " : "";
+    send_control_event(EVENT_CIRCUIT_STATUS_MINOR,
+                       "650 CIRC_MINOR %lu %s%s%s%s\r\n",
+                       (unsigned long)circ->global_identifier,
+                       event_desc, sp,
+                       circdesc,
+                       event_tail);
+    tor_free(circdesc);
+  }
+
+  return 0;
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>: tell any
+ * interested controllers.
+ */
+int
+control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                      int old_purpose)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+                                            old_purpose,
+                                            NULL);
+}
+
+/**
+ * <b>circ</b> has changed its purpose from <b>old_purpose</b>, and its
+ * created-time from <b>old_tv_created</b>: tell any interested controllers.
+ */
+int
+control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                   int old_purpose,
+                                   const struct timeval *old_tv_created)
+{
+  return control_event_circuit_status_minor(circ,
+                                            CIRC_MINOR_EVENT_CANNIBALIZED,
+                                            old_purpose,
+                                            old_tv_created);
+}
+
+/** Something has happened to the stream associated with AP connection
+ * <b>conn</b>: tell any interested control connections. */
+int
+control_event_stream_status(entry_connection_t *conn, stream_status_event_t tp,
+                            int reason_code)
+{
+  char reason_buf[64];
+  char addrport_buf[64];
+  const char *status;
+  circuit_t *circ;
+  origin_circuit_t *origin_circ = NULL;
+  char buf[256];
+  const char *purpose = "";
+  tor_assert(conn->socks_request);
+
+  if (!EVENT_IS_INTERESTING(EVENT_STREAM_STATUS))
+    return 0;
+
+  if (tp == STREAM_EVENT_CLOSED &&
+      (reason_code & END_STREAM_REASON_FLAG_ALREADY_SENT_CLOSED))
+    return 0;
+
+  write_stream_target_to_buf(conn, buf, sizeof(buf));
+
+  reason_buf[0] = '\0';
+  switch (tp)
+    {
+    case STREAM_EVENT_SENT_CONNECT: status = "SENTCONNECT"; break;
+    case STREAM_EVENT_SENT_RESOLVE: status = "SENTRESOLVE"; break;
+    case STREAM_EVENT_SUCCEEDED: status = "SUCCEEDED"; break;
+    case STREAM_EVENT_FAILED: status = "FAILED"; break;
+    case STREAM_EVENT_CLOSED: status = "CLOSED"; break;
+    case STREAM_EVENT_NEW: status = "NEW"; break;
+    case STREAM_EVENT_NEW_RESOLVE: status = "NEWRESOLVE"; break;
+    case STREAM_EVENT_FAILED_RETRIABLE: status = "DETACHED"; break;
+    case STREAM_EVENT_REMAP: status = "REMAP"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (reason_code && (tp == STREAM_EVENT_FAILED ||
+                      tp == STREAM_EVENT_CLOSED ||
+                      tp == STREAM_EVENT_FAILED_RETRIABLE)) {
+    const char *reason_str = stream_end_reason_to_control_string(reason_code);
+    char *r = NULL;
+    if (!reason_str) {
+      tor_asprintf(&r, " UNKNOWN_%d", reason_code);
+      reason_str = r;
+    }
+    if (reason_code & END_STREAM_REASON_FLAG_REMOTE)
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=END REMOTE_REASON=%s", reason_str);
+    else
+      tor_snprintf(reason_buf, sizeof(reason_buf),
+                   " REASON=%s", reason_str);
+    tor_free(r);
+  } else if (reason_code && tp == STREAM_EVENT_REMAP) {
+    switch (reason_code) {
+    case REMAP_STREAM_SOURCE_CACHE:
+      strlcpy(reason_buf, " SOURCE=CACHE", sizeof(reason_buf));
+      break;
+    case REMAP_STREAM_SOURCE_EXIT:
+      strlcpy(reason_buf, " SOURCE=EXIT", sizeof(reason_buf));
+      break;
+    default:
+      tor_snprintf(reason_buf, sizeof(reason_buf), " REASON=UNKNOWN_%d",
+                   reason_code);
+      /* XXX do we want SOURCE=UNKNOWN_%d above instead? -RD */
+      break;
+    }
+  }
+
+  if (tp == STREAM_EVENT_NEW || tp == STREAM_EVENT_NEW_RESOLVE) {
+    /*
+     * When the control conn is an AF_UNIX socket and we have no address,
+     * it gets set to "(Tor_internal)"; see dnsserv_launch_request() in
+     * dnsserv.c.
+     */
+    if (strcmp(ENTRY_TO_CONN(conn)->address, "(Tor_internal)") != 0) {
+      tor_snprintf(addrport_buf,sizeof(addrport_buf), " SOURCE_ADDR=%s:%d",
+                   ENTRY_TO_CONN(conn)->address, ENTRY_TO_CONN(conn)->port);
+    } else {
+      /*
+       * else leave it blank so control on AF_UNIX doesn't need to make
+       * something up.
+       */
+      addrport_buf[0] = '\0';
+    }
+  } else {
+    addrport_buf[0] = '\0';
+  }
+
+  if (tp == STREAM_EVENT_NEW_RESOLVE) {
+    purpose = " PURPOSE=DNS_REQUEST";
+  } else if (tp == STREAM_EVENT_NEW) {
+    if (conn->use_begindir) {
+      connection_t *linked = ENTRY_TO_CONN(conn)->linked_conn;
+      int linked_dir_purpose = -1;
+      if (linked && linked->type == CONN_TYPE_DIR)
+        linked_dir_purpose = linked->purpose;
+      if (DIR_PURPOSE_IS_UPLOAD(linked_dir_purpose))
+        purpose = " PURPOSE=DIR_UPLOAD";
+      else
+        purpose = " PURPOSE=DIR_FETCH";
+    } else
+      purpose = " PURPOSE=USER";
+  }
+
+  circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+  if (circ && CIRCUIT_IS_ORIGIN(circ))
+    origin_circ = TO_ORIGIN_CIRCUIT(circ);
+
+  {
+    char *conndesc = entry_connection_describe_status_for_controller(conn);
+    const char *sp = strlen(conndesc) ? " " : "";
+    send_control_event(EVENT_STREAM_STATUS,
+                        "650 STREAM %"PRIu64" %s %lu %s%s%s%s%s%s\r\n",
+                     (ENTRY_TO_CONN(conn)->global_identifier),
+                     status,
+                        origin_circ?
+                           (unsigned long)origin_circ->global_identifier : 0ul,
+                        buf, reason_buf, addrport_buf, purpose, sp, conndesc);
+    tor_free(conndesc);
+  }
+
+  /* XXX need to specify its intended exit, etc? */
+
+  return 0;
+}
+
+/** Called when the status of an OR connection <b>conn</b> changes: tell any
+ * interested control connections. <b>tp</b> is the new status for the
+ * connection.  If <b>conn</b> has just closed or failed, then <b>reason</b>
+ * may be the reason why.
+ */
+int
+control_event_or_conn_status(or_connection_t *conn, or_conn_status_event_t tp,
+                             int reason)
+{
+  int ncircs = 0;
+  const char *status;
+  char name[128];
+  char ncircs_buf[32] = {0}; /* > 8 + log10(2^32)=10 + 2 */
+
+  if (!EVENT_IS_INTERESTING(EVENT_OR_CONN_STATUS))
+    return 0;
+
+  switch (tp)
+    {
+    case OR_CONN_EVENT_LAUNCHED: status = "LAUNCHED"; break;
+    case OR_CONN_EVENT_CONNECTED: status = "CONNECTED"; break;
+    case OR_CONN_EVENT_FAILED: status = "FAILED"; break;
+    case OR_CONN_EVENT_CLOSED: status = "CLOSED"; break;
+    case OR_CONN_EVENT_NEW: status = "NEW"; break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status code %d", (int)tp);
+      return 0;
+    }
+  if (conn->chan) {
+    ncircs = circuit_count_pending_on_channel(TLS_CHAN_TO_BASE(conn->chan));
+  } else {
+    ncircs = 0;
+  }
+  ncircs += connection_or_get_num_circuits(conn);
+  if (ncircs && (tp == OR_CONN_EVENT_FAILED || tp == OR_CONN_EVENT_CLOSED)) {
+    tor_snprintf(ncircs_buf, sizeof(ncircs_buf), " NCIRCS=%d", ncircs);
+  }
+
+  orconn_target_get_name(name, sizeof(name), conn);
+  send_control_event(EVENT_OR_CONN_STATUS,
+                              "650 ORCONN %s %s%s%s%s ID=%"PRIu64"\r\n",
+                              name, status,
+                              reason ? " REASON=" : "",
+                              orconn_end_reason_to_control_string(reason),
+                              ncircs_buf,
+                              (conn->base_.global_identifier));
+
+  return 0;
+}
+
+/**
+ * Print out STREAM_BW event for a single conn
+ */
+int
+control_event_stream_bandwidth(edge_connection_t *edge_conn)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    if (!edge_conn->n_read && !edge_conn->n_written)
+      return 0;
+
+    tor_gettimeofday(&now);
+    format_iso_time_nospace_usec(tbuf, &now);
+    send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                       "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                       (edge_conn->base_.global_identifier),
+                       (unsigned long)edge_conn->n_read,
+                       (unsigned long)edge_conn->n_written,
+                       tbuf);
+
+    edge_conn->n_written = edge_conn->n_read = 0;
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth streams have used. */
+int
+control_event_stream_bandwidth_used(void)
+{
+  if (EVENT_IS_INTERESTING(EVENT_STREAM_BANDWIDTH_USED)) {
+    smartlist_t *conns = get_connection_array();
+    edge_connection_t *edge_conn;
+    struct timeval now;
+    char tbuf[ISO_TIME_USEC_LEN+1];
+
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn)
+    {
+        if (conn->type != CONN_TYPE_AP)
+          continue;
+        edge_conn = TO_EDGE_CONN(conn);
+        if (!edge_conn->n_read && !edge_conn->n_written)
+          continue;
+
+        tor_gettimeofday(&now);
+        format_iso_time_nospace_usec(tbuf, &now);
+        send_control_event(EVENT_STREAM_BANDWIDTH_USED,
+                           "650 STREAM_BW %"PRIu64" %lu %lu %s\r\n",
+                           (edge_conn->base_.global_identifier),
+                           (unsigned long)edge_conn->n_read,
+                           (unsigned long)edge_conn->n_written,
+                           tbuf);
+
+        edge_conn->n_written = edge_conn->n_read = 0;
+    }
+    SMARTLIST_FOREACH_END(conn);
+  }
+
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control connections
+ * how much bandwidth origin circuits have used. */
+int
+control_event_circ_bandwidth_used(void)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!CIRCUIT_IS_ORIGIN(circ))
+      continue;
+
+    control_event_circ_bandwidth_used_for_circ(TO_ORIGIN_CIRCUIT(circ));
+  }
+  SMARTLIST_FOREACH_END(circ);
+
+  return 0;
+}
+
+/**
+ * Emit a CIRC_BW event line for a specific circuit.
+ *
+ * This function sets the values it emits to 0, and does not emit
+ * an event if there is no new data to report since the last call.
+ *
+ * Therefore, it may be called at any frequency.
+ */
+int
+control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc)
+{
+  struct timeval now;
+  char tbuf[ISO_TIME_USEC_LEN+1];
+
+  tor_assert(ocirc);
+
+  if (!EVENT_IS_INTERESTING(EVENT_CIRC_BANDWIDTH_USED))
+    return 0;
+
+  /* n_read_circ_bw and n_written_circ_bw are always updated
+   * when there is any new cell on a circuit, and set to 0 after
+   * the event, below.
+   *
+   * Therefore, checking them is sufficient to determine if there
+   * is new data to report. */
+  if (!ocirc->n_read_circ_bw && !ocirc->n_written_circ_bw)
+    return 0;
+
+  tor_gettimeofday(&now);
+  format_iso_time_nospace_usec(tbuf, &now);
+  send_control_event(EVENT_CIRC_BANDWIDTH_USED,
+                     "650 CIRC_BW ID=%d READ=%lu WRITTEN=%lu TIME=%s "
+                     "DELIVERED_READ=%lu OVERHEAD_READ=%lu "
+                     "DELIVERED_WRITTEN=%lu OVERHEAD_WRITTEN=%lu\r\n",
+                     ocirc->global_identifier,
+                     (unsigned long)ocirc->n_read_circ_bw,
+                     (unsigned long)ocirc->n_written_circ_bw,
+                     tbuf,
+                     (unsigned long)ocirc->n_delivered_read_circ_bw,
+                     (unsigned long)ocirc->n_overhead_read_circ_bw,
+                     (unsigned long)ocirc->n_delivered_written_circ_bw,
+                     (unsigned long)ocirc->n_overhead_written_circ_bw);
+  ocirc->n_written_circ_bw = ocirc->n_read_circ_bw = 0;
+  ocirc->n_overhead_written_circ_bw = ocirc->n_overhead_read_circ_bw = 0;
+  ocirc->n_delivered_written_circ_bw = ocirc->n_delivered_read_circ_bw = 0;
+
+  return 0;
+}
+
+/** Print out CONN_BW event for a single OR/DIR/EXIT <b>conn</b> and reset
+  * bandwidth counters. */
+int
+control_event_conn_bandwidth(connection_t *conn)
+{
+  const char *conn_type_str;
+  if (!get_options()->TestingEnableConnBwEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CONN_BW))
+    return 0;
+  if (!conn->n_read_conn_bw && !conn->n_written_conn_bw)
+    return 0;
+  switch (conn->type) {
+    case CONN_TYPE_OR:
+      conn_type_str = "OR";
+      break;
+    case CONN_TYPE_DIR:
+      conn_type_str = "DIR";
+      break;
+    case CONN_TYPE_EXIT:
+      conn_type_str = "EXIT";
+      break;
+    default:
+      return 0;
+  }
+  send_control_event(EVENT_CONN_BW,
+                     "650 CONN_BW ID=%"PRIu64" TYPE=%s "
+                     "READ=%lu WRITTEN=%lu\r\n",
+                     (conn->global_identifier),
+                     conn_type_str,
+                     (unsigned long)conn->n_read_conn_bw,
+                     (unsigned long)conn->n_written_conn_bw);
+  conn->n_written_conn_bw = conn->n_read_conn_bw = 0;
+  return 0;
+}
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth connections have used. */
+int
+control_event_conn_bandwidth_used(void)
+{
+  if (get_options()->TestingEnableConnBwEvent &&
+      EVENT_IS_INTERESTING(EVENT_CONN_BW)) {
+    SMARTLIST_FOREACH(get_connection_array(), connection_t *, conn,
+                      control_event_conn_bandwidth(conn));
+  }
+  return 0;
+}
+
+/** Helper: iterate over cell statistics of <b>circ</b> and sum up added
+ * cells, removed cells, and waiting times by cell command and direction.
+ * Store results in <b>cell_stats</b>.  Free cell statistics of the
+ * circuit afterwards. */
+void
+sum_up_cell_stats_by_command(circuit_t *circ, cell_stats_t *cell_stats)
+{
+  memset(cell_stats, 0, sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circ->testing_cell_stats,
+                          const testing_cell_stats_entry_t *, ent) {
+    tor_assert(ent->command <= CELL_COMMAND_MAX_);
+    if (!ent->removed && !ent->exitward) {
+      cell_stats->added_cells_appward[ent->command] += 1;
+    } else if (!ent->removed && ent->exitward) {
+      cell_stats->added_cells_exitward[ent->command] += 1;
+    } else if (!ent->exitward) {
+      cell_stats->removed_cells_appward[ent->command] += 1;
+      cell_stats->total_time_appward[ent->command] += ent->waiting_time * 10;
+    } else {
+      cell_stats->removed_cells_exitward[ent->command] += 1;
+      cell_stats->total_time_exitward[ent->command] += ent->waiting_time * 10;
+    }
+  } SMARTLIST_FOREACH_END(ent);
+  circuit_clear_testing_cell_stats(circ);
+}
+
+/** Helper: append a cell statistics string to <code>event_parts</code>,
+ * prefixed with <code>key</code>=.  Statistics consist of comma-separated
+ * key:value pairs with lower-case command strings as keys and cell
+ * numbers or total waiting times as values.  A key:value pair is included
+ * if the entry in <code>include_if_non_zero</code> is not zero, but with
+ * the (possibly zero) entry from <code>number_to_include</code>.  Both
+ * arrays are expected to have a length of CELL_COMMAND_MAX_ + 1.  If no
+ * entry in <code>include_if_non_zero</code> is positive, no string will
+ * be added to <code>event_parts</code>. */
+void
+append_cell_stats_by_command(smartlist_t *event_parts, const char *key,
+                             const uint64_t *include_if_non_zero,
+                             const uint64_t *number_to_include)
+{
+  smartlist_t *key_value_strings = smartlist_new();
+  int i;
+  for (i = 0; i <= CELL_COMMAND_MAX_; i++) {
+    if (include_if_non_zero[i] > 0) {
+      smartlist_add_asprintf(key_value_strings, "%s:%"PRIu64,
+                             cell_command_to_string(i),
+                             (number_to_include[i]));
+    }
+  }
+  if (smartlist_len(key_value_strings) > 0) {
+    char *joined = smartlist_join_strings(key_value_strings, ",", 0, NULL);
+    smartlist_add_asprintf(event_parts, "%s=%s", key, joined);
+    SMARTLIST_FOREACH(key_value_strings, char *, cp, tor_free(cp));
+    tor_free(joined);
+  }
+  smartlist_free(key_value_strings);
+}
+
+/** Helper: format <b>cell_stats</b> for <b>circ</b> for inclusion in a
+ * CELL_STATS event and write result string to <b>event_string</b>. */
+void
+format_cell_stats(char **event_string, circuit_t *circ,
+                  cell_stats_t *cell_stats)
+{
+  smartlist_t *event_parts = smartlist_new();
+  if (CIRCUIT_IS_ORIGIN(circ)) {
+    origin_circuit_t *ocirc = TO_ORIGIN_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "ID=%lu",
+                 (unsigned long)ocirc->global_identifier);
+  } else if (TO_OR_CIRCUIT(circ)->p_chan) {
+    or_circuit_t *or_circ = TO_OR_CIRCUIT(circ);
+    smartlist_add_asprintf(event_parts, "InboundQueue=%lu",
+                 (unsigned long)or_circ->p_circ_id);
+    smartlist_add_asprintf(event_parts, "InboundConn=%"PRIu64,
+                 (or_circ->p_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "InboundAdded",
+                                 cell_stats->added_cells_appward,
+                                 cell_stats->added_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundRemoved",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->removed_cells_appward);
+    append_cell_stats_by_command(event_parts, "InboundTime",
+                                 cell_stats->removed_cells_appward,
+                                 cell_stats->total_time_appward);
+  }
+  if (circ->n_chan) {
+    smartlist_add_asprintf(event_parts, "OutboundQueue=%lu",
+                     (unsigned long)circ->n_circ_id);
+    smartlist_add_asprintf(event_parts, "OutboundConn=%"PRIu64,
+                 (circ->n_chan->global_identifier));
+    append_cell_stats_by_command(event_parts, "OutboundAdded",
+                                 cell_stats->added_cells_exitward,
+                                 cell_stats->added_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundRemoved",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->removed_cells_exitward);
+    append_cell_stats_by_command(event_parts, "OutboundTime",
+                                 cell_stats->removed_cells_exitward,
+                                 cell_stats->total_time_exitward);
+  }
+  *event_string = smartlist_join_strings(event_parts, " ", 0, NULL);
+  SMARTLIST_FOREACH(event_parts, char *, cp, tor_free(cp));
+  smartlist_free(event_parts);
+}
+
+/** A second or more has elapsed: tell any interested control connection
+ * how many cells have been processed for a given circuit. */
+int
+control_event_circuit_cell_stats(void)
+{
+  cell_stats_t *cell_stats;
+  char *event_string;
+  if (!get_options()->TestingEnableCellStatsEvent ||
+      !EVENT_IS_INTERESTING(EVENT_CELL_STATS))
+    return 0;
+  cell_stats = tor_malloc(sizeof(cell_stats_t));
+  SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ) {
+    if (!circ->testing_cell_stats)
+      continue;
+    sum_up_cell_stats_by_command(circ, cell_stats);
+    format_cell_stats(&event_string, circ, cell_stats);
+    send_control_event(EVENT_CELL_STATS,
+                       "650 CELL_STATS %s\r\n", event_string);
+    tor_free(event_string);
+  }
+  SMARTLIST_FOREACH_END(circ);
+  tor_free(cell_stats);
+  return 0;
+}
+
+/* about 5 minutes worth. */
+#define N_BW_EVENTS_TO_CACHE 300
+/* Index into cached_bw_events to next write. */
+static int next_measurement_idx = 0;
+/* number of entries set in n_measurements */
+static int n_measurements = 0;
+static struct cached_bw_event_t {
+  uint32_t n_read;
+  uint32_t n_written;
+} cached_bw_events[N_BW_EVENTS_TO_CACHE];
+
+/** A second or more has elapsed: tell any interested control
+ * connections how much bandwidth we used. */
+int
+control_event_bandwidth_used(uint32_t n_read, uint32_t n_written)
+{
+  cached_bw_events[next_measurement_idx].n_read = n_read;
+  cached_bw_events[next_measurement_idx].n_written = n_written;
+  if (++next_measurement_idx == N_BW_EVENTS_TO_CACHE)
+    next_measurement_idx = 0;
+  if (n_measurements < N_BW_EVENTS_TO_CACHE)
+    ++n_measurements;
+
+  if (EVENT_IS_INTERESTING(EVENT_BANDWIDTH_USED)) {
+    send_control_event(EVENT_BANDWIDTH_USED,
+                       "650 BW %lu %lu\r\n",
+                       (unsigned long)n_read,
+                       (unsigned long)n_written);
+  }
+
+  return 0;
+}
+
+char *
+get_bw_samples(void)
+{
+  int i;
+  int idx = (next_measurement_idx + N_BW_EVENTS_TO_CACHE - n_measurements)
+    % N_BW_EVENTS_TO_CACHE;
+  tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+
+  smartlist_t *elements = smartlist_new();
+
+  for (i = 0; i < n_measurements; ++i) {
+    tor_assert(0 <= idx && idx < N_BW_EVENTS_TO_CACHE);
+    const struct cached_bw_event_t *bwe = &cached_bw_events[idx];
+
+    smartlist_add_asprintf(elements, "%u,%u",
+                           (unsigned)bwe->n_read,
+                           (unsigned)bwe->n_written);
+
+    idx = (idx + 1) % N_BW_EVENTS_TO_CACHE;
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/** Called when we are sending a log message to the controllers: suspend
+ * sending further log messages to the controllers until we're done.  Used by
+ * CONN_LOG_PROTECT. */
+void
+disable_control_logging(void)
+{
+  ++disable_log_messages;
+}
+
+/** We're done sending a log message to the controllers: re-enable controller
+ * logging.  Used by CONN_LOG_PROTECT. */
+void
+enable_control_logging(void)
+{
+  if (--disable_log_messages < 0)
+    tor_assert(0);
+}
+
+/** We got a log message: tell any interested control connections. */
+void
+control_event_logmsg(int severity, log_domain_mask_t domain, const char *msg)
+{
+  int event;
+
+  /* Don't even think of trying to add stuff to a buffer from a cpuworker
+   * thread. (See #25987 for plan to fix.) */
+  if (! in_main_thread())
+    return;
+
+  if (disable_log_messages)
+    return;
+
+  if (domain == LD_BUG && EVENT_IS_INTERESTING(EVENT_STATUS_GENERAL) &&
+      severity <= LOG_NOTICE) {
+    char *esc = esc_for_log(msg);
+    ++disable_log_messages;
+    control_event_general_status(severity, "BUG REASON=%s", esc);
+    --disable_log_messages;
+    tor_free(esc);
+  }
+
+  event = log_severity_to_event(severity);
+  if (event >= 0 && EVENT_IS_INTERESTING(event)) {
+    char *b = NULL;
+    const char *s;
+    if (strchr(msg, '\n')) {
+      char *cp;
+      b = tor_strdup(msg);
+      for (cp = b; *cp; ++cp)
+        if (*cp == '\r' || *cp == '\n')
+          *cp = ' ';
+    }
+    switch (severity) {
+      case LOG_DEBUG: s = "DEBUG"; break;
+      case LOG_INFO: s = "INFO"; break;
+      case LOG_NOTICE: s = "NOTICE"; break;
+      case LOG_WARN: s = "WARN"; break;
+      case LOG_ERR: s = "ERR"; break;
+      default: s = "UnknownLogSeverity"; break;
+    }
+    ++disable_log_messages;
+    send_control_event(event,  "650 %s %s\r\n", s, b?b:msg);
+    if (severity == LOG_ERR) {
+      /* Force a flush, since we may be about to die horribly */
+      queued_events_flush_all(1);
+    }
+    --disable_log_messages;
+    tor_free(b);
+  }
+}
+
+/**
+ * Logging callback: called when there is a queued pending log callback.
+ */
+void
+control_event_logmsg_pending(void)
+{
+  if (! in_main_thread()) {
+    /* We can't handle this case yet, since we're using a
+     * mainloop_event_t to invoke queued_events_flush_all.  We ought to
+     * use a different mechanism instead: see #25987.
+     **/
+    return;
+  }
+  tor_assert(flush_queued_events_event);
+  mainloop_event_activate(flush_queued_events_event);
+}
+
+/** Called whenever we receive new router descriptors: tell any
+ * interested control connections.  <b>routers</b> is a list of
+ * routerinfo_t's.
+ */
+int
+control_event_descriptors_changed(smartlist_t *routers)
+{
+  char *msg;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NEW_DESC))
+    return 0;
+
+  {
+    smartlist_t *names = smartlist_new();
+    char *ids;
+    SMARTLIST_FOREACH(routers, routerinfo_t *, ri, {
+        char *b = tor_malloc(MAX_VERBOSE_NICKNAME_LEN+1);
+        router_get_verbose_nickname(b, ri);
+        smartlist_add(names, b);
+      });
+    ids = smartlist_join_strings(names, " ", 0, NULL);
+    tor_asprintf(&msg, "650 NEWDESC %s\r\n", ids);
+    send_control_event_string(EVENT_NEW_DESC,  msg);
+    tor_free(ids);
+    tor_free(msg);
+    SMARTLIST_FOREACH(names, char *, cp, tor_free(cp));
+    smartlist_free(names);
+  }
+  return 0;
+}
+
+/** Called when an address mapping on <b>from</b> from changes to <b>to</b>.
+ * <b>expires</b> values less than 3 are special; see connection_edge.c.  If
+ * <b>error</b> is non-NULL, it is an error code describing the failure
+ * mode of the mapping.
+ */
+int
+control_event_address_mapped(const char *from, const char *to, time_t expires,
+                             const char *error, const int cached)
+{
+  if (!EVENT_IS_INTERESTING(EVENT_ADDRMAP))
+    return 0;
+
+  if (expires < 3 || expires == TIME_MAX)
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s NEVER %s%s"
+                                "CACHED=\"%s\"\r\n",
+                                  from, to, error?error:"", error?" ":"",
+                                cached?"YES":"NO");
+  else {
+    char buf[ISO_TIME_LEN+1];
+    char buf2[ISO_TIME_LEN+1];
+    format_local_iso_time(buf,expires);
+    format_iso_time(buf2,expires);
+    send_control_event(EVENT_ADDRMAP,
+                                "650 ADDRMAP %s %s \"%s\""
+                                " %s%sEXPIRES=\"%s\" CACHED=\"%s\"\r\n",
+                                from, to, buf,
+                                error?error:"", error?" ":"",
+                                buf2, cached?"YES":"NO");
+  }
+
+  return 0;
+}
+/** The network liveness has changed; this is called from circuitstats.c
+ * whenever we receive a cell, or when timeout expires and we assume the
+ * network is down. */
+int
+control_event_network_liveness_update(int liveness)
+{
+  if (liveness > 0) {
+    if (get_cached_network_liveness() <= 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(1);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS UP");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS UP\r\n");
+    }
+    /* else was already live, no-op */
+  } else {
+    if (get_cached_network_liveness() > 0) {
+      /* Update cached liveness */
+      set_cached_network_liveness(0);
+      log_debug(LD_CONTROL, "Sending NETWORK_LIVENESS DOWN");
+      send_control_event_string(EVENT_NETWORK_LIVENESS,
+                                "650 NETWORK_LIVENESS DOWN\r\n");
+    }
+    /* else was already dead, no-op */
+  }
+
+  return 0;
+}
+
+/** Helper function for NS-style events. Constructs and sends an event
+ * of type <b>event</b> with string <b>event_string</b> out of the set of
+ * networkstatuses <b>statuses</b>. Currently it is used for NS events
+ * and NEWCONSENSUS events. */
+static int
+control_event_networkstatus_changed_helper(smartlist_t *statuses,
+                                           uint16_t event,
+                                           const char *event_string)
+{
+  smartlist_t *strs;
+  char *s, *esc = NULL;
+  if (!EVENT_IS_INTERESTING(event) || !smartlist_len(statuses))
+    return 0;
+
+  strs = smartlist_new();
+  smartlist_add_strdup(strs, "650+");
+  smartlist_add_strdup(strs, event_string);
+  smartlist_add_strdup(strs, "\r\n");
+  SMARTLIST_FOREACH(statuses, const routerstatus_t *, rs,
+    {
+      s = networkstatus_getinfo_helper_single(rs);
+      if (!s) continue;
+      smartlist_add(strs, s);
+    });
+
+  s = smartlist_join_strings(strs, "", 0, NULL);
+  write_escaped_data(s, strlen(s), &esc);
+  SMARTLIST_FOREACH(strs, char *, cp, tor_free(cp));
+  smartlist_free(strs);
+  tor_free(s);
+  send_control_event_string(event,  esc);
+  send_control_event_string(event,
+                            "650 OK\r\n");
+
+  tor_free(esc);
+  return 0;
+}
+
+/** Called when the routerstatus_ts <b>statuses</b> have changed: sends
+ * an NS event to any controller that cares. */
+int
+control_event_networkstatus_changed(smartlist_t *statuses)
+{
+  return control_event_networkstatus_changed_helper(statuses, EVENT_NS, "NS");
+}
+
+/** Called when we get a new consensus networkstatus. Sends a NEWCONSENSUS
+ * event consisting of an NS-style line for each relay in the consensus. */
+int
+control_event_newconsensus(const networkstatus_t *consensus)
+{
+  if (!control_event_is_interesting(EVENT_NEWCONSENSUS))
+    return 0;
+  return control_event_networkstatus_changed_helper(
+           consensus->routerstatus_list, EVENT_NEWCONSENSUS, "NEWCONSENSUS");
+}
+
+/** Called when we compute a new circuitbuildtimeout */
+int
+control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                               const char *args)
+{
+  const char *type_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_BUILDTIMEOUT_SET))
+    return 0;
+
+  switch (type) {
+    case BUILDTIMEOUT_SET_EVENT_COMPUTED:
+      type_string = "COMPUTED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESET:
+      type_string = "RESET";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_SUSPENDED:
+      type_string = "SUSPENDED";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_DISCARD:
+      type_string = "DISCARD";
+      break;
+    case BUILDTIMEOUT_SET_EVENT_RESUME:
+      type_string = "RESUME";
+      break;
+    default:
+      type_string = "UNKNOWN";
+      break;
+  }
+
+  send_control_event(EVENT_BUILDTIMEOUT_SET,
+                     "650 BUILDTIMEOUT_SET %s %s\r\n",
+                     type_string, args);
+
+  return 0;
+}
+
+/** Called when a signal has been processed from signal_callback */
+int
+control_event_signal(uintptr_t signal_num)
+{
+  const char *signal_string = NULL;
+
+  if (!control_event_is_interesting(EVENT_GOT_SIGNAL))
+    return 0;
+
+  for (unsigned i = 0; signal_table[i].signal_name != NULL; ++i) {
+    if ((int)signal_num == signal_table[i].sig) {
+      signal_string = signal_table[i].signal_name;
+      break;
+    }
+  }
+
+  if (signal_string == NULL) {
+    log_warn(LD_BUG, "Unrecognized signal %lu in control_event_signal",
+             (unsigned long)signal_num);
+    return -1;
+  }
+
+  send_control_event(EVENT_GOT_SIGNAL,  "650 SIGNAL %s\r\n",
+                     signal_string);
+  return 0;
+}
+
+/** Called when a single local_routerstatus_t has changed: Sends an NS event
+ * to any controller that cares. */
+int
+control_event_networkstatus_changed_single(const routerstatus_t *rs)
+{
+  smartlist_t *statuses;
+  int r;
+
+  if (!EVENT_IS_INTERESTING(EVENT_NS))
+    return 0;
+
+  statuses = smartlist_new();
+  smartlist_add(statuses, (void*)rs);
+  r = control_event_networkstatus_changed(statuses);
+  smartlist_free(statuses);
+  return r;
+}
+
+/** Our own router descriptor has changed; tell any controllers that care.
+ */
+int
+control_event_my_descriptor_changed(void)
+{
+  send_control_event(EVENT_DESCCHANGED,  "650 DESCCHANGED\r\n");
+  return 0;
+}
+
+/** Helper: sends a status event where <b>type</b> is one of
+ * EVENT_STATUS_{GENERAL,CLIENT,SERVER}, where <b>severity</b> is one of
+ * LOG_{NOTICE,WARN,ERR}, and where <b>format</b> is a printf-style format
+ * string corresponding to <b>args</b>. */
+static int
+control_event_status(int type, int severity, const char *format, va_list args)
+{
+  char *user_buf = NULL;
+  char format_buf[160];
+  const char *status, *sev;
+
+  switch (type) {
+    case EVENT_STATUS_GENERAL:
+      status = "STATUS_GENERAL";
+      break;
+    case EVENT_STATUS_CLIENT:
+      status = "STATUS_CLIENT";
+      break;
+    case EVENT_STATUS_SERVER:
+      status = "STATUS_SERVER";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status type %d", type);
+      return -1;
+  }
+  switch (severity) {
+    case LOG_NOTICE:
+      sev = "NOTICE";
+      break;
+    case LOG_WARN:
+      sev = "WARN";
+      break;
+    case LOG_ERR:
+      sev = "ERR";
+      break;
+    default:
+      log_warn(LD_BUG, "Unrecognized status severity %d", severity);
+      return -1;
+  }
+  if (tor_snprintf(format_buf, sizeof(format_buf), "650 %s %s",
+                   status, sev)<0) {
+    log_warn(LD_BUG, "Format string too long.");
+    return -1;
+  }
+  if (tor_vasprintf(&user_buf, format, args)<0) {
+    log_warn(LD_BUG, "Failed to create user buffer.");
+    return -1;
+  }
+
+  send_control_event(type,  "%s %s\r\n", format_buf, user_buf);
+  tor_free(user_buf);
+  return 0;
+}
+
+#ifndef COCCI
+#define CONTROL_EVENT_STATUS_BODY(event, sev)                   \
+  int r;                                                        \
+  do {                                                          \
+    va_list ap;                                                 \
+    if (!EVENT_IS_INTERESTING(event))                           \
+      return 0;                                                 \
+                                                                \
+    va_start(ap, format);                                       \
+    r = control_event_status((event), (sev), format, ap);       \
+    va_end(ap);                                                 \
+  } while (0)
+#endif /* !defined(COCCI) */
+
+/** Format and send an EVENT_STATUS_GENERAL event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_general_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_GENERAL LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_general_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_GENERAL, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_client_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_CLIENT LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_client_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_CLIENT, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER event whose main text is obtained
+ * by formatting the arguments using the printf-style <b>format</b>. */
+int
+control_event_server_status(int severity, const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, severity);
+  return r;
+}
+
+/** Format and send an EVENT_STATUS_SERVER LOG_ERR event, and flush it to the
+ * controller(s) immediately. */
+int
+control_event_server_error(const char *format, ...)
+{
+  CONTROL_EVENT_STATUS_BODY(EVENT_STATUS_SERVER, LOG_ERR);
+  /* Force a flush, since we may be about to die horribly */
+  queued_events_flush_all(1);
+  return r;
+}
+
+/** Called when the status of an entry guard with the given <b>nickname</b>
+ * and identity <b>digest</b> has changed to <b>status</b>: tells any
+ * controllers that care. */
+int
+control_event_guard(const char *nickname, const char *digest,
+                    const char *status)
+{
+  char hbuf[HEX_DIGEST_LEN+1];
+  base16_encode(hbuf, sizeof(hbuf), digest, DIGEST_LEN);
+  if (!EVENT_IS_INTERESTING(EVENT_GUARD))
+    return 0;
+
+  {
+    char buf[MAX_VERBOSE_NICKNAME_LEN+1];
+    const node_t *node = node_get_by_id(digest);
+    if (node) {
+      node_get_verbose_nickname(node, buf);
+    } else {
+      tor_snprintf(buf, sizeof(buf), "$%s~%s", hbuf, nickname);
+    }
+    send_control_event(EVENT_GUARD,
+                       "650 GUARD ENTRY %s %s\r\n", buf, status);
+  }
+  return 0;
+}
+
+/** Called when a configuration option changes. This is generally triggered
+ * by SETCONF requests and RELOAD/SIGHUP signals. The <b>changes</b> are
+ * a linked list of configuration key-values.
+ * <b>changes</b> can be NULL, meaning "no changes".
+ */
+void
+control_event_conf_changed(const config_line_t *changes)
+{
+  char *result;
+  smartlist_t *lines;
+  if (!EVENT_IS_INTERESTING(EVENT_CONF_CHANGED) || !changes) {
+    return;
+  }
+  lines = smartlist_new();
+  for (const config_line_t *line = changes; line; line = line->next) {
+    if (line->value == NULL) {
+      smartlist_add_asprintf(lines, "650-%s", line->key);
+    } else {
+      smartlist_add_asprintf(lines, "650-%s=%s", line->key, line->value);
+    }
+  }
+  result = smartlist_join_strings(lines, "\r\n", 0, NULL);
+  send_control_event(EVENT_CONF_CHANGED,
+    "650-CONF_CHANGED\r\n%s\r\n650 OK\r\n", result);
+  tor_free(result);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+}
+
+/** We just generated a new summary of which countries we've seen clients
+ * from recently. Send a copy to the controller in case it wants to
+ * display it for the user. */
+void
+control_event_clients_seen(const char *controller_str)
+{
+  send_control_event(EVENT_CLIENTS_SEEN,
+    "650 CLIENTS_SEEN %s\r\n", controller_str);
+}
+
+/** A new pluggable transport called <b>transport_name</b> was
+ *  launched on <b>addr</b>:<b>port</b>. <b>mode</b> is either
+ *  "server" or "client" depending on the mode of the pluggable
+ *  transport.
+ *  "650" SP "TRANSPORT_LAUNCHED" SP Mode SP Name SP Address SP Port
+ */
+void
+control_event_transport_launched(const char *mode, const char *transport_name,
+                                 tor_addr_t *addr, uint16_t port)
+{
+  send_control_event(EVENT_TRANSPORT_LAUNCHED,
+                     "650 TRANSPORT_LAUNCHED %s %s %s %u\r\n",
+                     mode, transport_name, fmt_addr(addr), port);
+}
+
+/** A pluggable transport called <b>pt_name</b> has emitted a log message
+ * found in <b>message</b> at <b>severity</b> log level. */
+void
+control_event_pt_log(const char *log)
+{
+  send_control_event(EVENT_PT_LOG,
+                     "650 PT_LOG %s\r\n",
+                     log);
+}
+
+/** A pluggable transport has emitted a STATUS message found in
+ * <b>status</b>. */
+void
+control_event_pt_status(const char *status)
+{
+  send_control_event(EVENT_PT_STATUS,
+                     "650 PT_STATUS %s\r\n",
+                     status);
+}
+
+/** Convert rendezvous auth type to string for HS_DESC control events
+ */
+const char *
+rend_auth_type_to_string(rend_auth_type_t auth_type)
+{
+  const char *str;
+
+  switch (auth_type) {
+    case REND_NO_AUTH:
+      str = "NO_AUTH";
+      break;
+    case REND_BASIC_AUTH:
+      str = "BASIC_AUTH";
+      break;
+    case REND_STEALTH_AUTH:
+      str = "STEALTH_AUTH";
+      break;
+    default:
+      str = "UNKNOWN";
+  }
+
+  return str;
+}
+
+/** Return either the onion address if the given pointer is a non empty
+ * string else the unknown string. */
+static const char *
+rend_hsaddress_str_or_unknown(const char *onion_address)
+{
+  static const char *str_unknown = "UNKNOWN";
+  const char *str_ret = str_unknown;
+
+  /* No valid pointer, unknown it is. */
+  if (!onion_address) {
+    goto end;
+  }
+  /* Empty onion address thus we don't know, unknown it is. */
+  if (onion_address[0] == '\0') {
+    goto end;
+  }
+  /* All checks are good so return the given onion address. */
+  str_ret = onion_address;
+
+ end:
+  return str_ret;
+}
+
+/** send HS_DESC requested event.
+ *
+ * <b>rend_query</b> is used to fetch requested onion address and auth type.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id_base32</b> is the ID of requested hs descriptor.
+ * <b>hsdir_index</b> is the HSDir fetch index value for v3, an hex string.
+ */
+void
+control_event_hs_descriptor_requested(const char *onion_address,
+                                      rend_auth_type_t auth_type,
+                                      const char *id_digest,
+                                      const char *desc_id,
+                                      const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC REQUESTED %s %s %s %s%s\r\n",
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC CREATED event when a local service generates a descriptor.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>desc_id</b> is the descriptor ID.
+ * <b>replica</b> is the the descriptor replica number. If it is negative, it
+ * is ignored.
+ */
+void
+control_event_hs_descriptor_created(const char *onion_address,
+                                    const char *desc_id,
+                                    int replica)
+{
+  char *replica_field = NULL;
+
+  if (BUG(!onion_address || !desc_id)) {
+    return;
+  }
+
+  if (replica >= 0) {
+    tor_asprintf(&replica_field, " REPLICA=%d", replica);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC CREATED %s UNKNOWN UNKNOWN %s%s\r\n",
+                     onion_address, desc_id,
+                     replica_field ? replica_field : "");
+  tor_free(replica_field);
+}
+
+/** send HS_DESC upload event.
+ *
+ * <b>onion_address</b> is service address.
+ * <b>hs_dir</b> is the description of contacting hs directory.
+ * <b>desc_id</b> is the ID of requested hs descriptor.
+ */
+void
+control_event_hs_descriptor_upload(const char *onion_address,
+                                   const char *id_digest,
+                                   const char *desc_id,
+                                   const char *hsdir_index)
+{
+  char *hsdir_index_field = NULL;
+
+  if (BUG(!onion_address || !id_digest || !desc_id)) {
+    return;
+  }
+
+  if (hsdir_index) {
+    tor_asprintf(&hsdir_index_field, " HSDIR_INDEX=%s", hsdir_index);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC UPLOAD %s UNKNOWN %s %s%s\r\n",
+                     onion_address,
+                     node_describe_longname_by_id(id_digest),
+                     desc_id,
+                     hsdir_index_field ? hsdir_index_field : "");
+  tor_free(hsdir_index_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hsv2_descriptor_received
+ * control_event_hsv2_descriptor_failed
+ * control_event_hsv3_descriptor_failed
+ *
+ * So do not call this function directly.
+ */
+static void
+event_hs_descriptor_receive_end(const char *action,
+                                const char *onion_address,
+                                const char *desc_id,
+                                rend_auth_type_t auth_type,
+                                const char *hsdir_id_digest,
+                                const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !onion_address)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s %s %s%s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     rend_auth_type_to_string(auth_type),
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     desc_id ? desc_id : "",
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** send HS_DESC event after got response from hs directory.
+ *
+ * NOTE: this is an internal function used by following functions:
+ * control_event_hs_descriptor_uploaded
+ * control_event_hs_descriptor_upload_failed
+ *
+ * So do not call this function directly.
+ */
+void
+control_event_hs_descriptor_upload_end(const char *action,
+                                       const char *onion_address,
+                                       const char *id_digest,
+                                       const char *reason)
+{
+  char *reason_field = NULL;
+
+  if (BUG(!action || !id_digest)) {
+    return;
+  }
+
+  if (reason) {
+    tor_asprintf(&reason_field, " REASON=%s", reason);
+  }
+
+  send_control_event(EVENT_HS_DESC,
+                     "650 HS_DESC %s %s UNKNOWN %s%s\r\n",
+                     action,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     node_describe_longname_by_id(id_digest),
+                     reason_field ? reason_field : "");
+
+  tor_free(reason_field);
+}
+
+/** For an HS descriptor query <b>rend_data</b>, using the
+ * <b>onion_address</b> and HSDir fingerprint <b>hsdir_fp</b>, find out
+ * which descriptor ID in the query is the right one.
+ *
+ * Return a pointer of the binary descriptor ID found in the query's object
+ * or NULL if not found. */
+static const char *
+get_desc_id_from_query(const rend_data_t *rend_data, const char *hsdir_fp)
+{
+  int replica;
+  const char *desc_id = NULL;
+  const rend_data_v2_t *rend_data_v2 = TO_REND_DATA_V2(rend_data);
+
+  /* Possible if the fetch was done using a descriptor ID. This means that
+   * the HSFETCH command was used. */
+  if (!tor_digest_is_zero(rend_data_v2->desc_id_fetch)) {
+    desc_id = rend_data_v2->desc_id_fetch;
+    goto end;
+  }
+
+  /* Without a directory fingerprint at this stage, we can't do much. */
+  if (hsdir_fp == NULL) {
+     goto end;
+  }
+
+  /* OK, we have an onion address so now let's find which descriptor ID
+   * is the one associated with the HSDir fingerprint. */
+  for (replica = 0; replica < REND_NUMBER_OF_NON_CONSECUTIVE_REPLICAS;
+       replica++) {
+    const char *digest = rend_data_get_desc_id(rend_data, replica, NULL);
+
+    SMARTLIST_FOREACH_BEGIN(rend_data->hsdirs_fp, char *, fingerprint) {
+      if (tor_memcmp(fingerprint, hsdir_fp, DIGEST_LEN) == 0) {
+        /* Found it! This descriptor ID is the right one. */
+        desc_id = digest;
+        goto end;
+      }
+    } SMARTLIST_FOREACH_END(fingerprint);
+  }
+
+ end:
+  return desc_id;
+}
+
+/** send HS_DESC RECEIVED event
+ *
+ * called when we successfully received a hidden service descriptor.
+ */
+void
+control_event_hsv2_descriptor_received(const char *onion_address,
+                                       const rend_data_t *rend_data,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data || !hsdir_id_digest || !onion_address)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/* Send HS_DESC RECEIVED event
+ *
+ * Called when we successfully received a hidden service descriptor. */
+void
+control_event_hsv3_descriptor_received(const char *onion_address,
+                                       const char *desc_id,
+                                       const char *hsdir_id_digest)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !hsdir_id_digest)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("RECEIVED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, NULL);
+  tor_free(desc_id_field);
+}
+
+/** send HS_DESC UPLOADED event
+ *
+ * called when we successfully uploaded a hidden service descriptor.
+ */
+void
+control_event_hs_descriptor_uploaded(const char *id_digest,
+                                     const char *onion_address)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+
+  control_event_hs_descriptor_upload_end("UPLOADED", onion_address,
+                                         id_digest, NULL);
+}
+
+/** Send HS_DESC event to inform controller that query <b>rend_data</b>
+ * failed to retrieve hidden service descriptor from directory identified by
+ * <b>id_digest</b>. If NULL, "UNKNOWN" is used. If <b>reason</b> is not NULL,
+ * add it to REASON= field.
+ */
+void
+control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+  const char *desc_id;
+
+  if (BUG(!rend_data)) {
+    return;
+  }
+
+  desc_id = get_desc_id_from_query(rend_data, hsdir_id_digest);
+  if (desc_id != NULL) {
+    char desc_id_base32[REND_DESC_ID_V2_LEN_BASE32 + 1];
+    /* Set the descriptor ID digest to base32 so we can send it. */
+    base32_encode(desc_id_base32, sizeof(desc_id_base32), desc_id,
+                  DIGEST_LEN);
+    /* Extra whitespace is needed before the value. */
+    tor_asprintf(&desc_id_field, " %s", desc_id_base32);
+  }
+
+  event_hs_descriptor_receive_end("FAILED", rend_data_get_address(rend_data),
+                                  desc_id_field,
+                                  TO_REND_DATA_V2(rend_data)->auth_type,
+                                  hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC event to inform controller that the query to
+ * <b>onion_address</b> failed to retrieve hidden service descriptor
+ * <b>desc_id</b> from directory identified by <b>hsdir_id_digest</b>. If
+ * NULL, "UNKNOWN" is used.  If <b>reason</b> is not NULL, add it to REASON=
+ * field. */
+void
+control_event_hsv3_descriptor_failed(const char *onion_address,
+                                     const char *desc_id,
+                                     const char *hsdir_id_digest,
+                                     const char *reason)
+{
+  char *desc_id_field = NULL;
+
+  if (BUG(!onion_address || !desc_id || !reason)) {
+    return;
+  }
+
+  /* Because DescriptorID is an optional positional value, we need to add a
+   * whitespace before in order to not be next to the HsDir value. */
+  tor_asprintf(&desc_id_field, " %s", desc_id);
+
+  event_hs_descriptor_receive_end("FAILED", onion_address, desc_id_field,
+                                  REND_NO_AUTH, hsdir_id_digest, reason);
+  tor_free(desc_id_field);
+}
+
+/** Send HS_DESC_CONTENT event after completion of a successful fetch
+ * from hs directory. If <b>hsdir_id_digest</b> is NULL, it is replaced
+ * by "UNKNOWN". If <b>content</b> is NULL, it is replaced by an empty
+ * string. The  <b>onion_address</b> or <b>desc_id</b> set to NULL will
+ * not trigger the control event. */
+void
+control_event_hs_descriptor_content(const char *onion_address,
+                                    const char *desc_id,
+                                    const char *hsdir_id_digest,
+                                    const char *content)
+{
+  static const char *event_name = "HS_DESC_CONTENT";
+  char *esc_content = NULL;
+
+  if (!onion_address || !desc_id) {
+    log_warn(LD_BUG, "Called with onion_address==%p, desc_id==%p, ",
+             onion_address, desc_id);
+    return;
+  }
+
+  if (content == NULL) {
+    /* Point it to empty content so it can still be escaped. */
+    content = "";
+  }
+  write_escaped_data(content, strlen(content), &esc_content);
+
+  send_control_event(EVENT_HS_DESC_CONTENT,
+                     "650+%s %s %s %s\r\n%s650 OK\r\n",
+                     event_name,
+                     rend_hsaddress_str_or_unknown(onion_address),
+                     desc_id,
+                     hsdir_id_digest ?
+                        node_describe_longname_by_id(hsdir_id_digest) :
+                        "UNKNOWN",
+                     esc_content);
+  tor_free(esc_content);
+}
+
+/** Send HS_DESC event to inform controller upload of hidden service
+ * descriptor identified by <b>id_digest</b> failed. If <b>reason</b>
+ * is not NULL, add it to REASON= field.
+ */
+void
+control_event_hs_descriptor_upload_failed(const char *id_digest,
+                                          const char *onion_address,
+                                          const char *reason)
+{
+  if (BUG(!id_digest)) {
+    return;
+  }
+  control_event_hs_descriptor_upload_end("FAILED", onion_address,
+                                         id_digest, reason);
+}
+
+void
+control_events_free_all(void)
+{
+  smartlist_t *queued_events = NULL;
+
+  stats_prev_n_read = stats_prev_n_written = 0;
+
+  if (queued_control_events_lock) {
+    tor_mutex_acquire(queued_control_events_lock);
+    flush_queued_event_pending = 0;
+    queued_events = queued_control_events;
+    queued_control_events = NULL;
+    tor_mutex_release(queued_control_events_lock);
+  }
+  if (queued_events) {
+    SMARTLIST_FOREACH(queued_events, queued_event_t *, ev,
+                      queued_event_free(ev));
+    smartlist_free(queued_events);
+  }
+  if (flush_queued_events_event) {
+    mainloop_event_free(flush_queued_events_event);
+    flush_queued_events_event = NULL;
+  }
+  global_event_mask = 0;
+  disable_log_messages = 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+/* For testing: change the value of global_event_mask */
+void
+control_testing_set_global_event_mask(uint64_t mask)
+{
+  global_event_mask = mask;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/control/control_events.h b/src/feature/control/control_events.h
new file mode 100644
index 0000000000..4a5492b510
--- /dev/null
+++ b/src/feature/control/control_events.h
@@ -0,0 +1,352 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_events.h
+ * \brief Header file for control_events.c.
+ **/
+
+#ifndef TOR_CONTROL_EVENTS_H
+#define TOR_CONTROL_EVENTS_H
+
+#include "lib/cc/ctassert.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+
+struct config_line_t;
+
+/** Used to indicate the type of a CIRC_MINOR event passed to the controller.
+ * The various types are defined in control-spec.txt . */
+typedef enum circuit_status_minor_event_t {
+  CIRC_MINOR_EVENT_PURPOSE_CHANGED,
+  CIRC_MINOR_EVENT_CANNIBALIZED,
+} circuit_status_minor_event_t;
+
+/** Used to indicate the type of a stream event passed to the controller.
+ * The various types are defined in control-spec.txt */
+typedef enum stream_status_event_t {
+  STREAM_EVENT_SENT_CONNECT = 0,
+  STREAM_EVENT_SENT_RESOLVE = 1,
+  STREAM_EVENT_SUCCEEDED    = 2,
+  STREAM_EVENT_FAILED       = 3,
+  STREAM_EVENT_CLOSED       = 4,
+  STREAM_EVENT_NEW          = 5,
+  STREAM_EVENT_NEW_RESOLVE  = 6,
+  STREAM_EVENT_FAILED_RETRIABLE = 7,
+  STREAM_EVENT_REMAP        = 8
+} stream_status_event_t;
+
+/** Used to indicate the type of a buildtime event */
+typedef enum buildtimeout_set_event_t {
+  BUILDTIMEOUT_SET_EVENT_COMPUTED  = 0,
+  BUILDTIMEOUT_SET_EVENT_RESET     = 1,
+  BUILDTIMEOUT_SET_EVENT_SUSPENDED = 2,
+  BUILDTIMEOUT_SET_EVENT_DISCARD = 3,
+  BUILDTIMEOUT_SET_EVENT_RESUME = 4
+} buildtimeout_set_event_t;
+
+/** Enum describing various stages of bootstrapping, for use with controller
+ * bootstrap status events. The values range from 0 to 100. */
+typedef enum {
+  BOOTSTRAP_STATUS_UNDEF=-1,
+  BOOTSTRAP_STATUS_STARTING=0,
+
+  /* Initial connection to any relay */
+
+  BOOTSTRAP_STATUS_CONN_PT=1,
+  BOOTSTRAP_STATUS_CONN_DONE_PT=2,
+  BOOTSTRAP_STATUS_CONN_PROXY=3,
+  BOOTSTRAP_STATUS_CONN_DONE_PROXY=4,
+  BOOTSTRAP_STATUS_CONN=5,
+  BOOTSTRAP_STATUS_CONN_DONE=10,
+  BOOTSTRAP_STATUS_HANDSHAKE=14,
+  BOOTSTRAP_STATUS_HANDSHAKE_DONE=15,
+
+  /* Loading directory info */
+
+  BOOTSTRAP_STATUS_ONEHOP_CREATE=20,
+  BOOTSTRAP_STATUS_REQUESTING_STATUS=25,
+  BOOTSTRAP_STATUS_LOADING_STATUS=30,
+  BOOTSTRAP_STATUS_LOADING_KEYS=40,
+  BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS=45,
+  BOOTSTRAP_STATUS_LOADING_DESCRIPTORS=50,
+  BOOTSTRAP_STATUS_ENOUGH_DIRINFO=75,
+
+  /* Connecting to a relay for AP circuits */
+
+  BOOTSTRAP_STATUS_AP_CONN_PT=76,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PT=77,
+  BOOTSTRAP_STATUS_AP_CONN_PROXY=78,
+  BOOTSTRAP_STATUS_AP_CONN_DONE_PROXY=79,
+  BOOTSTRAP_STATUS_AP_CONN=80,
+  BOOTSTRAP_STATUS_AP_CONN_DONE=85,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE=89,
+  BOOTSTRAP_STATUS_AP_HANDSHAKE_DONE=90,
+
+  /* Creating AP circuits */
+
+  BOOTSTRAP_STATUS_CIRCUIT_CREATE=95,
+  BOOTSTRAP_STATUS_DONE=100
+} bootstrap_status_t;
+
+/** Reason for remapping an AP connection's address: we have a cached
+ * answer. */
+#define REMAP_STREAM_SOURCE_CACHE 1
+/** Reason for remapping an AP connection's address: the exit node told us an
+ * answer. */
+#define REMAP_STREAM_SOURCE_EXIT 2
+
+void control_initialize_event_queue(void);
+
+void control_update_global_event_mask(void);
+void control_adjust_event_log_severity(void);
+
+#define EVENT_NS 0x000F
+int control_event_is_interesting(int event);
+
+void control_per_second_events(void);
+int control_any_per_second_event_enabled(void);
+
+int control_event_circuit_status(origin_circuit_t *circ,
+                                 circuit_status_event_t e, int reason);
+int control_event_circuit_purpose_changed(origin_circuit_t *circ,
+                                          int old_purpose);
+int control_event_circuit_cannibalized(origin_circuit_t *circ,
+                                       int old_purpose,
+                                       const struct timeval *old_tv_created);
+int control_event_stream_status(entry_connection_t *conn,
+                                stream_status_event_t e,
+                                int reason);
+int control_event_or_conn_status(or_connection_t *conn,
+                                 or_conn_status_event_t e, int reason);
+int control_event_bandwidth_used(uint32_t n_read, uint32_t n_written);
+int control_event_stream_bandwidth(edge_connection_t *edge_conn);
+int control_event_stream_bandwidth_used(void);
+int control_event_circ_bandwidth_used(void);
+int control_event_circ_bandwidth_used_for_circ(origin_circuit_t *ocirc);
+int control_event_conn_bandwidth(connection_t *conn);
+int control_event_conn_bandwidth_used(void);
+int control_event_circuit_cell_stats(void);
+void control_event_logmsg(int severity, log_domain_mask_t domain,
+                          const char *msg);
+void control_event_logmsg_pending(void);
+int control_event_descriptors_changed(smartlist_t *routers);
+int control_event_address_mapped(const char *from, const char *to,
+                                 time_t expires, const char *error,
+                                 const int cached);
+int control_event_my_descriptor_changed(void);
+int control_event_network_liveness_update(int liveness);
+int control_event_networkstatus_changed(smartlist_t *statuses);
+
+int control_event_newconsensus(const networkstatus_t *consensus);
+int control_event_networkstatus_changed_single(const routerstatus_t *rs);
+int control_event_general_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_client_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+int control_event_server_status(int severity, const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+int control_event_general_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_client_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+int control_event_server_error(const char *format, ...)
+  CHECK_PRINTF(1,2);
+
+int control_event_guard(const char *nickname, const char *digest,
+                        const char *status);
+void control_event_conf_changed(const struct config_line_t *changes);
+int control_event_buildtimeout_set(buildtimeout_set_event_t type,
+                                   const char *args);
+int control_event_signal(uintptr_t signal);
+
+void control_event_bootstrap(bootstrap_status_t status, int progress);
+int control_get_bootstrap_percent(void);
+MOCK_DECL(void, control_event_bootstrap_prob_or,(const char *warn,
+                                                 int reason,
+                                                 or_connection_t *or_conn));
+void control_event_boot_dir(bootstrap_status_t status, int progress);
+void control_event_boot_first_orconn(void);
+void control_event_bootstrap_problem(const char *warn, const char *reason,
+                                     const connection_t *conn, int dowarn);
+char *control_event_boot_last_msg(void);
+void control_event_bootstrap_reset(void);
+
+void control_event_clients_seen(const char *controller_str);
+void control_event_transport_launched(const char *mode,
+                                      const char *transport_name,
+                                      tor_addr_t *addr, uint16_t port);
+void control_event_pt_log(const char *log);
+void control_event_pt_status(const char *status);
+
+void control_event_hs_descriptor_requested(const char *onion_address,
+                                           rend_auth_type_t auth_type,
+                                           const char *id_digest,
+                                           const char *desc_id,
+                                           const char *hsdir_index);
+void control_event_hs_descriptor_created(const char *onion_address,
+                                         const char *desc_id,
+                                         int replica);
+void control_event_hs_descriptor_upload(const char *onion_address,
+                                        const char *desc_id,
+                                        const char *hs_dir,
+                                        const char *hsdir_index);
+void control_event_hs_descriptor_upload_end(const char *action,
+                                            const char *onion_address,
+                                            const char *hs_dir,
+                                            const char *reason);
+void control_event_hs_descriptor_uploaded(const char *hs_dir,
+                                          const char *onion_address);
+/* Hidden service v2 HS_DESC specific. */
+void control_event_hsv2_descriptor_failed(const rend_data_t *rend_data,
+                                          const char *id_digest,
+                                          const char *reason);
+void control_event_hsv2_descriptor_received(const char *onion_address,
+                                            const rend_data_t *rend_data,
+                                            const char *id_digest);
+/* Hidden service v3 HS_DESC specific. */
+void control_event_hsv3_descriptor_failed(const char *onion_address,
+                                          const char *desc_id,
+                                          const char *hsdir_id_digest,
+                                          const char *reason);
+void control_event_hsv3_descriptor_received(const char *onion_address,
+                                            const char *desc_id,
+                                            const char *hsdir_id_digest);
+void control_event_hs_descriptor_upload_failed(const char *hs_dir,
+                                               const char *onion_address,
+                                               const char *reason);
+void control_event_hs_descriptor_content(const char *onion_address,
+                                         const char *desc_id,
+                                         const char *hsdir_fp,
+                                         const char *content);
+
+void control_events_free_all(void);
+
+#ifdef CONTROL_MODULE_PRIVATE
+char *get_bw_samples(void);
+#endif /* defined(CONTROL_MODULE_PRIVATE) */
+
+#ifdef CONTROL_EVENTS_PRIVATE
+/** Bitfield: The bit 1&lt;&lt;e is set if <b>any</b> open control
+ * connection is interested in events of type <b>e</b>.  We use this
+ * so that we can decide to skip generating event messages that nobody
+ * has interest in without having to walk over the global connection
+ * list to find out.
+ **/
+typedef uint64_t event_mask_t;
+
+/* Recognized asynchronous event types.  It's okay to expand this list
+ * because it is used both as a list of v0 event types, and as indices
+ * into the bitfield to determine which controllers want which events.
+ */
+/* This bitfield has no event zero    0x0000 */
+#define EVENT_MIN_                    0x0001
+#define EVENT_CIRCUIT_STATUS          0x0001
+#define EVENT_STREAM_STATUS           0x0002
+#define EVENT_OR_CONN_STATUS          0x0003
+#define EVENT_BANDWIDTH_USED          0x0004
+#define EVENT_CIRCUIT_STATUS_MINOR    0x0005
+#define EVENT_NEW_DESC                0x0006
+#define EVENT_DEBUG_MSG               0x0007
+#define EVENT_INFO_MSG                0x0008
+#define EVENT_NOTICE_MSG              0x0009
+#define EVENT_WARN_MSG                0x000A
+#define EVENT_ERR_MSG                 0x000B
+#define EVENT_ADDRMAP                 0x000C
+/* There was an AUTHDIR_NEWDESCS event, but it no longer exists.  We
+   can reclaim 0x000D. */
+#define EVENT_DESCCHANGED             0x000E
+/* Exposed above */
+// #define EVENT_NS                   0x000F
+#define EVENT_STATUS_CLIENT           0x0010
+#define EVENT_STATUS_SERVER           0x0011
+#define EVENT_STATUS_GENERAL          0x0012
+#define EVENT_GUARD                   0x0013
+#define EVENT_STREAM_BANDWIDTH_USED   0x0014
+#define EVENT_CLIENTS_SEEN            0x0015
+#define EVENT_NEWCONSENSUS            0x0016
+#define EVENT_BUILDTIMEOUT_SET        0x0017
+#define EVENT_GOT_SIGNAL              0x0018
+#define EVENT_CONF_CHANGED            0x0019
+#define EVENT_CONN_BW                 0x001A
+#define EVENT_CELL_STATS              0x001B
+/* UNUSED :                           0x001C */
+#define EVENT_CIRC_BANDWIDTH_USED     0x001D
+#define EVENT_TRANSPORT_LAUNCHED      0x0020
+#define EVENT_HS_DESC                 0x0021
+#define EVENT_HS_DESC_CONTENT         0x0022
+#define EVENT_NETWORK_LIVENESS        0x0023
+#define EVENT_PT_LOG                  0x0024
+#define EVENT_PT_STATUS               0x0025
+#define EVENT_MAX_                    0x0025
+
+/* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */
+#define EVENT_CAPACITY_               0x0040
+
+/* If EVENT_MAX_ ever hits 0x0040, we need to make the mask into a
+ * different structure, as it can only handle a maximum left shift of 1<<63. */
+CTASSERT(EVENT_MAX_ < EVENT_CAPACITY_);
+
+#define EVENT_MASK_(e)               (((uint64_t)1)<<(e))
+
+#define EVENT_MASK_NONE_             ((uint64_t)0x0)
+
+#define EVENT_MASK_ABOVE_MIN_        ((~((uint64_t)0x0)) << EVENT_MIN_)
+#define EVENT_MASK_BELOW_MAX_        ((~((uint64_t)0x0)) \
+                                      >> (EVENT_CAPACITY_ - EVENT_MAX_ \
+                                          - EVENT_MIN_))
+
+#define EVENT_MASK_ALL_              (EVENT_MASK_ABOVE_MIN_ \
+                                      & EVENT_MASK_BELOW_MAX_)
+
+/** Helper structure: temporarily stores cell statistics for a circuit. */
+typedef struct cell_stats_t {
+  /** Number of cells added in app-ward direction by command. */
+  uint64_t added_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells added in exit-ward direction by command. */
+  uint64_t added_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in app-ward direction by command. */
+  uint64_t removed_cells_appward[CELL_COMMAND_MAX_ + 1];
+  /** Number of cells removed in exit-ward direction by command. */
+  uint64_t removed_cells_exitward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in app-ward direction by command. */
+  uint64_t total_time_appward[CELL_COMMAND_MAX_ + 1];
+  /** Total waiting time of cells in exit-ward direction by command. */
+  uint64_t total_time_exitward[CELL_COMMAND_MAX_ + 1];
+} cell_stats_t;
+
+void sum_up_cell_stats_by_command(circuit_t *circ,
+                                  cell_stats_t *cell_stats);
+void append_cell_stats_by_command(smartlist_t *event_parts,
+                                  const char *key,
+                                  const uint64_t *include_if_non_zero,
+                                  const uint64_t *number_to_include);
+void format_cell_stats(char **event_string, circuit_t *circ,
+                       cell_stats_t *cell_stats);
+
+/** Helper structure: maps event values to their names. */
+struct control_event_t {
+  uint16_t event_code;
+  const char *event_name;
+};
+
+extern const struct control_event_t control_event_table[];
+
+#ifdef TOR_UNIT_TESTS
+MOCK_DECL(STATIC void,
+          send_control_event_string,(uint16_t event, const char *msg));
+
+MOCK_DECL(STATIC void,
+          queue_control_event_string,(uint16_t event, char *msg));
+
+void control_testing_set_global_event_mask(uint64_t mask);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(CONTROL_EVENTS_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_EVENTS_H) */
diff --git a/src/feature/control/control_fmt.c b/src/feature/control/control_fmt.c
new file mode 100644
index 0000000000..d76e6ad8dd
--- /dev/null
+++ b/src/feature/control/control_fmt.c
@@ -0,0 +1,274 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+
+/** Given an AP connection <b>conn</b> and a <b>len</b>-character buffer
+ * <b>buf</b>, determine the address:port combination requested on
+ * <b>conn</b>, and write it to <b>buf</b>.  Return 0 on success, -1 on
+ * failure. */
+int
+write_stream_target_to_buf(entry_connection_t *conn, char *buf, size_t len)
+{
+  char buf2[256];
+  if (conn->chosen_exit_name)
+    if (tor_snprintf(buf2, sizeof(buf2), ".%s.exit", conn->chosen_exit_name)<0)
+      return -1;
+  if (!conn->socks_request)
+    return -1;
+  if (tor_snprintf(buf, len, "%s%s%s:%d",
+               conn->socks_request->address,
+               conn->chosen_exit_name ? buf2 : "",
+               !conn->chosen_exit_name && connection_edge_is_rendezvous_stream(
+                                     ENTRY_TO_EDGE_CONN(conn)) ? ".onion" : "",
+               conn->socks_request->port)<0)
+    return -1;
+  return 0;
+}
+
+/** Figure out the best name for the target router of an OR connection
+ * <b>conn</b>, and write it into the <b>len</b>-character buffer
+ * <b>name</b>. */
+void
+orconn_target_get_name(char *name, size_t len, or_connection_t *conn)
+{
+  const node_t *node = node_get_by_id(conn->identity_digest);
+  if (node) {
+    tor_assert(len > MAX_VERBOSE_NICKNAME_LEN);
+    node_get_verbose_nickname(node, name);
+  } else if (! tor_digest_is_zero(conn->identity_digest)) {
+    name[0] = '$';
+    base16_encode(name+1, len-1, conn->identity_digest,
+                  DIGEST_LEN);
+  } else {
+    tor_snprintf(name, len, "%s:%d",
+                 conn->base_.address, conn->base_.port);
+  }
+}
+
+/** Allocate and return a description of <b>circ</b>'s current status,
+ * including its path (if any). */
+char *
+circuit_describe_status_for_controller(origin_circuit_t *circ)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  {
+    char *vpath = circuit_list_path_for_controller(circ);
+    if (*vpath) {
+      smartlist_add(descparts, vpath);
+    } else {
+      tor_free(vpath); /* empty path; don't put an extra space in the result */
+    }
+  }
+
+  {
+    cpath_build_state_t *build_state = circ->build_state;
+    smartlist_t *flaglist = smartlist_new();
+    char *flaglist_joined;
+
+    if (build_state->onehop_tunnel)
+      smartlist_add(flaglist, (void *)"ONEHOP_TUNNEL");
+    if (build_state->is_internal)
+      smartlist_add(flaglist, (void *)"IS_INTERNAL");
+    if (build_state->need_capacity)
+      smartlist_add(flaglist, (void *)"NEED_CAPACITY");
+    if (build_state->need_uptime)
+      smartlist_add(flaglist, (void *)"NEED_UPTIME");
+
+    /* Only emit a BUILD_FLAGS argument if it will have a non-empty value. */
+    if (smartlist_len(flaglist)) {
+      flaglist_joined = smartlist_join_strings(flaglist, ",", 0, NULL);
+
+      smartlist_add_asprintf(descparts, "BUILD_FLAGS=%s", flaglist_joined);
+
+      tor_free(flaglist_joined);
+    }
+
+    smartlist_free(flaglist);
+  }
+
+  smartlist_add_asprintf(descparts, "PURPOSE=%s",
+                    circuit_purpose_to_controller_string(circ->base_.purpose));
+
+  {
+    const char *hs_state =
+      circuit_purpose_to_controller_hs_state_string(circ->base_.purpose);
+
+    if (hs_state != NULL) {
+      smartlist_add_asprintf(descparts, "HS_STATE=%s", hs_state);
+    }
+  }
+
+  if (circ->rend_data != NULL || circ->hs_ident != NULL) {
+    char addr[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+    const char *onion_address;
+    if (circ->rend_data) {
+      onion_address = rend_data_get_address(circ->rend_data);
+    } else {
+      hs_build_address(&circ->hs_ident->identity_pk, HS_VERSION_THREE, addr);
+      onion_address = addr;
+    }
+    smartlist_add_asprintf(descparts, "REND_QUERY=%s", onion_address);
+  }
+
+  {
+    char tbuf[ISO_TIME_USEC_LEN+1];
+    format_iso_time_nospace_usec(tbuf, &circ->base_.timestamp_created);
+
+    smartlist_add_asprintf(descparts, "TIME_CREATED=%s", tbuf);
+  }
+
+  // Show username and/or password if available.
+  if (circ->socks_username_len > 0) {
+    char* socks_username_escaped = esc_for_log_len(circ->socks_username,
+                                     (size_t) circ->socks_username_len);
+    smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                           socks_username_escaped);
+    tor_free(socks_username_escaped);
+  }
+  if (circ->socks_password_len > 0) {
+    char* socks_password_escaped = esc_for_log_len(circ->socks_password,
+                                     (size_t) circ->socks_password_len);
+    smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                           socks_password_escaped);
+    tor_free(socks_password_escaped);
+  }
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Allocate and return a description of <b>conn</b>'s current status. */
+char *
+entry_connection_describe_status_for_controller(const entry_connection_t *conn)
+{
+  char *rv;
+  smartlist_t *descparts = smartlist_new();
+
+  if (conn->socks_request != NULL) {
+    // Show username and/or password if available; used by IsolateSOCKSAuth.
+    if (conn->socks_request->usernamelen > 0) {
+      char* username_escaped = esc_for_log_len(conn->socks_request->username,
+                                 (size_t) conn->socks_request->usernamelen);
+      smartlist_add_asprintf(descparts, "SOCKS_USERNAME=%s",
+                             username_escaped);
+      tor_free(username_escaped);
+    }
+    if (conn->socks_request->passwordlen > 0) {
+      char* password_escaped = esc_for_log_len(conn->socks_request->password,
+                                 (size_t) conn->socks_request->passwordlen);
+      smartlist_add_asprintf(descparts, "SOCKS_PASSWORD=%s",
+                             password_escaped);
+      tor_free(password_escaped);
+    }
+
+    const char *client_protocol;
+    // Show the client protocol; used by IsolateClientProtocol.
+    switch (conn->socks_request->listener_type)
+      {
+      case CONN_TYPE_AP_LISTENER:
+        switch (conn->socks_request->socks_version)
+          {
+          case 4: client_protocol = "SOCKS4"; break;
+          case 5: client_protocol = "SOCKS5"; break;
+          default: client_protocol = "UNKNOWN";
+          }
+        break;
+      case CONN_TYPE_AP_TRANS_LISTENER: client_protocol = "TRANS"; break;
+      case CONN_TYPE_AP_NATD_LISTENER: client_protocol = "NATD"; break;
+      case CONN_TYPE_AP_DNS_LISTENER: client_protocol = "DNS"; break;
+      case CONN_TYPE_AP_HTTP_CONNECT_LISTENER:
+        client_protocol = "HTTPCONNECT"; break;
+      default: client_protocol = "UNKNOWN";
+      }
+    smartlist_add_asprintf(descparts, "CLIENT_PROTOCOL=%s",
+                           client_protocol);
+  }
+
+  // Show newnym epoch; used for stream isolation when NEWNYM is used.
+  smartlist_add_asprintf(descparts, "NYM_EPOCH=%u",
+                         conn->nym_epoch);
+
+  // Show session group; used for stream isolation of multiple listener ports.
+  smartlist_add_asprintf(descparts, "SESSION_GROUP=%d",
+                         conn->entry_cfg.session_group);
+
+  // Show isolation flags.
+  smartlist_t *isoflaglist = smartlist_new();
+  char *isoflaglist_joined;
+  if (conn->entry_cfg.isolation_flags & ISO_DESTPORT) {
+    smartlist_add(isoflaglist, (void *)"DESTPORT");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_DESTADDR) {
+    smartlist_add(isoflaglist, (void *)"DESTADDR");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_SOCKSAUTH) {
+    smartlist_add(isoflaglist, (void *)"SOCKS_USERNAME");
+    smartlist_add(isoflaglist, (void *)"SOCKS_PASSWORD");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_CLIENTPROTO) {
+    smartlist_add(isoflaglist, (void *)"CLIENT_PROTOCOL");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_CLIENTADDR) {
+    smartlist_add(isoflaglist, (void *)"CLIENTADDR");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_SESSIONGRP) {
+    smartlist_add(isoflaglist, (void *)"SESSION_GROUP");
+  }
+  if (conn->entry_cfg.isolation_flags & ISO_NYM_EPOCH) {
+    smartlist_add(isoflaglist, (void *)"NYM_EPOCH");
+  }
+  isoflaglist_joined = smartlist_join_strings(isoflaglist, ",", 0, NULL);
+  smartlist_add_asprintf(descparts, "ISO_FIELDS=%s", isoflaglist_joined);
+  tor_free(isoflaglist_joined);
+  smartlist_free(isoflaglist);
+
+  rv = smartlist_join_strings(descparts, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(descparts, char *, cp, tor_free(cp));
+  smartlist_free(descparts);
+
+  return rv;
+}
+
+/** Return a longname the node whose identity is <b>id_digest</b>. If
+ * node_get_by_id() returns NULL, base 16 encoding of <b>id_digest</b> is
+ * returned instead.
+ *
+ * This function is not thread-safe.  Each call to this function invalidates
+ * previous values returned by this function.
+ */
+MOCK_IMPL(const char *,
+node_describe_longname_by_id,(const char *id_digest))
+{
+  static char longname[MAX_VERBOSE_NICKNAME_LEN+1];
+  node_get_verbose_nickname_by_id(id_digest, longname);
+  return longname;
+}
diff --git a/src/feature/control/control_fmt.h b/src/feature/control/control_fmt.h
new file mode 100644
index 0000000000..f3357cfc4e
--- /dev/null
+++ b/src/feature/control/control_fmt.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_fmt.h
+ * \brief Header file for control_fmt.c.
+ **/
+
+#ifndef TOR_CONTROL_FMT_H
+#define TOR_CONTROL_FMT_H
+
+int write_stream_target_to_buf(entry_connection_t *conn, char *buf,
+                               size_t len);
+void orconn_target_get_name(char *buf, size_t len,
+                            or_connection_t *conn);
+char *circuit_describe_status_for_controller(origin_circuit_t *circ);
+char *entry_connection_describe_status_for_controller(const
+                                                     entry_connection_t *conn);
+
+MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest));
+
+#endif /* !defined(TOR_CONTROL_FMT_H) */
diff --git a/src/feature/control/control_getinfo.c b/src/feature/control/control_getinfo.c
new file mode 100644
index 0000000000..0823acbe07
--- /dev/null
+++ b/src/feature/control/control_getinfo.c
@@ -0,0 +1,1773 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_getinfo.c
+ * \brief Implementation for miscellaneous controller getinfo commands.
+ */
+
+#define CONTROL_EVENTS_PRIVATE
+#define CONTROL_MODULE_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/policies.h"
+#include "core/or/versions.h"
+#include "feature/client/addressmap.h"
+#include "feature/client/bridges.h"
+#include "feature/client/entrynodes.h"
+#include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
+#include "feature/control/fmt_serverstatus.h"
+#include "feature/control/getinfo_geoip.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dlstatus.h"
+#include "feature/dircommon/directory.h"
+#include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_cache.h"
+#include "feature/hs_common/shared_random_client.h"
+#include "feature/nodelist/authcert.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerinfo.h"
+#include "feature/nodelist/routerlist.h"
+#include "feature/relay/relay_find_addr.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/rend/rendcache.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "lib/version/torversion.h"
+#include "lib/encoding/kvline.h"
+
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifndef _WIN32
+#include <pwd.h>
+#endif
+
+static char *list_getinfo_options(void);
+static char *download_status_to_string(const download_status_t *dl);
+
+/** Implementation helper for GETINFO: knows the answers for various
+ * trivial-to-implement questions. */
+static int
+getinfo_helper_misc(control_connection_t *conn, const char *question,
+                    char **answer, const char **errmsg)
+{
+  (void) conn;
+  if (!strcmp(question, "version")) {
+    *answer = tor_strdup(get_version());
+  } else if (!strcmp(question, "bw-event-cache")) {
+    *answer = get_bw_samples();
+  } else if (!strcmp(question, "config-file")) {
+    const char *a = get_torrc_fname(0);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-defaults-file")) {
+    const char *a = get_torrc_fname(1);
+    if (a)
+      *answer = tor_strdup(a);
+  } else if (!strcmp(question, "config-text")) {
+    *answer = options_dump(get_options(), OPTIONS_DUMP_MINIMAL);
+  } else if (!strcmp(question, "config-can-saveconf")) {
+    *answer = tor_strdup(get_options()->IncludeUsed ? "0" : "1");
+  } else if (!strcmp(question, "info/names")) {
+    *answer = list_getinfo_options();
+  } else if (!strcmp(question, "dormant")) {
+    int dormant = rep_hist_circbuilding_dormant(time(NULL));
+    *answer = tor_strdup(dormant ? "1" : "0");
+  } else if (!strcmp(question, "events/names")) {
+    int i;
+    smartlist_t *event_names = smartlist_new();
+
+    for (i = 0; control_event_table[i].event_name != NULL; ++i) {
+      smartlist_add(event_names, (char *)control_event_table[i].event_name);
+    }
+
+    *answer = smartlist_join_strings(event_names, " ", 0, NULL);
+
+    smartlist_free(event_names);
+  } else if (!strcmp(question, "signal/names")) {
+    smartlist_t *signal_names = smartlist_new();
+    int j;
+    for (j = 0; signal_table[j].signal_name != NULL; ++j) {
+      smartlist_add(signal_names, (char*)signal_table[j].signal_name);
+    }
+
+    *answer = smartlist_join_strings(signal_names, " ", 0, NULL);
+
+    smartlist_free(signal_names);
+  } else if (!strcmp(question, "features/names")) {
+    *answer = tor_strdup("VERBOSE_NAMES EXTENDED_EVENTS");
+  } else if (!strcmp(question, "address")) {
+    uint32_t addr;
+    if (router_pick_published_address(get_options(), &addr, 0) < 0) {
+      *errmsg = "Address unknown";
+      return -1;
+    }
+    *answer = tor_dup_ip(addr);
+    tor_assert_nonfatal(*answer);
+  } else if (!strcmp(question, "traffic/read")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_read()));
+  } else if (!strcmp(question, "traffic/written")) {
+    tor_asprintf(answer, "%"PRIu64, (get_bytes_written()));
+  } else if (!strcmp(question, "uptime")) {
+    long uptime_secs = get_uptime();
+    tor_asprintf(answer, "%ld", uptime_secs);
+  } else if (!strcmp(question, "process/pid")) {
+    int myPid = -1;
+
+#ifdef _WIN32
+      myPid = _getpid();
+#else
+      myPid = getpid();
+#endif
+
+    tor_asprintf(answer, "%d", myPid);
+  } else if (!strcmp(question, "process/uid")) {
+#ifdef _WIN32
+      *answer = tor_strdup("-1");
+#else
+      int myUid = geteuid();
+      tor_asprintf(answer, "%d", myUid);
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/user")) {
+#ifdef _WIN32
+      *answer = tor_strdup("");
+#else
+      int myUid = geteuid();
+      const struct passwd *myPwEntry = tor_getpwuid(myUid);
+
+      if (myPwEntry) {
+        *answer = tor_strdup(myPwEntry->pw_name);
+      } else {
+        *answer = tor_strdup("");
+      }
+#endif /* defined(_WIN32) */
+  } else if (!strcmp(question, "process/descriptor-limit")) {
+    int max_fds = get_max_sockets();
+    tor_asprintf(answer, "%d", max_fds);
+  } else if (!strcmp(question, "limits/max-mem-in-queues")) {
+    tor_asprintf(answer, "%"PRIu64,
+                 (get_options()->MaxMemInQueues));
+  } else if (!strcmp(question, "fingerprint")) {
+    crypto_pk_t *server_key;
+    if (!server_mode(get_options())) {
+      *errmsg = "Not running in server mode";
+      return -1;
+    }
+    server_key = get_server_identity_key();
+    *answer = tor_malloc(HEX_DIGEST_LEN+1);
+    crypto_pk_get_fingerprint(server_key, *answer, 0);
+  }
+  return 0;
+}
+
+/** Awful hack: return a newly allocated string based on a routerinfo and
+ * (possibly) an extrainfo, sticking the read-history and write-history from
+ * <b>ei</b> into the resulting string.  The thing you get back won't
+ * necessarily have a valid signature.
+ *
+ * New code should never use this; it's for backward compatibility.
+ *
+ * NOTE: <b>ri_body</b> is as returned by signed_descriptor_get_body: it might
+ * not be NUL-terminated. */
+static char *
+munge_extrainfo_into_routerinfo(const char *ri_body,
+                                const signed_descriptor_t *ri,
+                                const signed_descriptor_t *ei)
+{
+  char *out = NULL, *outp;
+  int i;
+  const char *router_sig;
+  const char *ei_body = signed_descriptor_get_body(ei);
+  size_t ri_len = ri->signed_descriptor_len;
+  size_t ei_len = ei->signed_descriptor_len;
+  if (!ei_body)
+    goto bail;
+
+  outp = out = tor_malloc(ri_len+ei_len+1);
+  if (!(router_sig = tor_memstr(ri_body, ri_len, "\nrouter-signature")))
+    goto bail;
+  ++router_sig;
+  memcpy(out, ri_body, router_sig-ri_body);
+  outp += router_sig-ri_body;
+
+  for (i=0; i < 2; ++i) {
+    const char *kwd = i ? "\nwrite-history " : "\nread-history ";
+    const char *cp, *eol;
+    if (!(cp = tor_memstr(ei_body, ei_len, kwd)))
+      continue;
+    ++cp;
+    if (!(eol = memchr(cp, '\n', ei_len - (cp-ei_body))))
+      continue;
+    memcpy(outp, cp, eol-cp+1);
+    outp += eol-cp+1;
+  }
+  memcpy(outp, router_sig, ri_len - (router_sig-ri_body));
+  *outp++ = '\0';
+  tor_assert(outp-out < (int)(ri_len+ei_len+1));
+
+  return out;
+ bail:
+  tor_free(out);
+  return tor_strndup(ri_body, ri->signed_descriptor_len);
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * which ports are bound. */
+static int
+getinfo_helper_listeners(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  int type;
+  smartlist_t *res;
+
+  (void)control_conn;
+  (void)errmsg;
+
+  if (!strcmp(question, "net/listeners/or"))
+    type = CONN_TYPE_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/extor"))
+    type = CONN_TYPE_EXT_OR_LISTENER;
+  else if (!strcmp(question, "net/listeners/dir"))
+    type = CONN_TYPE_DIR_LISTENER;
+  else if (!strcmp(question, "net/listeners/socks"))
+    type = CONN_TYPE_AP_LISTENER;
+  else if (!strcmp(question, "net/listeners/trans"))
+    type = CONN_TYPE_AP_TRANS_LISTENER;
+  else if (!strcmp(question, "net/listeners/natd"))
+    type = CONN_TYPE_AP_NATD_LISTENER;
+  else if (!strcmp(question, "net/listeners/httptunnel"))
+    type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  else if (!strcmp(question, "net/listeners/dns"))
+    type = CONN_TYPE_AP_DNS_LISTENER;
+  else if (!strcmp(question, "net/listeners/control"))
+    type = CONN_TYPE_CONTROL_LISTENER;
+  else
+    return 0; /* unknown key */
+
+  res = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(get_connection_array(), connection_t *, conn) {
+    struct sockaddr_storage ss;
+    socklen_t ss_len = sizeof(ss);
+
+    if (conn->type != type || conn->marked_for_close || !SOCKET_OK(conn->s))
+      continue;
+
+    if (getsockname(conn->s, (struct sockaddr *)&ss, &ss_len) < 0) {
+      smartlist_add_asprintf(res, "%s:%d", conn->address, (int)conn->port);
+    } else {
+      char *tmp = tor_sockaddr_to_str((struct sockaddr *)&ss);
+      smartlist_add(res, esc_for_log(tmp));
+      tor_free(tmp);
+    }
+
+  } SMARTLIST_FOREACH_END(conn);
+
+  *answer = smartlist_join_strings(res, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(res, char *, cp, tor_free(cp));
+  smartlist_free(res);
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers requests for information about
+ * the current time in both local and UTC forms. */
+STATIC int
+getinfo_helper_current_time(control_connection_t *control_conn,
+                         const char *question,
+                         char **answer, const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+
+  struct timeval now;
+  tor_gettimeofday(&now);
+  char timebuf[ISO_TIME_LEN+1];
+
+  if (!strcmp(question, "current-time/local"))
+    format_local_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else if (!strcmp(question, "current-time/utc"))
+    format_iso_time_nospace(timebuf, (time_t)now.tv_sec);
+  else
+    return 0;
+
+  *answer = tor_strdup(timebuf);
+  return 0;
+}
+
+/** GETINFO helper for dumping different consensus flavors
+ * returns: 0 on success -1 on error. */
+STATIC int
+getinfo_helper_current_consensus(consensus_flavor_t flavor,
+                                 char** answer,
+                                 const char** errmsg)
+{
+  const char *flavor_name = networkstatus_get_flavor_name(flavor);
+  if (BUG(!strcmp(flavor_name, "??"))) {
+    *errmsg = "Internal error: unrecognized flavor name.";
+    return -1;
+  }
+  if (we_want_to_fetch_flavor(get_options(), flavor)) {
+    /** Check from the cache */
+    const cached_dir_t *consensus = dirserv_get_consensus(flavor_name);
+    if (consensus) {
+      *answer = tor_strdup(consensus->dir);
+    }
+  }
+  if (!*answer) { /* try loading it from disk */
+
+    tor_mmap_t *mapped = networkstatus_map_cached_consensus(flavor_name);
+    if (mapped) {
+      *answer = tor_memdup_nulterm(mapped->data, mapped->size);
+      tor_munmap_file(mapped);
+    }
+    if (!*answer) { /* generate an error */
+      *errmsg = "Could not open cached consensus. "
+        "Make sure FetchUselessDescriptors is set to 1.";
+      return -1;
+    }
+  }
+  return 0;
+}
+
+/** Helper for getinfo_helper_dir.
+ *
+ * Add a signed_descriptor_t to <b>descs_out</b> for each router matching
+ * <b>key</b>.  The key should be either
+ *   - "/tor/server/authority" for our own routerinfo;
+ *   - "/tor/server/all" for all the routerinfos we have, concatenated;
+ *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
+ *     hex identity digests; or
+ *   - "/tor/server/d/D" where D is a plus-separated sequence
+ *     of server descriptor digests, in hex.
+ *
+ * Return 0 if we found some matching descriptors, or -1 if we do not
+ * have any descriptors, no matching descriptors, or if we did not
+ * recognize the key (URL).
+ * If -1 is returned *<b>msg</b> will be set to an appropriate error
+ * message.
+ */
+static int
+controller_get_routerdescs(smartlist_t *descs_out, const char *key,
+                        const char **msg)
+{
+  *msg = NULL;
+
+  if (!strcmp(key, "/tor/server/all")) {
+    routerlist_t *rl = router_get_routerlist();
+    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
+                      smartlist_add(descs_out, &(r->cache_info)));
+  } else if (!strcmp(key, "/tor/server/authority")) {
+    const routerinfo_t *ri = router_get_my_routerinfo();
+    if (ri)
+      smartlist_add(descs_out, (void*) &(ri->cache_info));
+  } else if (!strcmpstart(key, "/tor/server/d/")) {
+    smartlist_t *digests = smartlist_new();
+    key += strlen("/tor/server/d/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH(digests, const char *, d,
+       {
+         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
+         if (sd)
+           smartlist_add(descs_out,sd);
+       });
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else if (!strcmpstart(key, "/tor/server/fp/")) {
+    smartlist_t *digests = smartlist_new();
+    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
+    key += strlen("/tor/server/fp/");
+    dir_split_resource_into_fingerprints(key, digests, NULL,
+                                         DSR_HEX|DSR_SORT_UNIQ);
+    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
+         if (router_digest_is_me(d)) {
+           /* calling router_get_my_routerinfo() to make sure it exists */
+           const routerinfo_t *ri = router_get_my_routerinfo();
+           if (ri)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         } else {
+           const routerinfo_t *ri = router_get_by_id_digest(d);
+           /* Don't actually serve a descriptor that everyone will think is
+            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
+            * Tors from downloading descriptors that they will throw away.
+            */
+           if (ri && ri->cache_info.published_on > cutoff)
+             smartlist_add(descs_out, (void*) &(ri->cache_info));
+         }
+    } SMARTLIST_FOREACH_END(d);
+    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
+    smartlist_free(digests);
+  } else {
+    *msg = "Key not recognized";
+    return -1;
+  }
+
+  if (!smartlist_len(descs_out)) {
+    *msg = "Servers unavailable";
+    return -1;
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * directory information. */
+STATIC int
+getinfo_helper_dir(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  (void) control_conn;
+  if (!strcmpstart(question, "desc/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node = node_get_by_hex_id(question+strlen("desc/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/id/* "
+                "instead of desc/id/*.";
+      return 0;
+    }
+  } else if (!strcmpstart(question, "desc/name/")) {
+    const routerinfo_t *ri = NULL;
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node =
+      node_get_by_nickname(question+strlen("desc/name/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *body = signed_descriptor_get_body(&ri->cache_info);
+      if (body)
+        *answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
+    } else if (! we_fetch_router_descriptors(get_options())) {
+      /* Descriptors won't be available, provide proper error */
+      *errmsg = "We fetch microdescriptors, not router "
+                "descriptors. You'll need to use md/name/* "
+                "instead of desc/name/*.";
+      return 0;
+    }
+  } else if (!strcmp(question, "desc/download-enabled")) {
+    int r = we_fetch_router_descriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmp(question, "desc/all-recent")) {
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH(routerlist->routers, const routerinfo_t *, ri,
+      {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        if (body)
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+      });
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmp(question, "desc/all-recent-extrainfo-hack")) {
+    /* XXXX Remove this once Torstat asks for extrainfos. */
+    routerlist_t *routerlist = router_get_routerlist();
+    smartlist_t *sl = smartlist_new();
+    if (routerlist && routerlist->routers) {
+      SMARTLIST_FOREACH_BEGIN(routerlist->routers, const routerinfo_t *, ri) {
+        const char *body = signed_descriptor_get_body(&ri->cache_info);
+        signed_descriptor_t *ei = extrainfo_get_by_descriptor_digest(
+                                     ri->cache_info.extra_info_digest);
+        if (ei && body) {
+          smartlist_add(sl, munge_extrainfo_into_routerinfo(body,
+                                                        &ri->cache_info, ei));
+        } else if (body) {
+          smartlist_add(sl,
+                  tor_strndup(body, ri->cache_info.signed_descriptor_len));
+        }
+      } SMARTLIST_FOREACH_END(ri);
+    }
+    *answer = smartlist_join_strings(sl, "", 0, NULL);
+    SMARTLIST_FOREACH(sl, char *, c, tor_free(c));
+    smartlist_free(sl);
+  } else if (!strcmpstart(question, "hs/client/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/client/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      rend_cache_entry_t *e = NULL;
+      if (!rend_cache_lookup_entry(question, -1, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      const char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_cache_lookup_encoded_as_client(&service_pk);
+      if (desc) {
+        *answer = tor_strdup(desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmpstart(question, "hs/service/desc/id/")) {
+    hostname_type_t addr_type;
+
+    question += strlen("hs/service/desc/id/");
+    if (rend_valid_v2_service_id(question)) {
+      addr_type = ONION_V2_HOSTNAME;
+    } else if (hs_address_is_valid(question)) {
+      addr_type = ONION_V3_HOSTNAME;
+    } else {
+      *errmsg = "Invalid address";
+      return -1;
+    }
+    rend_cache_entry_t *e = NULL;
+
+    if (addr_type == ONION_V2_HOSTNAME) {
+      if (!rend_cache_lookup_v2_desc_as_service(question, &e)) {
+        /* Descriptor found in cache */
+        *answer = tor_strdup(e->desc);
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    } else {
+      ed25519_public_key_t service_pk;
+      char *desc;
+
+      /* The check before this if/else makes sure of this. */
+      tor_assert(addr_type == ONION_V3_HOSTNAME);
+
+      if (hs_parse_address(question, &service_pk, NULL, NULL) < 0) {
+        *errmsg = "Invalid v3 address";
+        return -1;
+      }
+
+      desc = hs_service_lookup_current_desc(&service_pk);
+      if (desc) {
+        /* Newly allocated string, we have ownership. */
+        *answer = desc;
+      } else {
+        *errmsg = "Not found in cache";
+        return -1;
+      }
+    }
+  } else if (!strcmp(question, "md/all")) {
+    const smartlist_t *nodes = nodelist_get_list();
+    tor_assert(nodes);
+
+    if (smartlist_len(nodes) == 0) {
+      *answer = tor_strdup("");
+      return 0;
+    }
+
+    smartlist_t *microdescs = smartlist_new();
+
+    SMARTLIST_FOREACH_BEGIN(nodes, node_t *, n) {
+      if (n->md && n->md->body) {
+        char *copy = tor_strndup(n->md->body, n->md->bodylen);
+        smartlist_add(microdescs, copy);
+      }
+    } SMARTLIST_FOREACH_END(n);
+
+    *answer = smartlist_join_strings(microdescs, "", 0, NULL);
+    SMARTLIST_FOREACH(microdescs, char *, md, tor_free(md));
+    smartlist_free(microdescs);
+  } else if (!strcmpstart(question, "md/id/")) {
+    const node_t *node = node_get_by_hex_id(question+strlen("md/id/"), 0);
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmpstart(question, "md/name/")) {
+    /* XXX Setting 'warn_if_unnamed' here is a bit silly -- the
+     * warning goes to the user, not to the controller. */
+    const node_t *node = node_get_by_nickname(question+strlen("md/name/"), 0);
+    /* XXXX duplicated code */
+    const microdesc_t *md = NULL;
+    if (node) md = node->md;
+    if (md && md->body) {
+      *answer = tor_strndup(md->body, md->bodylen);
+    }
+  } else if (!strcmp(question, "md/download-enabled")) {
+    int r = we_fetch_microdescriptors(get_options());
+    tor_asprintf(answer, "%d", !!r);
+  } else if (!strcmpstart(question, "desc-annotations/id/")) {
+    const routerinfo_t *ri = NULL;
+    const node_t *node =
+      node_get_by_hex_id(question+strlen("desc-annotations/id/"), 0);
+    if (node)
+      ri = node->ri;
+    if (ri) {
+      const char *annotations =
+        signed_descriptor_get_annotations(&ri->cache_info);
+      if (annotations)
+        *answer = tor_strndup(annotations,
+                              ri->cache_info.annotations_len);
+    }
+  } else if (!strcmpstart(question, "dir/server/")) {
+    size_t answer_len = 0;
+    char *url = NULL;
+    smartlist_t *descs = smartlist_new();
+    const char *msg;
+    int res;
+    char *cp;
+    tor_asprintf(&url, "/tor/%s", question+4);
+    res = controller_get_routerdescs(descs, url, &msg);
+    if (res) {
+      log_warn(LD_CONTROL, "getinfo '%s': %s", question, msg);
+      smartlist_free(descs);
+      tor_free(url);
+      *errmsg = msg;
+      return -1;
+    }
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      answer_len += sd->signed_descriptor_len);
+    cp = *answer = tor_malloc(answer_len+1);
+    SMARTLIST_FOREACH(descs, signed_descriptor_t *, sd,
+                      {
+                        memcpy(cp, signed_descriptor_get_body(sd),
+                               sd->signed_descriptor_len);
+                        cp += sd->signed_descriptor_len;
+                      });
+    *cp = '\0';
+    tor_free(url);
+    smartlist_free(descs);
+  } else if (!strcmpstart(question, "dir/status/")) {
+    *answer = tor_strdup("");
+  } else if (!strcmp(question, "dir/status-vote/current/consensus")) {
+    int consensus_result = getinfo_helper_current_consensus(FLAV_NS,
+                                                            answer, errmsg);
+    if (consensus_result < 0) {
+      return -1;
+    }
+  } else if (!strcmp(question,
+                     "dir/status-vote/current/consensus-microdesc")) {
+    int consensus_result = getinfo_helper_current_consensus(FLAV_MICRODESC,
+                                                            answer, errmsg);
+    if (consensus_result < 0) {
+      return -1;
+    }
+  } else if (!strcmp(question, "network-status")) { /* v1 */
+    static int network_status_warned = 0;
+    if (!network_status_warned) {
+      log_warn(LD_CONTROL, "GETINFO network-status is deprecated; it will "
+               "go away in a future version of Tor.");
+      network_status_warned = 1;
+    }
+    routerlist_t *routerlist = router_get_routerlist();
+    if (!routerlist || !routerlist->routers ||
+        list_server_status_v1(routerlist->routers, answer, 1) < 0) {
+      return -1;
+    }
+  } else if (!strcmpstart(question, "extra-info/digest/")) {
+    question += strlen("extra-info/digest/");
+    if (strlen(question) == HEX_DIGEST_LEN) {
+      char d[DIGEST_LEN];
+      signed_descriptor_t *sd = NULL;
+      if (base16_decode(d, sizeof(d), question, strlen(question))
+                        == sizeof(d)) {
+        /* XXXX this test should move into extrainfo_get_by_descriptor_digest,
+         * but I don't want to risk affecting other parts of the code,
+         * especially since the rules for using our own extrainfo (including
+         * when it might be freed) are different from those for using one
+         * we have downloaded. */
+        if (router_extrainfo_digest_is_me(d))
+          sd = &(router_get_my_extrainfo()->cache_info);
+        else
+          sd = extrainfo_get_by_descriptor_digest(d);
+      }
+      if (sd) {
+        const char *body = signed_descriptor_get_body(sd);
+        if (body)
+          *answer = tor_strndup(body, sd->signed_descriptor_len);
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Given a smartlist of 20-byte digests, return a newly allocated string
+ * containing each of those digests in order, formatted in HEX, and terminated
+ * with a newline. */
+static char *
+digest_list_to_string(const smartlist_t *sl)
+{
+  int len;
+  char *result, *s;
+
+  /* Allow for newlines, and a \0 at the end */
+  len = smartlist_len(sl) * (HEX_DIGEST_LEN + 1) + 1;
+  result = tor_malloc_zero(len);
+
+  s = result;
+  SMARTLIST_FOREACH_BEGIN(sl, const char *, digest) {
+    base16_encode(s, HEX_DIGEST_LEN + 1, digest, DIGEST_LEN);
+    s[HEX_DIGEST_LEN] = '\n';
+    s += HEX_DIGEST_LEN + 1;
+  } SMARTLIST_FOREACH_END(digest);
+  *s = '\0';
+
+  return result;
+}
+
+/** Turn a download_status_t into a human-readable description in a newly
+ * allocated string.  The format is specified in control-spec.txt, under
+ * the documentation for "GETINFO download/..." .  */
+static char *
+download_status_to_string(const download_status_t *dl)
+{
+  char *rv = NULL;
+  char tbuf[ISO_TIME_LEN+1];
+  const char *schedule_str, *want_authority_str;
+  const char *increment_on_str, *backoff_str;
+
+  if (dl) {
+    /* Get some substrings of the eventual output ready */
+    format_iso_time(tbuf, download_status_get_next_attempt_at(dl));
+
+    switch (dl->schedule) {
+      case DL_SCHED_GENERIC:
+        schedule_str = "DL_SCHED_GENERIC";
+        break;
+      case DL_SCHED_CONSENSUS:
+        schedule_str = "DL_SCHED_CONSENSUS";
+        break;
+      case DL_SCHED_BRIDGE:
+        schedule_str = "DL_SCHED_BRIDGE";
+        break;
+      default:
+        schedule_str = "unknown";
+        break;
+    }
+
+    switch (dl->want_authority) {
+      case DL_WANT_ANY_DIRSERVER:
+        want_authority_str = "DL_WANT_ANY_DIRSERVER";
+        break;
+      case DL_WANT_AUTHORITY:
+        want_authority_str = "DL_WANT_AUTHORITY";
+        break;
+      default:
+        want_authority_str = "unknown";
+        break;
+    }
+
+    switch (dl->increment_on) {
+      case DL_SCHED_INCREMENT_FAILURE:
+        increment_on_str = "DL_SCHED_INCREMENT_FAILURE";
+        break;
+      case DL_SCHED_INCREMENT_ATTEMPT:
+        increment_on_str = "DL_SCHED_INCREMENT_ATTEMPT";
+        break;
+      default:
+        increment_on_str = "unknown";
+        break;
+    }
+
+    backoff_str = "DL_SCHED_RANDOM_EXPONENTIAL";
+
+    /* Now assemble them */
+    tor_asprintf(&rv,
+                 "next-attempt-at %s\n"
+                 "n-download-failures %u\n"
+                 "n-download-attempts %u\n"
+                 "schedule %s\n"
+                 "want-authority %s\n"
+                 "increment-on %s\n"
+                 "backoff %s\n"
+                 "last-backoff-position %u\n"
+                 "last-delay-used %d\n",
+                 tbuf,
+                 dl->n_download_failures,
+                 dl->n_download_attempts,
+                 schedule_str,
+                 want_authority_str,
+                 increment_on_str,
+                 backoff_str,
+                 dl->last_backoff_position,
+                 dl->last_delay_used);
+  }
+
+  return rv;
+}
+
+/** Handle the consensus download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_networkstatus(const char *flavor,
+                                       download_status_t **dl_to_emit,
+                                       const char **errmsg)
+{
+  /*
+   * We get the one for the current bootstrapped status by default, or
+   * take an extra /bootstrap or /running suffix
+   */
+  if (strcmp(flavor, "ns") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_NS);
+  } else if (strcmp(flavor, "ns/bootstrap") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_NS);
+  } else if (strcmp(flavor, "ns/running") == 0 ) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor_running(FLAV_NS);
+  } else if (strcmp(flavor, "microdesc") == 0) {
+    *dl_to_emit = networkstatus_get_dl_status_by_flavor(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/bootstrap") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_bootstrap(FLAV_MICRODESC);
+  } else if (strcmp(flavor, "microdesc/running") == 0) {
+    *dl_to_emit =
+      networkstatus_get_dl_status_by_flavor_running(FLAV_MICRODESC);
+  } else {
+    *errmsg = "Unknown flavor";
+  }
+}
+
+/** Handle the cert download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_cert(const char *fp_sk_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  const char *sk_req;
+  char id_digest[DIGEST_LEN];
+  char sk_digest[DIGEST_LEN];
+
+  /*
+   * We have to handle four cases; fp_sk_req is the request with
+   * a prefix of "downloads/cert/" snipped off.
+   *
+   * Case 1: fp_sk_req = "fps"
+   *  - We should emit a digest_list with a list of all the identity
+   *    fingerprints that can be queried for certificate download status;
+   *    get it by calling list_authority_ids_with_downloads().
+   *
+   * Case 2: fp_sk_req = "fp/<fp>" for some fingerprint fp
+   *  - We want the default certificate for this identity fingerprint's
+   *    download status; this is the download we get from URLs starting
+   *    in /fp/ on the directory server.  We can get it with
+   *    id_only_download_status_for_authority_id().
+   *
+   * Case 3: fp_sk_req = "fp/<fp>/sks" for some fingerprint fp
+   *  - We want a list of all signing key digests for this identity
+   *    fingerprint which can be queried for certificate download status.
+   *    Get it with list_sk_digests_for_authority_id().
+   *
+   * Case 4: fp_sk_req = "fp/<fp>/<sk>" for some fingerprint fp and
+   *         signing key digest sk
+   *   - We want the download status for the certificate for this specific
+   *     signing key and fingerprint.  These correspond to the ones we get
+   *     from URLs starting in /fp-sk/ on the directory server.  Get it with
+   *     list_sk_digests_for_authority_id().
+   */
+
+  if (strcmp(fp_sk_req, "fps") == 0) {
+    *digest_list = list_authority_ids_with_downloads();
+    if (!(*digest_list)) {
+      *errmsg = "Failed to get list of authority identity digests (!)";
+    }
+  } else if (!strcmpstart(fp_sk_req, "fp/")) {
+    fp_sk_req += strlen("fp/");
+    /* Okay, look for another / to tell the fp from fp-sk cases */
+    sk_req = strchr(fp_sk_req, '/');
+    if (sk_req) {
+      /* okay, split it here and try to parse <fp> */
+      if (base16_decode(id_digest, DIGEST_LEN,
+                        fp_sk_req, sk_req - fp_sk_req) == DIGEST_LEN) {
+        /* Skip past the '/' */
+        ++sk_req;
+        if (strcmp(sk_req, "sks") == 0) {
+          /* We're asking for the list of signing key fingerprints */
+          *digest_list = list_sk_digests_for_authority_id(id_digest);
+          if (!(*digest_list)) {
+            *errmsg = "Failed to get list of signing key digests for this "
+                      "authority identity digest";
+          }
+        } else {
+          /* We've got a signing key digest */
+          if (base16_decode(sk_digest, DIGEST_LEN,
+                            sk_req, strlen(sk_req)) == DIGEST_LEN) {
+            *dl_to_emit =
+              download_status_for_authority_id_and_sk(id_digest, sk_digest);
+            if (!(*dl_to_emit)) {
+              *errmsg = "Failed to get download status for this identity/"
+                        "signing key digest pair";
+            }
+          } else {
+            *errmsg = "That didn't look like a signing key digest";
+          }
+        }
+      } else {
+        *errmsg = "That didn't look like an identity digest";
+      }
+    } else {
+      /* We're either in downloads/certs/fp/<fp>, or we can't parse <fp> */
+      if (strlen(fp_sk_req) == HEX_DIGEST_LEN) {
+        if (base16_decode(id_digest, DIGEST_LEN,
+                          fp_sk_req, strlen(fp_sk_req)) == DIGEST_LEN) {
+          *dl_to_emit = id_only_download_status_for_authority_id(id_digest);
+          if (!(*dl_to_emit)) {
+            *errmsg = "Failed to get download status for this authority "
+                      "identity digest";
+          }
+        } else {
+          *errmsg = "That didn't look like a digest";
+        }
+      } else {
+        *errmsg = "That didn't look like a digest";
+      }
+    }
+  } else {
+    *errmsg = "Unknown certificate download status query";
+  }
+}
+
+/** Handle the routerdesc download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_desc(const char *desc_req,
+                              download_status_t **dl_to_emit,
+                              smartlist_t **digest_list,
+                              const char **errmsg)
+{
+  char desc_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: desc_req = "descs"
+   *   - Emit a list of all router descriptor digests, which we get by
+   *     calling router_get_descriptor_digests(); this can return NULL
+   *     if we have no current ns-flavor consensus.
+   *
+   * Case 2: desc_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using router_get_dl_status_by_descriptor_digest().
+   */
+
+  if (strcmp(desc_req, "descs") == 0) {
+    *digest_list = router_get_descriptor_digests();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to have a networkstatus-flavored consensus";
+    }
+    /*
+     * Microdescs don't use the download_status_t mechanism, so we don't
+     * answer queries about their downloads here; see microdesc.c.
+     */
+  } else if (strlen(desc_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(desc_digest, DIGEST_LEN,
+                      desc_req, strlen(desc_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = router_get_dl_status_by_descriptor_digest(desc_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such descriptor digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown router descriptor download status query";
+  }
+}
+
+/** Handle the bridge download cases for getinfo_helper_downloads() */
+STATIC void
+getinfo_helper_downloads_bridge(const char *bridge_req,
+                                download_status_t **dl_to_emit,
+                                smartlist_t **digest_list,
+                                const char **errmsg)
+{
+  char bridge_digest[DIGEST_LEN];
+  /*
+   * Two cases to handle here:
+   *
+   * Case 1: bridge_req = "bridges"
+   *   - Emit a list of all bridge identity digests, which we get by
+   *     calling list_bridge_identities(); this can return NULL if we are
+   *     not using bridges.
+   *
+   * Case 2: bridge_req = <fp>
+   *   - Check on the specified fingerprint and emit its download_status_t
+   *     using get_bridge_dl_status_by_id().
+   */
+
+  if (strcmp(bridge_req, "bridges") == 0) {
+    *digest_list = list_bridge_identities();
+    if (!(*digest_list)) {
+      *errmsg = "We don't seem to be using bridges";
+    }
+  } else if (strlen(bridge_req) == HEX_DIGEST_LEN) {
+    if (base16_decode(bridge_digest, DIGEST_LEN,
+                      bridge_req, strlen(bridge_req)) == DIGEST_LEN) {
+      /* Okay we got a digest-shaped thing; try asking for it */
+      *dl_to_emit = get_bridge_dl_status_by_id(bridge_digest);
+      if (!(*dl_to_emit)) {
+        *errmsg = "No such bridge identity digest found";
+      }
+    } else {
+      *errmsg = "That didn't look like a digest";
+    }
+  } else {
+    *errmsg = "Unknown bridge descriptor download status query";
+  }
+}
+
+/** Implementation helper for GETINFO: knows the answers for questions about
+ * download status information. */
+STATIC int
+getinfo_helper_downloads(control_connection_t *control_conn,
+                   const char *question, char **answer,
+                   const char **errmsg)
+{
+  download_status_t *dl_to_emit = NULL;
+  smartlist_t *digest_list = NULL;
+
+  /* Assert args are sane */
+  tor_assert(control_conn != NULL);
+  tor_assert(question != NULL);
+  tor_assert(answer != NULL);
+  tor_assert(errmsg != NULL);
+
+  /* We check for this later to see if we should supply a default */
+  *errmsg = NULL;
+
+  /* Are we after networkstatus downloads? */
+  if (!strcmpstart(question, "downloads/networkstatus/")) {
+    getinfo_helper_downloads_networkstatus(
+        question + strlen("downloads/networkstatus/"),
+        &dl_to_emit, errmsg);
+  /* Certificates? */
+  } else if (!strcmpstart(question, "downloads/cert/")) {
+    getinfo_helper_downloads_cert(
+        question + strlen("downloads/cert/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Router descriptors? */
+  } else if (!strcmpstart(question, "downloads/desc/")) {
+    getinfo_helper_downloads_desc(
+        question + strlen("downloads/desc/"),
+        &dl_to_emit, &digest_list, errmsg);
+  /* Bridge descriptors? */
+  } else if (!strcmpstart(question, "downloads/bridge/")) {
+    getinfo_helper_downloads_bridge(
+        question + strlen("downloads/bridge/"),
+        &dl_to_emit, &digest_list, errmsg);
+  } else {
+    *errmsg = "Unknown download status query";
+  }
+
+  if (dl_to_emit) {
+    *answer = download_status_to_string(dl_to_emit);
+
+    return 0;
+  } else if (digest_list) {
+    *answer = digest_list_to_string(digest_list);
+    SMARTLIST_FOREACH(digest_list, void *, s, tor_free(s));
+    smartlist_free(digest_list);
+
+    return 0;
+  } else {
+    if (!(*errmsg)) {
+      *errmsg = "Unknown error";
+    }
+
+    return -1;
+  }
+}
+
+/** Implementation helper for GETINFO: knows how to generate summaries of the
+ * current states of things we send events about. */
+static int
+getinfo_helper_events(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  const or_options_t *options = get_options();
+  (void) control_conn;
+  if (!strcmp(question, "circuit-status")) {
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(circuit_get_global_list(), circuit_t *, circ_) {
+      origin_circuit_t *circ;
+      char *circdesc;
+      const char *state;
+      if (! CIRCUIT_IS_ORIGIN(circ_) || circ_->marked_for_close)
+        continue;
+      circ = TO_ORIGIN_CIRCUIT(circ_);
+
+      if (circ->base_.state == CIRCUIT_STATE_OPEN)
+        state = "BUILT";
+      else if (circ->base_.state == CIRCUIT_STATE_GUARD_WAIT)
+        state = "GUARD_WAIT";
+      else if (circ->cpath)
+        state = "EXTENDED";
+      else
+        state = "LAUNCHED";
+
+      circdesc = circuit_describe_status_for_controller(circ);
+
+      smartlist_add_asprintf(status, "%lu %s%s%s",
+                   (unsigned long)circ->global_identifier,
+                   state, *circdesc ? " " : "", circdesc);
+      tor_free(circdesc);
+    }
+    SMARTLIST_FOREACH_END(circ_);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "stream-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    char buf[256];
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      entry_connection_t *conn;
+      circuit_t *circ;
+      origin_circuit_t *origin_circ = NULL;
+      if (base_conn->type != CONN_TYPE_AP ||
+          base_conn->marked_for_close ||
+          base_conn->state == AP_CONN_STATE_SOCKS_WAIT ||
+          base_conn->state == AP_CONN_STATE_NATD_WAIT)
+        continue;
+      conn = TO_ENTRY_CONN(base_conn);
+      switch (base_conn->state)
+        {
+        case AP_CONN_STATE_CONTROLLER_WAIT:
+        case AP_CONN_STATE_CIRCUIT_WAIT:
+          if (conn->socks_request &&
+              SOCKS_COMMAND_IS_RESOLVE(conn->socks_request->command))
+            state = "NEWRESOLVE";
+          else
+            state = "NEW";
+          break;
+        case AP_CONN_STATE_RENDDESC_WAIT:
+        case AP_CONN_STATE_CONNECT_WAIT:
+          state = "SENTCONNECT"; break;
+        case AP_CONN_STATE_RESOLVE_WAIT:
+          state = "SENTRESOLVE"; break;
+        case AP_CONN_STATE_OPEN:
+          state = "SUCCEEDED"; break;
+        default:
+          log_warn(LD_BUG, "Asked for stream in unknown state %d",
+                   base_conn->state);
+          continue;
+        }
+      circ = circuit_get_by_edge_conn(ENTRY_TO_EDGE_CONN(conn));
+      if (circ && CIRCUIT_IS_ORIGIN(circ))
+        origin_circ = TO_ORIGIN_CIRCUIT(circ);
+      write_stream_target_to_buf(conn, buf, sizeof(buf));
+      smartlist_add_asprintf(status, "%lu %s %lu %s",
+                   (unsigned long) base_conn->global_identifier,state,
+                   origin_circ?
+                         (unsigned long)origin_circ->global_identifier : 0ul,
+                   buf);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmp(question, "orconn-status")) {
+    smartlist_t *conns = get_connection_array();
+    smartlist_t *status = smartlist_new();
+    SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+      const char *state;
+      char name[128];
+      or_connection_t *conn;
+      if (base_conn->type != CONN_TYPE_OR || base_conn->marked_for_close)
+        continue;
+      conn = TO_OR_CONN(base_conn);
+      if (conn->base_.state == OR_CONN_STATE_OPEN)
+        state = "CONNECTED";
+      else if (conn->nickname)
+        state = "LAUNCHED";
+      else
+        state = "NEW";
+      orconn_target_get_name(name, sizeof(name), conn);
+      smartlist_add_asprintf(status, "%s %s", name, state);
+    } SMARTLIST_FOREACH_END(base_conn);
+    *answer = smartlist_join_strings(status, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(status, char *, cp, tor_free(cp));
+    smartlist_free(status);
+  } else if (!strcmpstart(question, "address-mappings/")) {
+    time_t min_e, max_e;
+    smartlist_t *mappings;
+    question += strlen("address-mappings/");
+    if (!strcmp(question, "all")) {
+      min_e = 0; max_e = TIME_MAX;
+    } else if (!strcmp(question, "cache")) {
+      min_e = 2; max_e = TIME_MAX;
+    } else if (!strcmp(question, "config")) {
+      min_e = 0; max_e = 0;
+    } else if (!strcmp(question, "control")) {
+      min_e = 1; max_e = 1;
+    } else {
+      return 0;
+    }
+    mappings = smartlist_new();
+    addressmap_get_mappings(mappings, min_e, max_e, 1);
+    *answer = smartlist_join_strings(mappings, "\r\n", 0, NULL);
+    SMARTLIST_FOREACH(mappings, char *, cp, tor_free(cp));
+    smartlist_free(mappings);
+  } else if (!strcmpstart(question, "status/")) {
+    /* Note that status/ is not a catch-all for events; there's only supposed
+     * to be a status GETINFO if there's a corresponding STATUS event. */
+    if (!strcmp(question, "status/circuit-established")) {
+      *answer = tor_strdup(have_completed_a_circuit() ? "1" : "0");
+    } else if (!strcmp(question, "status/enough-dir-info")) {
+      *answer = tor_strdup(router_have_minimum_dir_info() ? "1" : "0");
+    } else if (!strcmp(question, "status/good-server-descriptor") ||
+               !strcmp(question, "status/accepted-server-descriptor")) {
+      /* They're equivalent for now, until we can figure out how to make
+       * good-server-descriptor be what we want. See comment in
+       * control-spec.txt. */
+      *answer = tor_strdup(directories_have_accepted_server_descriptor()
+                           ? "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/or")) {
+      *answer = tor_strdup(check_whether_orport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded/dir")) {
+      *answer = tor_strdup(check_whether_dirport_reachable(options) ?
+                            "1" : "0");
+    } else if (!strcmp(question, "status/reachability-succeeded")) {
+      tor_asprintf(answer, "OR=%d DIR=%d",
+                   check_whether_orport_reachable(options) ? 1 : 0,
+                   check_whether_dirport_reachable(options) ? 1 : 0);
+    } else if (!strcmp(question, "status/bootstrap-phase")) {
+      *answer = control_event_boot_last_msg();
+    } else if (!strcmpstart(question, "status/version/")) {
+      int is_server = server_mode(options);
+      networkstatus_t *c = networkstatus_get_latest_consensus();
+      version_status_t status;
+      const char *recommended;
+      if (c) {
+        recommended = is_server ? c->server_versions : c->client_versions;
+        status = tor_version_is_obsolete(VERSION, recommended);
+      } else {
+        recommended = "?";
+        status = VS_UNKNOWN;
+      }
+
+      if (!strcmp(question, "status/version/recommended")) {
+        *answer = tor_strdup(recommended);
+        return 0;
+      }
+      if (!strcmp(question, "status/version/current")) {
+        switch (status)
+          {
+          case VS_RECOMMENDED: *answer = tor_strdup("recommended"); break;
+          case VS_OLD: *answer = tor_strdup("obsolete"); break;
+          case VS_NEW: *answer = tor_strdup("new"); break;
+          case VS_NEW_IN_SERIES: *answer = tor_strdup("new in series"); break;
+          case VS_UNRECOMMENDED: *answer = tor_strdup("unrecommended"); break;
+          case VS_EMPTY: *answer = tor_strdup("none recommended"); break;
+          case VS_UNKNOWN: *answer = tor_strdup("unknown"); break;
+          default: tor_fragile_assert();
+          }
+      }
+    } else if (!strcmp(question, "status/clients-seen")) {
+      char *bridge_stats = geoip_get_bridge_stats_controller(time(NULL));
+      if (!bridge_stats) {
+        *errmsg = "No bridge-client stats available";
+        return -1;
+      }
+      *answer = bridge_stats;
+    } else if (!strcmp(question, "status/fresh-relay-descs")) {
+      if (!server_mode(options)) {
+        *errmsg = "Only relays have descriptors";
+        return -1;
+      }
+      routerinfo_t *r;
+      extrainfo_t *e;
+      int result;
+      if ((result = router_build_fresh_descriptor(&r, &e)) < 0) {
+        switch (result) {
+          case TOR_ROUTERINFO_ERROR_NO_EXT_ADDR:
+            *errmsg = "Cannot get relay address while generating descriptor";
+            break;
+          case TOR_ROUTERINFO_ERROR_DIGEST_FAILED:
+            *errmsg = "Key digest failed";
+            break;
+          case TOR_ROUTERINFO_ERROR_CANNOT_GENERATE:
+            *errmsg = "Cannot generate router descriptor";
+            break;
+          default:
+            *errmsg = "Error generating descriptor";
+            break;
+        }
+        return -1;
+      }
+      size_t size = r->cache_info.signed_descriptor_len + 1;
+      if (e) {
+        size += e->cache_info.signed_descriptor_len + 1;
+      }
+      tor_assert(r->cache_info.signed_descriptor_len);
+      char *descs = tor_malloc(size);
+      char *cp = descs;
+      memcpy(cp, signed_descriptor_get_body(&r->cache_info),
+             r->cache_info.signed_descriptor_len);
+      cp += r->cache_info.signed_descriptor_len - 1;
+      if (e) {
+        if (cp[0] == '\0') {
+          cp[0] = '\n';
+        } else if (cp[0] != '\n') {
+          cp[1] = '\n';
+          cp++;
+        }
+        memcpy(cp, signed_descriptor_get_body(&e->cache_info),
+               e->cache_info.signed_descriptor_len);
+        cp += e->cache_info.signed_descriptor_len - 1;
+      }
+      if (cp[0] == '\n') {
+        cp[0] = '\0';
+      } else if (cp[0] != '\0') {
+        cp[1] = '\0';
+      }
+      *answer = descs;
+      routerinfo_free(r);
+      extrainfo_free(e);
+    } else {
+      return 0;
+    }
+  }
+  return 0;
+}
+
+/** Implementation helper for GETINFO: knows how to enumerate hidden services
+ * created via the control port. */
+STATIC int
+getinfo_helper_onions(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  smartlist_t *onion_list = NULL;
+  (void) errmsg;  /* no errors from this method */
+
+  if (control_conn && !strcmp(question, "onions/current")) {
+    onion_list = control_conn->ephemeral_onion_services;
+  } else if (!strcmp(question, "onions/detached")) {
+    onion_list = get_detached_onion_services();
+  } else {
+    return 0;
+  }
+  if (!onion_list || smartlist_len(onion_list) == 0) {
+    if (answer) {
+      *answer = tor_strdup("");
+    }
+  } else {
+    if (answer) {
+      *answer = smartlist_join_strings(onion_list, "\r\n", 0, NULL);
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about network
+ * liveness. */
+static int
+getinfo_helper_liveness(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **errmsg)
+{
+  (void)control_conn;
+  (void)errmsg;
+  if (strcmp(question, "network-liveness") == 0) {
+    if (get_cached_network_liveness()) {
+      *answer = tor_strdup("up");
+    } else {
+      *answer = tor_strdup("down");
+    }
+  }
+
+  return 0;
+}
+
+/** Implementation helper for GETINFO: answers queries about shared random
+ * value. */
+static int
+getinfo_helper_sr(control_connection_t *control_conn,
+                  const char *question, char **answer,
+                  const char **errmsg)
+{
+  (void) control_conn;
+  (void) errmsg;
+
+  if (!strcmp(question, "sr/current")) {
+    *answer = sr_get_current_for_control();
+  } else if (!strcmp(question, "sr/previous")) {
+    *answer = sr_get_previous_for_control();
+  }
+  /* Else statement here is unrecognized key so do nothing. */
+
+  return 0;
+}
+
+/** Callback function for GETINFO: on a given control connection, try to
+ * answer the question <b>q</b> and store the newly-allocated answer in
+ * *<b>a</b>. If an internal error occurs, return -1 and optionally set
+ * *<b>error_out</b> to point to an error message to be delivered to the
+ * controller. On success, _or if the key is not recognized_, return 0. Do not
+ * set <b>a</b> if the key is not recognized but you may set <b>error_out</b>
+ * to improve the error message.
+ */
+typedef int (*getinfo_helper_t)(control_connection_t *,
+                                const char *q, char **a,
+                                const char **error_out);
+
+/** A single item for the GETINFO question-to-answer-function table. */
+typedef struct getinfo_item_t {
+  const char *varname; /**< The value (or prefix) of the question. */
+  getinfo_helper_t fn; /**< The function that knows the answer: NULL if
+                        * this entry is documentation-only. */
+  const char *desc; /**< Description of the variable. */
+  int is_prefix; /** Must varname match exactly, or must it be a prefix? */
+} getinfo_item_t;
+
+#define ITEM(name, fn, desc) { name, getinfo_helper_##fn, desc, 0 }
+#define PREFIX(name, fn, desc) { name, getinfo_helper_##fn, desc, 1 }
+#define DOC(name, desc) { name, NULL, desc, 0 }
+
+/** Table mapping questions accepted by GETINFO to the functions that know how
+ * to answer them. */
+static const getinfo_item_t getinfo_items[] = {
+  ITEM("version", misc, "The current version of Tor."),
+  ITEM("bw-event-cache", misc, "Cached BW events for a short interval."),
+  ITEM("config-file", misc, "Current location of the \"torrc\" file."),
+  ITEM("config-defaults-file", misc, "Current location of the defaults file."),
+  ITEM("config-text", misc,
+       "Return the string that would be written by a saveconf command."),
+  ITEM("config-can-saveconf", misc,
+       "Is it possible to save the configuration to the \"torrc\" file?"),
+  ITEM("accounting/bytes", accounting,
+       "Number of bytes read/written so far in the accounting interval."),
+  ITEM("accounting/bytes-left", accounting,
+      "Number of bytes left to write/read so far in the accounting interval."),
+  ITEM("accounting/enabled", accounting, "Is accounting currently enabled?"),
+  ITEM("accounting/hibernating", accounting, "Are we hibernating or awake?"),
+  ITEM("accounting/interval-start", accounting,
+       "Time when the accounting period starts."),
+  ITEM("accounting/interval-end", accounting,
+       "Time when the accounting period ends."),
+  ITEM("accounting/interval-wake", accounting,
+       "Time to wake up in this accounting period."),
+  ITEM("helper-nodes", entry_guards, NULL), /* deprecated */
+  ITEM("entry-guards", entry_guards,
+       "Which nodes are we using as entry guards?"),
+  ITEM("fingerprint", misc, NULL),
+  PREFIX("config/", config, "Current configuration values."),
+  DOC("config/names",
+      "List of configuration options, types, and documentation."),
+  DOC("config/defaults",
+      "List of default values for configuration options. "
+      "See also config/names"),
+  PREFIX("current-time/", current_time, "Current time."),
+  DOC("current-time/local", "Current time on the local system."),
+  DOC("current-time/utc", "Current UTC time."),
+  PREFIX("downloads/networkstatus/", downloads,
+         "Download statuses for networkstatus objects"),
+  DOC("downloads/networkstatus/ns",
+      "Download status for current-mode networkstatus download"),
+  DOC("downloads/networkstatus/ns/bootstrap",
+      "Download status for bootstrap-time networkstatus download"),
+  DOC("downloads/networkstatus/ns/running",
+      "Download status for run-time networkstatus download"),
+  DOC("downloads/networkstatus/microdesc",
+      "Download status for current-mode microdesc download"),
+  DOC("downloads/networkstatus/microdesc/bootstrap",
+      "Download status for bootstrap-time microdesc download"),
+  DOC("downloads/networkstatus/microdesc/running",
+      "Download status for run-time microdesc download"),
+  PREFIX("downloads/cert/", downloads,
+         "Download statuses for certificates, by id fingerprint and "
+         "signing key"),
+  DOC("downloads/cert/fps",
+      "List of authority fingerprints for which any download statuses "
+      "exist"),
+  DOC("downloads/cert/fp/<fp>",
+      "Download status for <fp> with the default signing key; corresponds "
+      "to /fp/ URLs on directory server."),
+  DOC("downloads/cert/fp/<fp>/sks",
+      "List of signing keys for which specific download statuses are "
+      "available for this id fingerprint"),
+  DOC("downloads/cert/fp/<fp>/<sk>",
+      "Download status for <fp> with signing key <sk>; corresponds "
+      "to /fp-sk/ URLs on directory server."),
+  PREFIX("downloads/desc/", downloads,
+         "Download statuses for router descriptors, by descriptor digest"),
+  DOC("downloads/desc/descs",
+      "Return a list of known router descriptor digests"),
+  DOC("downloads/desc/<desc>",
+      "Return a download status for a given descriptor digest"),
+  PREFIX("downloads/bridge/", downloads,
+         "Download statuses for bridge descriptors, by bridge identity "
+         "digest"),
+  DOC("downloads/bridge/bridges",
+      "Return a list of configured bridge identity digests with download "
+      "statuses"),
+  DOC("downloads/bridge/<desc>",
+      "Return a download status for a given bridge identity digest"),
+  ITEM("info/names", misc,
+       "List of GETINFO options, types, and documentation."),
+  ITEM("events/names", misc,
+       "Events that the controller can ask for with SETEVENTS."),
+  ITEM("signal/names", misc, "Signal names recognized by the SIGNAL command"),
+  ITEM("features/names", misc, "What arguments can USEFEATURE take?"),
+  PREFIX("desc/id/", dir, "Router descriptors by ID."),
+  PREFIX("desc/name/", dir, "Router descriptors by nickname."),
+  ITEM("desc/all-recent", dir,
+       "All non-expired, non-superseded router descriptors."),
+  ITEM("desc/download-enabled", dir,
+       "Do we try to download router descriptors?"),
+  ITEM("desc/all-recent-extrainfo-hack", dir, NULL), /* Hack. */
+  ITEM("md/all", dir, "All known microdescriptors."),
+  PREFIX("md/id/", dir, "Microdescriptors by ID"),
+  PREFIX("md/name/", dir, "Microdescriptors by name"),
+  ITEM("md/download-enabled", dir,
+       "Do we try to download microdescriptors?"),
+  PREFIX("extra-info/digest/", dir, "Extra-info documents by digest."),
+  PREFIX("hs/client/desc/id", dir,
+         "Hidden Service descriptor in client's cache by onion."),
+  PREFIX("hs/service/desc/id/", dir,
+         "Hidden Service descriptor in services's cache by onion."),
+  PREFIX("net/listeners/", listeners, "Bound addresses by type"),
+  ITEM("ns/all", networkstatus,
+       "Brief summary of router status (v2 directory format)"),
+  PREFIX("ns/id/", networkstatus,
+         "Brief summary of router status by ID (v2 directory format)."),
+  PREFIX("ns/name/", networkstatus,
+         "Brief summary of router status by nickname (v2 directory format)."),
+  PREFIX("ns/purpose/", networkstatus,
+         "Brief summary of router status by purpose (v2 directory format)."),
+  PREFIX("consensus/", networkstatus,
+         "Information about and from the ns consensus."),
+  ITEM("network-status", dir,
+       "Brief summary of router status (v1 directory format)"),
+  ITEM("network-liveness", liveness,
+       "Current opinion on whether the network is live"),
+  ITEM("circuit-status", events, "List of current circuits originating here."),
+  ITEM("stream-status", events,"List of current streams."),
+  ITEM("orconn-status", events, "A list of current OR connections."),
+  ITEM("dormant", misc,
+       "Is Tor dormant (not building circuits because it's idle)?"),
+  PREFIX("address-mappings/", events, NULL),
+  DOC("address-mappings/all", "Current address mappings."),
+  DOC("address-mappings/cache", "Current cached DNS replies."),
+  DOC("address-mappings/config",
+      "Current address mappings from configuration."),
+  DOC("address-mappings/control", "Current address mappings from controller."),
+  PREFIX("status/", events, NULL),
+  DOC("status/circuit-established",
+      "Whether we think client functionality is working."),
+  DOC("status/enough-dir-info",
+      "Whether we have enough up-to-date directory information to build "
+      "circuits."),
+  DOC("status/bootstrap-phase",
+      "The last bootstrap phase status event that Tor sent."),
+  DOC("status/clients-seen",
+      "Breakdown of client countries seen by a bridge."),
+  DOC("status/fresh-relay-descs",
+      "A fresh relay/ei descriptor pair for Tor's current state. Not stored."),
+  DOC("status/version/recommended", "List of currently recommended versions."),
+  DOC("status/version/current", "Status of the current version."),
+  ITEM("address", misc, "IP address of this Tor host, if we can guess it."),
+  ITEM("traffic/read", misc,"Bytes read since the process was started."),
+  ITEM("traffic/written", misc,
+       "Bytes written since the process was started."),
+  ITEM("uptime", misc, "Uptime of the Tor daemon in seconds."),
+  ITEM("process/pid", misc, "Process id belonging to the main tor process."),
+  ITEM("process/uid", misc, "User id running the tor process."),
+  ITEM("process/user", misc,
+       "Username under which the tor process is running."),
+  ITEM("process/descriptor-limit", misc, "File descriptor limit."),
+  ITEM("limits/max-mem-in-queues", misc, "Actual limit on memory in queues"),
+  PREFIX("desc-annotations/id/", dir, "Router annotations by hexdigest."),
+  PREFIX("dir/server/", dir,"Router descriptors as retrieved from a DirPort."),
+  PREFIX("dir/status/", dir,
+         "v2 networkstatus docs as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus", dir,
+       "v3 Networkstatus consensus as retrieved from a DirPort."),
+  ITEM("dir/status-vote/current/consensus-microdesc", dir,
+       "v3 Microdescriptor consensus as retrieved from a DirPort."),
+  ITEM("exit-policy/default", policies,
+       "The default value appended to the configured exit policy."),
+  ITEM("exit-policy/reject-private/default", policies,
+       "The default rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate."),
+  ITEM("exit-policy/reject-private/relay", policies,
+       "The relay-specific rules appended to the configured exit policy by"
+       " ExitPolicyRejectPrivate and/or ExitPolicyRejectLocalInterfaces."),
+  ITEM("exit-policy/full", policies, "The entire exit policy of onion router"),
+  ITEM("exit-policy/ipv4", policies, "IPv4 parts of exit policy"),
+  ITEM("exit-policy/ipv6", policies, "IPv6 parts of exit policy"),
+  PREFIX("ip-to-country/", geoip, "Perform a GEOIP lookup"),
+  ITEM("onions/current", onions,
+       "Onion services owned by the current control connection."),
+  ITEM("onions/detached", onions,
+       "Onion services detached from the control connection."),
+  ITEM("sr/current", sr, "Get current shared random value."),
+  ITEM("sr/previous", sr, "Get previous shared random value."),
+  { NULL, NULL, NULL, 0 }
+};
+
+/** Allocate and return a list of recognized GETINFO options. */
+static char *
+list_getinfo_options(void)
+{
+  int i;
+  smartlist_t *lines = smartlist_new();
+  char *ans;
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    if (!getinfo_items[i].desc)
+      continue;
+
+    smartlist_add_asprintf(lines, "%s%s -- %s\n",
+                 getinfo_items[i].varname,
+                 getinfo_items[i].is_prefix ? "*" : "",
+                 getinfo_items[i].desc);
+  }
+  smartlist_sort_strings(lines);
+
+  ans = smartlist_join_strings(lines, "", 0, NULL);
+  SMARTLIST_FOREACH(lines, char *, cp, tor_free(cp));
+  smartlist_free(lines);
+
+  return ans;
+}
+
+/** Lookup the 'getinfo' entry <b>question</b>, and return
+ * the answer in <b>*answer</b> (or NULL if key not recognized).
+ * Return 0 if success or unrecognized, or -1 if recognized but
+ * internal error. */
+static int
+handle_getinfo_helper(control_connection_t *control_conn,
+                      const char *question, char **answer,
+                      const char **err_out)
+{
+  int i;
+  *answer = NULL; /* unrecognized key by default */
+
+  for (i = 0; getinfo_items[i].varname; ++i) {
+    int match;
+    if (getinfo_items[i].is_prefix)
+      match = !strcmpstart(question, getinfo_items[i].varname);
+    else
+      match = !strcmp(question, getinfo_items[i].varname);
+    if (match) {
+      tor_assert(getinfo_items[i].fn);
+      return getinfo_items[i].fn(control_conn, question, answer, err_out);
+    }
+  }
+
+  return 0; /* unrecognized */
+}
+
+const control_cmd_syntax_t getinfo_syntax = {
+  .max_args = UINT_MAX,
+};
+
+/** Called when we receive a GETINFO command.  Try to fetch all requested
+ * information, and reply with information or error message. */
+int
+handle_control_getinfo(control_connection_t *conn,
+                       const control_cmd_args_t *args)
+{
+  const smartlist_t *questions = args->args;
+  smartlist_t *answers = smartlist_new();
+  smartlist_t *unrecognized = smartlist_new();
+  char *ans = NULL;
+
+  SMARTLIST_FOREACH_BEGIN(questions, const char *, q) {
+    const char *errmsg = NULL;
+
+    if (handle_getinfo_helper(conn, q, &ans, &errmsg) < 0) {
+      if (!errmsg)
+        errmsg = "Internal error";
+      control_write_endreply(conn, 551, errmsg);
+      goto done;
+    }
+    if (!ans) {
+      if (errmsg) {
+        /* use provided error message */
+        control_reply_add_str(unrecognized, 552, errmsg);
+      } else {
+        /* use default error message */
+        control_reply_add_printf(unrecognized, 552,
+                                 "Unrecognized key \"%s\"", q);
+      }
+    } else {
+      control_reply_add_one_kv(answers, 250, KV_RAW, q, ans);
+      tor_free(ans);
+    }
+  } SMARTLIST_FOREACH_END(q);
+
+  control_reply_add_done(answers);
+
+  if (smartlist_len(unrecognized)) {
+    control_write_reply_lines(conn, unrecognized);
+    /* If there were any unrecognized queries, don't write real answers */
+    goto done;
+  }
+
+  control_write_reply_lines(conn, answers);
+
+ done:
+  control_reply_free(answers);
+  control_reply_free(unrecognized);
+
+  return 0;
+}
diff --git a/src/feature/control/control_getinfo.h b/src/feature/control/control_getinfo.h
new file mode 100644
index 0000000000..0ada49258e
--- /dev/null
+++ b/src/feature/control/control_getinfo.h
@@ -0,0 +1,65 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control.h
+ * \brief Header file for control.c.
+ **/
+
+#ifndef TOR_CONTROL_GETINFO_H
+#define TOR_CONTROL_GETINFO_H
+
+struct control_cmd_syntax_t;
+struct control_cmd_args_t;
+extern const struct control_cmd_syntax_t getinfo_syntax;
+
+int handle_control_getinfo(control_connection_t *conn,
+                           const struct control_cmd_args_t *args);
+
+#ifdef CONTROL_GETINFO_PRIVATE
+STATIC int getinfo_helper_onions(
+    control_connection_t *control_conn,
+    const char *question,
+    char **answer,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_networkstatus(
+    const char *flavor,
+    download_status_t **dl_to_emit,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_cert(
+    const char *fp_sk_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_desc(
+    const char *desc_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC void getinfo_helper_downloads_bridge(
+    const char *bridge_req,
+    download_status_t **dl_to_emit,
+    smartlist_t **digest_list,
+    const char **errmsg);
+STATIC int getinfo_helper_downloads(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_consensus(
+    consensus_flavor_t flavor,
+    char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_dir(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+STATIC int getinfo_helper_current_time(
+    control_connection_t *control_conn,
+    const char *question, char **answer,
+    const char **errmsg);
+#endif /* defined(CONTROL_GETINFO_PRIVATE) */
+
+#endif /* !defined(TOR_CONTROL_GETINFO_H) */
diff --git a/src/feature/control/control_hs.c b/src/feature/control/control_hs.c
new file mode 100644
index 0000000000..54b767cd0d
--- /dev/null
+++ b/src/feature/control/control_hs.c
@@ -0,0 +1,354 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_hs.c
+ *
+ * \brief Implement commands for Tor's control-socket interface that are
+ *        related to onion services.
+ **/
+
+#include "core/or/or.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_hs.h"
+#include "feature/control/control_proto.h"
+#include "feature/hs/hs_client.h"
+#include "lib/encoding/confline.h"
+
+#include "feature/control/control_cmd_args_st.h"
+
+/** Parse the 'KeyType ":" PrivateKey' from <b>client_privkey_str</b> and store
+ *  it into <b>privkey</b>. Use <b>conn</b> to output any errors if needed.
+ *
+ *  Return 0 if all went well, -1 otherwise. */
+static int
+parse_private_key_from_control_port(const char *client_privkey_str,
+                                    curve25519_secret_key_t *privkey,
+                                    control_connection_t *conn)
+{
+  int retval = -1;
+  smartlist_t *key_args = smartlist_new();
+
+  tor_assert(privkey);
+
+  smartlist_split_string(key_args, client_privkey_str, ":",
+                         SPLIT_IGNORE_BLANK, 0);
+  if (smartlist_len(key_args) != 2) {
+    control_printf_endreply(conn, 512, "Invalid key type/blob");
+    goto err;
+  }
+
+  const char *key_type = smartlist_get(key_args, 0);
+  const char *key_blob = smartlist_get(key_args, 1);
+
+  if (strcasecmp(key_type, "x25519")) {
+    control_printf_endreply(conn, 552,
+                            "Unrecognized key type \"%s\"", key_type);
+    goto err;
+  }
+
+  if (base64_decode((char*)privkey->secret_key, sizeof(privkey->secret_key),
+                    key_blob,
+                    strlen(key_blob)) != sizeof(privkey->secret_key)) {
+    control_printf_endreply(conn, 512, "Failed to decode x25519 private key");
+    goto err;
+  }
+
+  if (fast_mem_is_zero((const char*)privkey->secret_key,
+                       sizeof(privkey->secret_key))) {
+    control_printf_endreply(conn, 553,
+                            "Invalid private key \"%s\"", key_blob);
+    goto err;
+  }
+
+  retval = 0;
+
+ err:
+  SMARTLIST_FOREACH(key_args, char *, c, tor_free(c));
+  smartlist_free(key_args);
+  return retval;
+}
+
+/** Syntax details for ONION_CLIENT_AUTH_ADD */
+const control_cmd_syntax_t onion_client_auth_add_syntax = {
+  .max_args = 2,
+  .accept_keywords = true,
+};
+
+/** Called when we get an ONION_CLIENT_AUTH_ADD command; parse the body, and
+ *  register the new client-side client auth credentials:
+ *  "ONION_CLIENT_AUTH_ADD" SP HSAddress
+ *                          SP KeyType ":" PrivateKeyBlob
+ *                          [SP "Type=" TYPE] CRLF
+ */
+int
+handle_control_onion_client_auth_add(control_connection_t *conn,
+                                     const control_cmd_args_t *args)
+{
+  int retval = -1;
+  smartlist_t *flags = smartlist_new();
+  hs_client_service_authorization_t *creds = NULL;
+
+  tor_assert(args);
+
+  int argc = smartlist_len(args->args);
+  /* We need at least 'HSAddress' and 'PrivateKeyBlob' */
+  if (argc < 2) {
+    control_printf_endreply(conn, 512,
+                            "Incomplete ONION_CLIENT_AUTH_ADD command");
+    goto err;
+  }
+
+  creds = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+
+  const char *hsaddress = smartlist_get(args->args, 0);
+  if (!hs_address_is_valid(hsaddress)) {
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",hsaddress);
+    goto err;
+  }
+  strlcpy(creds->onion_address, hsaddress, sizeof(creds->onion_address));
+
+  /* Parse the client private key */
+  const char *client_privkey = smartlist_get(args->args, 1);
+  if (parse_private_key_from_control_port(client_privkey,
+                                          &creds->enc_seckey, conn) < 0) {
+    goto err;
+  }
+
+  /* Now let's parse the remaining arguments (variable size) */
+  for (const config_line_t *line = args->kwargs; line; line = line->next) {
+    if (!strcasecmpstart(line->key, "Flags")) {
+      smartlist_split_string(flags, line->value, ",", SPLIT_IGNORE_BLANK, 0);
+      if (smartlist_len(flags) < 1) {
+        control_write_endreply(conn, 512, "Invalid 'Flags' argument");
+        goto err;
+      }
+      SMARTLIST_FOREACH_BEGIN(flags, const char *, flag) {
+        if (!strcasecmp(flag, "Permanent")) {
+          creds->flags |= CLIENT_AUTH_FLAG_IS_PERMANENT;
+        } else {
+          control_printf_endreply(conn, 512, "Invalid 'Flags' argument: %s",
+                                  escaped(flag));
+          goto err;
+        }
+      } SMARTLIST_FOREACH_END(flag);
+    }
+    if (!strcasecmp(line->key, "ClientName")) {
+      if (strlen(line->value) > REND_CLIENTNAME_MAX_LEN) {
+        control_printf_endreply(conn, 512, "ClientName longer than %d chars",
+                                REND_CLIENTNAME_MAX_LEN);
+      }
+      creds->client_name = tor_strdup(line->value);
+    }
+  }
+
+  hs_client_register_auth_status_t register_status;
+  /* Register the credential (register func takes ownership of cred.) */
+  register_status = hs_client_register_auth_credentials(creds);
+  switch (register_status) {
+  case REGISTER_FAIL_BAD_ADDRESS:
+    /* It's a bug because the service addr has already been validated above */
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"", hsaddress);
+    break;
+  case REGISTER_FAIL_PERMANENT_STORAGE:
+    control_printf_endreply(conn, 553, "Unable to store creds for \"%s\"",
+                            hsaddress);
+    break;
+  case REGISTER_SUCCESS_ALREADY_EXISTS:
+    control_printf_endreply(conn, 251,"Client for onion existed and replaced");
+    break;
+  case REGISTER_SUCCESS_AND_DECRYPTED:
+    control_printf_endreply(conn, 252,"Registered client and decrypted desc");
+    break;
+  case REGISTER_SUCCESS:
+    control_printf_endreply(conn, 250, "OK");
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+
+  retval = 0;
+  goto done;
+
+ err:
+  client_service_authorization_free(creds);
+
+ done:
+  SMARTLIST_FOREACH(flags, char *, s, tor_free(s));
+  smartlist_free(flags);
+  return retval;
+}
+
+/** Syntax details for ONION_CLIENT_AUTH_REMOVE */
+const control_cmd_syntax_t onion_client_auth_remove_syntax = {
+  .max_args = 1,
+  .accept_keywords = true,
+};
+
+/** Called when we get an ONION_CLIENT_AUTH_REMOVE command; parse the body, and
+ *  register the new client-side client auth credentials.
+ *    "ONION_CLIENT_AUTH_REMOVE" SP HSAddress
+ */
+int
+handle_control_onion_client_auth_remove(control_connection_t *conn,
+                                        const control_cmd_args_t *args)
+{
+  int retval = -1;
+
+  tor_assert(args);
+
+  int argc = smartlist_len(args->args);
+  if (argc < 1) {
+    control_printf_endreply(conn, 512,
+                            "Incomplete ONION_CLIENT_AUTH_REMOVE command");
+    goto err;
+  }
+
+  const char *hsaddress = smartlist_get(args->args, 0);
+  if (!hs_address_is_valid(hsaddress)) {
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",hsaddress);
+    goto err;
+  }
+
+  hs_client_removal_auth_status_t removal_status;
+  removal_status = hs_client_remove_auth_credentials(hsaddress);
+  switch (removal_status) {
+  case REMOVAL_BAD_ADDRESS:
+    /* It's a bug because the service addr has already been validated above */
+    control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",hsaddress);
+    break;
+  case REMOVAL_SUCCESS_NOT_FOUND:
+    control_printf_endreply(conn, 251, "No credentials for \"%s\"",hsaddress);
+    break;
+  case REMOVAL_SUCCESS:
+    control_printf_endreply(conn, 250, "OK");
+    break;
+  default:
+    tor_assert_nonfatal_unreached();
+  }
+
+  retval = 0;
+
+ err:
+  return retval;
+}
+
+/** Helper: Return a newly allocated string with the encoding of client
+ *  authorization credentials */
+static char *
+encode_client_auth_cred_for_control_port(
+                                       hs_client_service_authorization_t *cred)
+{
+  smartlist_t *control_line = smartlist_new();
+  char x25519_b64[128];
+  char *msg_str = NULL;
+
+  tor_assert(cred);
+
+  if (base64_encode(x25519_b64, sizeof(x25519_b64),
+                    (char *)cred->enc_seckey.secret_key,
+                    sizeof(cred->enc_seckey.secret_key), 0) < 0) {
+    tor_assert_nonfatal_unreached();
+    goto err;
+  }
+
+  smartlist_add_asprintf(control_line, "CLIENT %s x25519:%s",
+                         cred->onion_address, x25519_b64);
+
+  if (cred->flags) { /* flags are also optional */
+    if (cred->flags & CLIENT_AUTH_FLAG_IS_PERMANENT) {
+      smartlist_add_asprintf(control_line, " Flags=Permanent");
+    }
+  }
+
+  if (cred->client_name) {
+    smartlist_add_asprintf(control_line, " ClientName=%s", cred->client_name);
+  }
+
+  /* Join all the components into a single string */
+  msg_str = smartlist_join_strings(control_line, "", 0, NULL);
+
+ err:
+  SMARTLIST_FOREACH(control_line, char *, cp, tor_free(cp));
+  smartlist_free(control_line);
+
+  return msg_str;
+}
+
+/** Syntax details for ONION_CLIENT_AUTH_VIEW */
+const control_cmd_syntax_t onion_client_auth_view_syntax = {
+  .max_args = 1,
+  .accept_keywords = true,
+};
+
+/** Called when we get an ONION_CLIENT_AUTH_VIEW command; parse the body, and
+ *  register the new client-side client auth credentials.
+ *        "ONION_CLIENT_AUTH_VIEW" [SP HSAddress] CRLF
+ */
+int
+handle_control_onion_client_auth_view(control_connection_t *conn,
+                                      const control_cmd_args_t *args)
+{
+  int retval = -1;
+  const char *hsaddress = NULL;
+  /* We are gonna put all the credential strings into a smartlist, and sort it
+     before printing, so that we can get a guaranteed order of printing. */
+  smartlist_t *creds_str_list = smartlist_new();
+
+  tor_assert(args);
+
+  int argc = smartlist_len(args->args);
+  if (argc >= 1) {
+    hsaddress = smartlist_get(args->args, 0);
+    if (!hs_address_is_valid(hsaddress)) {
+      control_printf_endreply(conn, 512, "Invalid v3 address \"%s\"",
+                              hsaddress);
+      goto err;
+    }
+  }
+
+  if (hsaddress) {
+    control_printf_midreply(conn, 250, "ONION_CLIENT_AUTH_VIEW %s", hsaddress);
+  } else {
+    control_printf_midreply(conn, 250, "ONION_CLIENT_AUTH_VIEW");
+  }
+
+  /* Create an iterator out of the digest256map */
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  digest256map_iter_t *itr = digest256map_iter_init(client_auths);
+  while (!digest256map_iter_done(itr)) {
+    const uint8_t *service_pubkey;
+    void *valp;
+    digest256map_iter_get(itr, &service_pubkey, &valp);
+    tor_assert(valp);
+    hs_client_service_authorization_t *cred = valp;
+
+    /* If a specific HS address was requested, only print creds for that one */
+    if (hsaddress && strcmp(cred->onion_address, hsaddress)) {
+      itr = digest256map_iter_next(client_auths, itr);
+      continue;
+    }
+
+    char *encoding_str = encode_client_auth_cred_for_control_port(cred);
+    tor_assert_nonfatal(encoding_str);
+    smartlist_add(creds_str_list, encoding_str);
+
+    itr = digest256map_iter_next(client_auths, itr);
+  }
+
+  /* We got everything: Now sort the strings and print them */
+  smartlist_sort_strings(creds_str_list);
+  SMARTLIST_FOREACH_BEGIN(creds_str_list, char *, c) {
+    control_printf_midreply(conn, 250, "%s", c);
+  } SMARTLIST_FOREACH_END(c);
+
+  send_control_done(conn);
+
+  retval = 0;
+
+ err:
+  SMARTLIST_FOREACH(creds_str_list, char *, cp, tor_free(cp));
+  smartlist_free(creds_str_list);
+  return retval;
+}
diff --git a/src/feature/control/control_hs.h b/src/feature/control/control_hs.h
new file mode 100644
index 0000000000..8a0cd6818d
--- /dev/null
+++ b/src/feature/control/control_hs.h
@@ -0,0 +1,34 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_hs.c
+ *
+ * \brief Header file for control_hs.c.
+ **/
+
+#ifndef TOR_CONTROL_HS_H
+#define TOR_CONTROL_HS_H
+
+struct control_connection_t;
+struct control_cmd_syntax_t;
+struct control_cmd_args_t;
+
+extern const struct control_cmd_syntax_t onion_client_auth_add_syntax;
+extern const struct control_cmd_syntax_t onion_client_auth_remove_syntax;
+extern const struct control_cmd_syntax_t onion_client_auth_view_syntax;
+
+int
+handle_control_onion_client_auth_add(struct control_connection_t *conn,
+                                     const struct control_cmd_args_t *args);
+
+int
+handle_control_onion_client_auth_remove(struct control_connection_t *conn,
+                                        const struct control_cmd_args_t *args);
+
+int
+handle_control_onion_client_auth_view(struct control_connection_t *conn,
+                                      const struct control_cmd_args_t *args);
+
+#endif /* !defined(TOR_CONTROL_HS_H) */
diff --git a/src/feature/control/control_proto.c b/src/feature/control/control_proto.c
new file mode 100644
index 0000000000..98715ad9d5
--- /dev/null
+++ b/src/feature/control/control_proto.c
@@ -0,0 +1,434 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.c
+ * \brief Formatting functions for controller data.
+ */
+
+#include "core/or/or.h"
+
+#include "core/mainloop/connection.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "feature/control/control_proto.h"
+#include "feature/nodelist/nodelist.h"
+
+#include "core/or/cpath_build_state_st.h"
+#include "core/or/entry_connection_st.h"
+#include "core/or/or_connection_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
+#include "feature/control/control_connection_st.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/kvline.h"
+
+/** Append a NUL-terminated string <b>s</b> to the end of
+ * <b>conn</b>-\>outbuf.
+ */
+void
+connection_write_str_to_buf(const char *s, control_connection_t *conn)
+{
+  size_t len = strlen(s);
+  connection_buf_add(s, len, TO_CONN(conn));
+}
+
+/** Acts like sprintf, but writes its formatted string to the end of
+ * <b>conn</b>-\>outbuf. */
+void
+connection_printf_to_buf(control_connection_t *conn, const char *format, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+  int len;
+
+  va_start(ap,format);
+  len = tor_vasprintf(&buf, format, ap);
+  va_end(ap);
+
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+
+  connection_buf_add(buf, (size_t)len, TO_CONN(conn));
+
+  tor_free(buf);
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy the
+ * contents of <b>data</b> into *<b>out</b>, adding a period before any period
+ * that appears at the start of a line, and adding a period-CRLF line at
+ * the end. Replace all LF characters sequences with CRLF.  Return the number
+ * of bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+write_escaped_data(const char *data, size_t len, char **out)
+{
+  tor_assert(len < SIZE_MAX - 9);
+  size_t sz_out = len+8+1;
+  char *outp;
+  const char *start = data, *end;
+  size_t i;
+  int start_of_line;
+  for (i=0; i < len; ++i) {
+    if (data[i] == '\n') {
+      sz_out += 2; /* Maybe add a CR; maybe add a dot. */
+      if (sz_out >= SIZE_T_CEILING) {
+        log_warn(LD_BUG, "Input to write_escaped_data was too long");
+        *out = tor_strdup(".\r\n");
+        return 3;
+      }
+    }
+  }
+  *out = outp = tor_malloc(sz_out);
+  end = data+len;
+  start_of_line = 1;
+  while (data < end) {
+    if (*data == '\n') {
+      if (data > start && data[-1] != '\r')
+        *outp++ = '\r';
+      start_of_line = 1;
+    } else if (*data == '.') {
+      if (start_of_line) {
+        start_of_line = 0;
+        *outp++ = '.';
+      }
+    } else {
+      start_of_line = 0;
+    }
+    *outp++ = *data++;
+  }
+  if (outp < *out+2 || fast_memcmp(outp-2, "\r\n", 2)) {
+    *outp++ = '\r';
+    *outp++ = '\n';
+  }
+  *outp++ = '.';
+  *outp++ = '\r';
+  *outp++ = '\n';
+  *outp = '\0'; /* NUL-terminate just in case. */
+  tor_assert(outp >= *out);
+  tor_assert((size_t)(outp - *out) <= sz_out);
+  return outp - *out;
+}
+
+/** Given a <b>len</b>-character string in <b>data</b>, made of lines
+ * terminated by CRLF, allocate a new string in *<b>out</b>, and copy
+ * the contents of <b>data</b> into *<b>out</b>, removing any period
+ * that appears at the start of a line, and replacing all CRLF sequences
+ * with LF.   Return the number of
+ * bytes in *<b>out</b>.
+ *
+ * This corresponds to CmdData in control-spec.txt.
+ */
+size_t
+read_escaped_data(const char *data, size_t len, char **out)
+{
+  char *outp;
+  const char *next;
+  const char *end;
+
+  *out = outp = tor_malloc(len+1);
+
+  end = data+len;
+
+  while (data < end) {
+    /* we're at the start of a line. */
+    if (*data == '.')
+      ++data;
+    next = memchr(data, '\n', end-data);
+    if (next) {
+      size_t n_to_copy = next-data;
+      /* Don't copy a CR that precedes this LF. */
+      if (n_to_copy && *(next-1) == '\r')
+        --n_to_copy;
+      memcpy(outp, data, n_to_copy);
+      outp += n_to_copy;
+      data = next+1; /* This will point at the start of the next line,
+                      * or the end of the string, or a period. */
+    } else {
+      memcpy(outp, data, end-data);
+      outp += (end-data);
+      *outp = '\0';
+      return outp - *out;
+    }
+    *outp++ = '\n';
+  }
+
+  *outp = '\0';
+  return outp - *out;
+}
+
+/** Send a "DONE" message down the control connection <b>conn</b>. */
+void
+send_control_done(control_connection_t *conn)
+{
+  control_write_endreply(conn, 250, "OK");
+}
+
+/** Write a reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param s string reply content
+ */
+MOCK_IMPL(void,
+control_write_reply, (control_connection_t *conn, int code, int c,
+                      const char *s))
+{
+  connection_printf_to_buf(conn, "%03d%c%s\r\n", code, c, s);
+}
+
+/** Write a formatted reply to the control channel.
+ *
+ * @param conn control connection
+ * @param code numeric result code
+ * @param c separator character, usually ' ', '-', or '+'
+ * @param fmt format string
+ * @param ap va_list from caller
+ */
+void
+control_vprintf_reply(control_connection_t *conn, int code, int c,
+                      const char *fmt, va_list ap)
+{
+  char *buf = NULL;
+  int len;
+
+  len = tor_vasprintf(&buf, fmt, ap);
+  if (len < 0) {
+    log_err(LD_BUG, "Unable to format string for controller.");
+    tor_assert(0);
+  }
+  control_write_reply(conn, code, c, buf);
+  tor_free(buf);
+}
+
+/** Write an EndReplyLine */
+void
+control_write_endreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, ' ', s);
+}
+
+/** Write a formatted EndReplyLine */
+void
+control_printf_endreply(control_connection_t *conn, int code,
+                        const char *fmt, ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, ' ', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a MidReplyLine */
+void
+control_write_midreply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '-', s);
+}
+
+/** Write a formatted MidReplyLine */
+void
+control_printf_midreply(control_connection_t *conn, int code, const char *fmt,
+                        ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '-', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a DataReplyLine */
+void
+control_write_datareply(control_connection_t *conn, int code, const char *s)
+{
+  control_write_reply(conn, code, '+', s);
+}
+
+/** Write a formatted DataReplyLine */
+void
+control_printf_datareply(control_connection_t *conn, int code, const char *fmt,
+                         ...)
+{
+  va_list ap;
+
+  va_start(ap, fmt);
+  control_vprintf_reply(conn, code, '+', fmt, ap);
+  va_end(ap);
+}
+
+/** Write a CmdData */
+void
+control_write_data(control_connection_t *conn, const char *data)
+{
+  char *esc = NULL;
+  size_t esc_len;
+
+  esc_len = write_escaped_data(data, strlen(data), &esc);
+  connection_buf_add(esc, esc_len, TO_CONN(conn));
+  tor_free(esc);
+}
+
+/** Write a single reply line to @a conn.
+ *
+ * @param conn control connection
+ * @param line control reply line to write
+ * @param lastone true if this is the last reply line of a multi-line reply
+ */
+void
+control_write_reply_line(control_connection_t *conn,
+                         const control_reply_line_t *line, bool lastone)
+{
+  const config_line_t *kvline = line->kvline;
+  char *s = NULL;
+
+  if (strpbrk(kvline->value, "\r\n") != NULL) {
+    /* If a key-value pair needs to be encoded as CmdData, it can be
+       the only key-value pair in that reply line */
+    tor_assert(kvline->next == NULL);
+    control_printf_datareply(conn, line->code, "%s=", kvline->key);
+    control_write_data(conn, kvline->value);
+    return;
+  }
+  s = kvline_encode(kvline, line->flags);
+  if (lastone) {
+    control_write_endreply(conn, line->code, s);
+  } else {
+    control_write_midreply(conn, line->code, s);
+  }
+  tor_free(s);
+}
+
+/** Write a set of reply lines to @a conn.
+ *
+ * @param conn control connection
+ * @param lines smartlist of pointers to control_reply_line_t to write
+ */
+void
+control_write_reply_lines(control_connection_t *conn, smartlist_t *lines)
+{
+  bool lastone = false;
+
+  SMARTLIST_FOREACH_BEGIN(lines, control_reply_line_t *, line) {
+    if (line_sl_idx >= line_sl_len - 1)
+      lastone = true;
+    control_write_reply_line(conn, line, lastone);
+  } SMARTLIST_FOREACH_END(line);
+}
+
+/** Add a single key-value pair as a new reply line to a control reply
+ * line list.
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param code numeric control reply code
+ * @param flags kvline encoding flags
+ * @param key key
+ * @param val value
+ */
+void
+control_reply_add_one_kv(smartlist_t *reply, int code, int flags,
+                         const char *key, const char *val)
+{
+  control_reply_line_t *line = tor_malloc_zero(sizeof(*line));
+
+  line->code = code;
+  line->flags = flags;
+  config_line_append(&line->kvline, key, val);
+  smartlist_add(reply, line);
+}
+
+/** Append a single key-value pair to last reply line in a control
+ * reply line list.
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param key key
+ * @param val value
+ */
+void
+control_reply_append_kv(smartlist_t *reply, const char *key, const char *val)
+{
+  int len = smartlist_len(reply);
+  control_reply_line_t *line;
+
+  tor_assert(len > 0);
+
+  line = smartlist_get(reply, len - 1);
+  config_line_append(&line->kvline, key, val);
+}
+
+/** Add new reply line consisting of the string @a s
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param code numeric control reply code
+ * @param s string containing the rest of the reply line
+ */
+void
+control_reply_add_str(smartlist_t *reply, int code, const char *s)
+{
+  control_reply_add_one_kv(reply, code, KV_OMIT_KEYS|KV_RAW, "", s);
+}
+
+/** Format a new reply line
+ *
+ * @param reply smartlist of pointers to control_reply_line_t
+ * @param code numeric control reply code
+ * @param fmt format string
+ */
+void
+control_reply_add_printf(smartlist_t *reply, int code, const char *fmt, ...)
+{
+  va_list ap;
+  char *buf = NULL;
+
+  va_start(ap, fmt);
+  (void)tor_vasprintf(&buf, fmt, ap);
+  va_end(ap);
+  control_reply_add_str(reply, code, buf);
+  tor_free(buf);
+}
+
+/** Add a "250 OK" line to a set of control reply lines */
+void
+control_reply_add_done(smartlist_t *reply)
+{
+  control_reply_add_str(reply, 250, "OK");
+}
+
+/** Free a control_reply_line_t.  Don't call this directly; use the
+ * control_reply_line_free() macro instead. */
+void
+control_reply_line_free_(control_reply_line_t *line)
+{
+  if (!line)
+    return;
+  config_free_lines(line->kvline);
+  tor_free_(line);
+}
+
+/** Clear a smartlist of control_reply_line_t.  Doesn't free the
+ * smartlist, but does free each individual line. */
+void
+control_reply_clear(smartlist_t *reply)
+{
+  SMARTLIST_FOREACH(reply, control_reply_line_t *, line,
+                    control_reply_line_free(line));
+  smartlist_clear(reply);
+}
+
+/** Free a smartlist of control_reply_line_t. Don't call this
+ * directly; use the control_reply_free() macro instead. */
+void
+control_reply_free_(smartlist_t *reply)
+{
+  control_reply_clear(reply);
+  smartlist_free_(reply);
+}
diff --git a/src/feature/control/control_proto.h b/src/feature/control/control_proto.h
new file mode 100644
index 0000000000..4c32b820d1
--- /dev/null
+++ b/src/feature/control/control_proto.h
@@ -0,0 +1,120 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file control_proto.h
+ * \brief Header file for control_proto.c.
+ *
+ * See @ref replylines for details about the key-value abstraction for
+ * generating reply lines.
+ **/
+
+#ifndef TOR_CONTROL_PROTO_H
+#define TOR_CONTROL_PROTO_H
+
+#include "lib/encoding/confline.h"
+
+/**
+ * @defgroup replylines Control reply lines
+ * @brief Key-value structures for control reply lines
+ *
+ * Control reply lines are config_line_t key-value structures with
+ * some additional information to help formatting, such as the numeric
+ * result code specified in the control protocol and flags affecting
+ * the way kvline_encode() formats the @a kvline.
+ *
+ * Generally, modules implementing control commands will work with
+ * smartlists of these structures, using functions like
+ * control_reply_add_str() for adding a reply line consisting of a
+ * single string, or control_reply_add_one_kv() and
+ * control_reply_append_kv() for composing a line containing one or
+ * more key-value pairs.
+ *
+ * @{
+ */
+/** @brief A reply line for the control protocol.
+ *
+ * This wraps config_line_t with some additional information that's
+ * useful when generating control reply lines.
+ */
+typedef struct control_reply_line_t {
+  int code;                     /**< numeric code */
+  int flags;                    /**< kvline encoding flags */
+  config_line_t *kvline;        /**< kvline */
+} control_reply_line_t;
+
+void control_reply_line_free_(control_reply_line_t *line);
+/**
+ * @brief Free and null a control_reply_line_t
+ *
+ * @param line pointer to control_reply_line_t to free
+ */
+#define control_reply_line_free(line)                   \
+  FREE_AND_NULL(control_reply_line_t,                   \
+                control_reply_line_free_, (line))
+/** @} */
+
+void connection_write_str_to_buf(const char *s, control_connection_t *conn);
+void connection_printf_to_buf(control_connection_t *conn,
+                                     const char *format, ...)
+  CHECK_PRINTF(2,3);
+
+size_t write_escaped_data(const char *data, size_t len, char **out);
+size_t read_escaped_data(const char *data, size_t len, char **out);
+void send_control_done(control_connection_t *conn);
+
+MOCK_DECL(void, control_write_reply, (control_connection_t *conn, int code,
+                                      int c, const char *s));
+void control_vprintf_reply(control_connection_t *conn, int code, int c,
+                           const char *fmt, va_list ap)
+  CHECK_PRINTF(4, 0);
+void control_write_endreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_endreply(control_connection_t *conn, int code,
+                             const char *fmt, ...)
+  CHECK_PRINTF(3, 4);
+void control_write_midreply(control_connection_t *conn, int code,
+                            const char *s);
+void control_printf_midreply(control_connection_t *conn, int code,
+                             const char *fmt,
+                             ...)
+  CHECK_PRINTF(3, 4);
+void control_write_datareply(control_connection_t *conn, int code,
+                             const char *s);
+void control_printf_datareply(control_connection_t *conn, int code,
+                              const char *fmt,
+                              ...)
+  CHECK_PRINTF(3, 4);
+void control_write_data(control_connection_t *conn, const char *data);
+
+/** @addtogroup replylines
+ * @{
+ */
+void control_write_reply_line(control_connection_t *conn,
+                              const control_reply_line_t *line, bool lastone);
+void control_write_reply_lines(control_connection_t *conn, smartlist_t *lines);
+
+void control_reply_add_one_kv(smartlist_t *reply, int code, int flags,
+                              const char *key, const char *val);
+void control_reply_append_kv(smartlist_t *reply, const char *key,
+                             const char *val);
+void control_reply_add_str(smartlist_t *reply, int code, const char *s);
+void control_reply_add_printf(smartlist_t *reply, int code,
+                              const char *fmt, ...)
+  CHECK_PRINTF(3, 4);
+void control_reply_add_done(smartlist_t *reply);
+
+void control_reply_clear(smartlist_t *reply);
+void control_reply_free_(smartlist_t *reply);
+
+/** @brief Free and null a smartlist of control_reply_line_t.
+ *
+ * @param r pointer to smartlist_t of control_reply_line_t to free */
+#define control_reply_free(r) \
+  FREE_AND_NULL(smartlist_t, control_reply_free_, (r))
+/** @} */
+
+#endif /* !defined(TOR_CONTROL_PROTO_H) */
diff --git a/src/feature/control/feature_control.md b/src/feature/control/feature_control.md
new file mode 100644
index 0000000000..9f1681ea91
--- /dev/null
+++ b/src/feature/control/feature_control.md
@@ -0,0 +1,8 @@
+@dir /feature/control
+@brief feature/control: Controller API.
+
+The Controller API is a text-based protocol that another program (or another
+thread, if you're running Tor in-process) can use to configure and control
+Tor while it is running.  The current protocol is documented in
+[control-spec.txt](https://gitweb.torproject.org/torspec.git/tree/control-spec.txt).
+
diff --git a/src/feature/control/fmt_serverstatus.c b/src/feature/control/fmt_serverstatus.c
index a1ddd2119a..ed9ad95ce2 100644
--- a/src/feature/control/fmt_serverstatus.c
+++ b/src/feature/control/fmt_serverstatus.c
@@ -1,16 +1,21 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file fmt_serverstatus.c
+ * @brief Format relay info for a controller.
+ **/
+
 #include "core/or/or.h"
 #include "feature/control/fmt_serverstatus.h"
 
 #include "app/config/config.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/voteflags.h"// XXXX remove
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -66,11 +71,9 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   smartlist_t *rs_entries;
   time_t now = time(NULL);
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
-  const or_options_t *options = get_options();
   /* We include v2 dir auths here too, because they need to answer
    * controllers. Eventually we'll deprecate this whole function;
    * see also networkstatus_getinfo_by_purpose(). */
-  int authdir = authdir_mode_publishes_statuses(options);
   tor_assert(router_status_out);
 
   rs_entries = smartlist_new();
@@ -78,10 +81,6 @@ list_server_status_v1(smartlist_t *routers, char **router_status_out,
   SMARTLIST_FOREACH_BEGIN(routers, routerinfo_t *, ri) {
     const node_t *node = node_get_by_id(ri->cache_info.identity_digest);
     tor_assert(node);
-    if (authdir) {
-      /* Update router status in routerinfo_t. */
-      dirserv_set_router_is_running(ri, now);
-    }
     if (for_controller) {
       char name_buf[MAX_VERBOSE_NICKNAME_LEN+2];
       char *cp = name_buf;
diff --git a/src/feature/control/fmt_serverstatus.h b/src/feature/control/fmt_serverstatus.h
index 4b95e5b59f..9dd9fe125c 100644
--- a/src/feature/control/fmt_serverstatus.h
+++ b/src/feature/control/fmt_serverstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,4 +15,4 @@
 int list_server_status_v1(smartlist_t *routers, char **router_status_out,
                           int for_controller);
 
-#endif
+#endif /* !defined(TOR_FMT_SERVERSTATUS_H) */
diff --git a/src/feature/control/getinfo_geoip.c b/src/feature/control/getinfo_geoip.c
index d188725fa3..33019207e6 100644
--- a/src/feature/control/getinfo_geoip.c
+++ b/src/feature/control/getinfo_geoip.c
@@ -1,3 +1,12 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file getinfo_geoip.c
+ * @brief GEOIP-related contoller GETINFO commands.
+ **/
 
 #include "core/or/or.h"
 #include "core/mainloop/connection.h"
diff --git a/src/feature/control/getinfo_geoip.h b/src/feature/control/getinfo_geoip.h
index fe22137859..5bc4b08414 100644
--- a/src/feature/control/getinfo_geoip.h
+++ b/src/feature/control/getinfo_geoip.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file getinfo_geoip.h
+ * @brief Header for getinfo_geoip.c
+ **/
+
 #ifndef TOR_GETINFO_GEOIP_H
 #define TOR_GETINFO_GEOIP_H
 
@@ -11,4 +16,4 @@ int getinfo_helper_geoip(control_connection_t *control_conn,
                      const char *question, char **answer,
                      const char **errmsg);
 
-#endif
+#endif /* !defined(TOR_GETINFO_GEOIP_H) */
diff --git a/src/feature/control/include.am b/src/feature/control/include.am
new file mode 100644
index 0000000000..07094f23bb
--- /dev/null
+++ b/src/feature/control/include.am
@@ -0,0 +1,39 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/control/btrack.c		\
+	src/feature/control/btrack_circuit.c	\
+	src/feature/control/btrack_orconn.c	\
+	src/feature/control/btrack_orconn_cevent.c	\
+	src/feature/control/btrack_orconn_maps.c	\
+	src/feature/control/control.c		\
+	src/feature/control/control_auth.c	\
+	src/feature/control/control_bootstrap.c	\
+	src/feature/control/control_cmd.c	\
+	src/feature/control/control_hs.c	\
+	src/feature/control/control_events.c	\
+	src/feature/control/control_fmt.c	\
+	src/feature/control/control_getinfo.c	\
+	src/feature/control/control_proto.c	\
+	src/feature/control/fmt_serverstatus.c  \
+	src/feature/control/getinfo_geoip.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/control/btrack_circuit.h		\
+	src/feature/control/btrack_orconn.h		\
+	src/feature/control/btrack_orconn_cevent.h	\
+	src/feature/control/btrack_orconn_maps.h	\
+	src/feature/control/btrack_sys.h		\
+	src/feature/control/control.h			\
+	src/feature/control/control_auth.h		\
+	src/feature/control/control_cmd.h		\
+	src/feature/control/control_hs.h		\
+	src/feature/control/control_cmd_args_st.h	\
+	src/feature/control/control_connection_st.h	\
+	src/feature/control/control_events.h		\
+	src/feature/control/control_fmt.h		\
+	src/feature/control/control_getinfo.h		\
+	src/feature/control/control_proto.h		\
+	src/feature/control/fmt_serverstatus.h		\
+	src/feature/control/getinfo_geoip.h
diff --git a/src/feature/dirauth/.may_include b/src/feature/dirauth/.may_include
new file mode 100644
index 0000000000..a9bb274699
--- /dev/null
+++ b/src/feature/dirauth/.may_include
@@ -0,0 +1,2 @@
+*.h
+feature/dirauth/*.inc
diff --git a/src/feature/dirauth/authmode.c b/src/feature/dirauth/authmode.c
index 29fcc6d1a9..0fde7bc679 100644
--- a/src/feature/dirauth/authmode.c
+++ b/src/feature/dirauth/authmode.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,6 +26,15 @@ authdir_mode(const or_options_t *options)
 {
   return options->AuthoritativeDir != 0;
 }
+
+/* Return true iff we believe ourselves to be a v3 authoritative directory
+ * server. */
+int
+authdir_mode_v3(const or_options_t *options)
+{
+  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
+}
+
 /** Return true iff we are an authoritative directory server that is
  * authoritative about receiving and serving descriptors of type
  * <b>purpose</b> on its dirport.
diff --git a/src/feature/dirauth/authmode.h b/src/feature/dirauth/authmode.h
index 876a1f947b..6e6ba7f8ae 100644
--- a/src/feature/dirauth/authmode.h
+++ b/src/feature/dirauth/authmode.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,22 +14,16 @@
 #ifdef HAVE_MODULE_DIRAUTH
 
 int authdir_mode(const or_options_t *options);
+int authdir_mode_v3(const or_options_t *options);
 int authdir_mode_handles_descs(const or_options_t *options, int purpose);
 int authdir_mode_publishes_statuses(const or_options_t *options);
 int authdir_mode_tests_reachability(const or_options_t *options);
 int authdir_mode_bridge(const or_options_t *options);
 
-/* Return true iff we believe ourselves to be a v3 authoritative directory
- * server. */
-static inline int
-authdir_mode_v3(const or_options_t *options)
-{
-  return authdir_mode(options) && options->V3AuthoritativeDir != 0;
-}
-
+/* Is the dirauth module enabled? */
 #define have_module_dirauth() (1)
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 #define authdir_mode(options) (((void)(options)),0)
 #define authdir_mode_handles_descs(options,purpose) \
@@ -41,6 +35,6 @@ authdir_mode_v3(const or_options_t *options)
 
 #define have_module_dirauth() (0)
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif /* TOR_MODE_H */
+#endif /* !defined(TOR_DIRAUTH_MODE_H) */
diff --git a/src/feature/dirauth/bridgeauth.c b/src/feature/dirauth/bridgeauth.c
new file mode 100644
index 0000000000..b7bf3e4e04
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.c
@@ -0,0 +1,60 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file bridgeauth.c
+ * @brief Bridge authority code
+ **/
+
+#include "core/or/or.h"
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/voteflags.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/router.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+
+/** Write out router status entries for all our bridge descriptors. Here, we
+ * also mark routers as running. */
+void
+bridgeauth_dump_bridge_status_to_file(time_t now)
+{
+  char *status;
+  char *fname = NULL;
+  char *thresholds = NULL;
+  char *published_thresholds_and_status = NULL;
+  char published[ISO_TIME_LEN+1];
+  const routerinfo_t *me = router_get_my_routerinfo();
+  char fingerprint[FINGERPRINT_LEN+1];
+  char *fingerprint_line = NULL;
+
+  dirserv_set_bridges_running(now);
+  status = networkstatus_getinfo_by_purpose("bridge", now);
+
+  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
+                                      fingerprint, 0) >= 0) {
+    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
+  } else {
+    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
+  }
+  format_iso_time(published, now);
+  dirserv_compute_bridge_flag_thresholds();
+  thresholds = dirserv_get_flag_thresholds_line();
+  tor_asprintf(&published_thresholds_and_status,
+               "published %s\nflag-thresholds %s\n%s%s",
+               published, thresholds, fingerprint_line ? fingerprint_line : "",
+               status);
+  fname = get_datadir_fname("networkstatus-bridges");
+  if (write_str_to_file(fname,published_thresholds_and_status,0)<0) {
+    log_warn(LD_DIRSERV, "Unable to write networkstatus-bridges file.");
+  }
+  tor_free(thresholds);
+  tor_free(published_thresholds_and_status);
+  tor_free(fname);
+  tor_free(status);
+  tor_free(fingerprint_line);
+}
diff --git a/src/feature/dirauth/bridgeauth.h b/src/feature/dirauth/bridgeauth.h
new file mode 100644
index 0000000000..382d1cfcb8
--- /dev/null
+++ b/src/feature/dirauth/bridgeauth.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file bridgeauth.h
+ * @brief Header for bridgeauth.c
+ **/
+
+#ifndef TOR_DIRAUTH_BRIDGEAUTH_H
+#define TOR_DIRAUTH_BRIDGEAUTH_H
+
+void bridgeauth_dump_bridge_status_to_file(time_t now);
+
+#endif /* !defined(TOR_DIRAUTH_BRIDGEAUTH_H) */
diff --git a/src/feature/dirauth/bwauth.c b/src/feature/dirauth/bwauth.c
index 12f9399e9f..ff0c78f018 100644
--- a/src/feature/dirauth/bwauth.c
+++ b/src/feature/dirauth/bwauth.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,13 +13,16 @@
 #include "feature/dirauth/bwauth.h"
 
 #include "app/config/config.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/dirparse/ns_parse.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
+#include "lib/crypt_ops/crypto_format.h"
 #include "lib/encoding/keyval.h"
 
 /** Total number of routers with measured bandwidth; this is set by
@@ -55,7 +58,7 @@ dirserv_get_last_n_measured_bws(void)
 }
 
 /** Measured bandwidth cache entry */
-typedef struct mbw_cache_entry_s {
+typedef struct mbw_cache_entry_t {
   long mbw_kb;
   time_t as_of;
 } mbw_cache_entry_t;
@@ -181,7 +184,7 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
   /* Check if we have a measured bandwidth, and check the threshold if not */
   if (!(dirserv_query_measured_bw_cache_kb(ri->cache_info.identity_digest,
                                        &mbw_kb, NULL))) {
-    threshold = get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    threshold = dirauth_get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
     if (routers_with_measured_bw > threshold) {
       /* Return zero for unmeasured bandwidth if we are above threshold */
       bw_kb = 0;
@@ -198,14 +201,38 @@ dirserv_get_credible_bandwidth_kb(const routerinfo_t *ri)
 }
 
 /**
- * Read the measured bandwidth list file, apply it to the list of
- * vote_routerstatus_t and store all the headers in <b>bw_file_headers</b>.
+ * Read the measured bandwidth list <b>from_file</b>:
+ * - store all the headers in <b>bw_file_headers</b>,
+ * - apply bandwidth lines to the list of vote_routerstatus_t in
+ *   <b>routerstatuses</b>,
+ * - cache bandwidth lines for dirserv_get_bandwidth_for_router(),
+ * - expire old entries in the measured bandwidth cache, and
+ * - store the DIGEST_SHA256 of the contents of the file in <b>digest_out</b>.
+ *
  * Returns -1 on error, 0 otherwise.
+ *
+ * If the file can't be read, or is empty:
+ * - <b>bw_file_headers</b> is empty,
+ * - <b>routerstatuses</b> is not modified,
+ * - the measured bandwidth cache is not modified, and
+ * - <b>digest_out</b> is the zero-byte digest.
+ *
+ * Otherwise, if there is an error later in the file:
+ * - <b>bw_file_headers</b> contains all the headers up to the error,
+ * - <b>routerstatuses</b> is updated with all the relay lines up to the error,
+ * - the measured bandwidth cache is updated with all the relay lines up to
+ *   the error,
+ * - if the timestamp is valid and recent, old entries in the  measured
+ *   bandwidth cache are expired, and
+ * - <b>digest_out</b> is the digest up to the first read error (if any).
+ *   The digest is taken over all the readable file contents, even if the
+ *   file is outdated or unparseable.
  */
 int
 dirserv_read_measured_bandwidths(const char *from_file,
                                  smartlist_t *routerstatuses,
-                                 smartlist_t *bw_file_headers)
+                                 smartlist_t *bw_file_headers,
+                                 uint8_t *digest_out)
 {
   FILE *fp = tor_fopen_cloexec(from_file, "r");
   int applied_lines = 0;
@@ -219,8 +246,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   int rv = -1;
   char *line = NULL;
   size_t n = 0;
-
-  /* Initialise line, so that we can't possibly run off the end. */
+  crypto_digest_t *digest = crypto_digest256_new(DIGEST_SHA256);
 
   if (fp == NULL) {
     log_warn(LD_CONFIG, "Can't open bandwidth file at configured location: %s",
@@ -228,16 +254,18 @@ dirserv_read_measured_bandwidths(const char *from_file,
     goto err;
   }
 
-  /* If fgets fails, line is either unmodified, or indeterminate. */
   if (tor_getline(&line,&n,fp) <= 0) {
     log_warn(LD_DIRSERV, "Empty bandwidth file");
     goto err;
   }
+  /* If the line could be gotten, add it to the digest */
+  crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
 
   if (!strlen(line) || line[strlen(line)-1] != '\n') {
     log_warn(LD_DIRSERV, "Long or truncated time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    /* Continue adding lines to the digest. */
+    goto continue_digest;
   }
 
   line[strlen(line)-1] = '\0';
@@ -245,14 +273,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
   if (!ok) {
     log_warn(LD_DIRSERV, "Non-integer time in bandwidth file: %s",
              escaped(line));
-    goto err;
+    goto continue_digest;
   }
 
-  now = time(NULL);
+  now = approx_time();
   if ((now - file_time) > MAX_MEASUREMENT_AGE) {
     log_warn(LD_DIRSERV, "Bandwidth measurement file stale. Age: %u",
              (unsigned)(time(NULL) - file_time));
-    goto err;
+    goto continue_digest;
   }
 
   /* If timestamp was correct and bw_file_headers is not NULL,
@@ -267,6 +295,7 @@ dirserv_read_measured_bandwidths(const char *from_file,
   while (!feof(fp)) {
     measured_bw_line_t parsed_line;
     if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
       if (measured_bw_line_parse(&parsed_line, line,
                                  line_is_after_headers) != -1) {
         /* This condition will be true when the first complete valid bw line
@@ -305,6 +334,14 @@ dirserv_read_measured_bandwidths(const char *from_file,
            "Applied %d measurements.", applied_lines);
   rv = 0;
 
+ continue_digest:
+  /* Continue parsing lines to return the digest of the Bandwidth File. */
+  while (!feof(fp)) {
+    if (tor_getline(&line, &n, fp) >= 0) {
+      crypto_digest_add_bytes(digest, (const char *) line, strlen(line));
+    }
+  }
+
  err:
   if (line) {
     // we need to raw_free this buffer because we got it from tor_getdelim()
@@ -312,6 +349,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
   }
   if (fp)
     fclose(fp);
+  if (digest_out)
+    crypto_digest_get_digest(digest, (char *) digest_out, DIGEST256_LEN);
+  crypto_digest_free(digest);
   return rv;
 }
 
@@ -327,6 +367,9 @@ dirserv_read_measured_bandwidths(const char *from_file,
  * the header block yet. If we encounter an incomplete bw line, return -1 but
  * don't warn since there could be additional header lines coming. If we
  * encounter a proper bw line, return 0 (and we got past the headers).
+ *
+ * If the line contains "vote=0", stop parsing it, and return -1, so that the
+ * line is ignored during voting.
  */
 STATIC int
 measured_bw_line_parse(measured_bw_line_t *out, const char *orig_line,
diff --git a/src/feature/dirauth/bwauth.h b/src/feature/dirauth/bwauth.h
index 4507728458..849c58e2fc 100644
--- a/src/feature/dirauth/bwauth.h
+++ b/src/feature/dirauth/bwauth.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,8 +21,8 @@
 
 int dirserv_read_measured_bandwidths(const char *from_file,
                                      smartlist_t *routerstatuses,
-                                     smartlist_t *bw_file_headers);
-
+                                     smartlist_t *bw_file_headers,
+                                     uint8_t *digest_out);
 int dirserv_query_measured_bw_cache_kb(const char *node_id,
                                        long *bw_out,
                                        time_t *as_of_out);
@@ -55,4 +55,4 @@ STATIC void dirserv_cache_measured_bw(const measured_bw_line_t *parsed_line,
 STATIC void dirserv_expire_measured_bw_cache(time_t now);
 #endif /* defined(BWAUTH_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_BWAUTH_H) */
diff --git a/src/feature/dirauth/dirauth_config.c b/src/feature/dirauth/dirauth_config.c
new file mode 100644
index 0000000000..a0b6de7eca
--- /dev/null
+++ b/src/feature/dirauth/dirauth_config.c
@@ -0,0 +1,471 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_config.c
+ * @brief Code to interpret the user's configuration of Tor's directory
+ *        authority module.
+ **/
+
+#include "orconfig.h"
+#include "feature/dirauth/dirauth_config.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confmgt.h"
+#include "lib/conf/confdecl.h"
+
+/* Required for dirinfo_type_t in or_options_t */
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "app/config/resolve_addr.h"
+
+#include "feature/dirauth/voting_schedule.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/guardfraction.h"
+#include "feature/dirauth/dirauth_options_st.h"
+
+/* Copied from config.c, we will refactor later in 29211. */
+#define REJECT(arg) \
+  STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
+#if defined(__GNUC__) && __GNUC__ <= 3
+#define COMPLAIN(args...) \
+  STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END
+#else
+#define COMPLAIN(args, ...)                                     \
+  STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
+
+#define YES_IF_CHANGED_INT(opt) \
+  if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
+
+/** Return true iff we are configured to reject request under load for non
+ * relay connections. */
+bool
+dirauth_should_reject_requests_under_load(void)
+{
+  return !!dirauth_get_options()->AuthDirRejectRequestsUnderLoad;
+}
+
+/**
+ * Legacy validation/normalization function for the dirauth mode options in
+ * options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_dirauth_mode(const or_options_t *old_options,
+                              or_options_t *options,
+                              char **msg)
+{
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!authdir_mode(options))
+    return 0;
+
+  /* confirm that our address isn't broken, so we can complain now */
+  uint32_t tmp;
+  if (resolve_my_address(LOG_WARN, options, &tmp, NULL, NULL) < 0)
+    REJECT("Failed to resolve/guess local address. See logs for details.");
+
+  if (!options->ContactInfo && !options->TestingTorNetwork)
+    REJECT("Authoritative directory servers must set ContactInfo");
+
+  if (options->UseEntryGuards) {
+    log_info(LD_CONFIG, "Authoritative directory servers can't set "
+             "UseEntryGuards. Disabling.");
+    options->UseEntryGuards = 0;
+  }
+  if (!options->DownloadExtraInfo && authdir_mode_v3(options)) {
+    log_info(LD_CONFIG, "Authoritative directories always try to download "
+             "extra-info documents. Setting DownloadExtraInfo.");
+    options->DownloadExtraInfo = 1;
+  }
+  if (!(options->BridgeAuthoritativeDir ||
+        options->V3AuthoritativeDir))
+    REJECT("AuthoritativeDir is set, but none of "
+           "(Bridge/V3)AuthoritativeDir is set.");
+
+  /* If we have a v3bandwidthsfile and it's broken, complain on startup */
+  if (options->V3BandwidthsFile && !old_options) {
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                     NULL);
+  }
+  /* same for guardfraction file */
+  if (options->GuardfractionFile && !old_options) {
+    dirserv_read_guardfraction_file(options->GuardfractionFile, NULL);
+  }
+
+  if (!options->DirPort_set)
+    REJECT("Running as authoritative directory, but no DirPort set.");
+
+  if (!options->ORPort_set)
+    REJECT("Running as authoritative directory, but no ORPort set.");
+
+  if (options->ClientOnly)
+    REJECT("Running as authoritative directory, but ClientOnly also set.");
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the dirauth schedule options
+ * in options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_dirauth_schedule(const or_options_t *old_options,
+                                  or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!authdir_mode_v3(options))
+    return 0;
+
+  if (options->V3AuthVoteDelay + options->V3AuthDistDelay >=
+      options->V3AuthVotingInterval/2) {
+    REJECT("V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
+           "V3AuthVotingInterval");
+  }
+
+  if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVoteDelay < MIN_VOTE_SECONDS_TESTING) {
+        REJECT("V3AuthVoteDelay is way too low.");
+      } else {
+        COMPLAIN("V3AuthVoteDelay is very low. "
+                 "This may lead to failure to vote for a consensus.");
+      }
+    } else {
+      REJECT("V3AuthVoteDelay is way too low.");
+    }
+  }
+
+  if (options->V3AuthDistDelay < MIN_DIST_SECONDS) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthDistDelay < MIN_DIST_SECONDS_TESTING) {
+        REJECT("V3AuthDistDelay is way too low.");
+      } else {
+        COMPLAIN("V3AuthDistDelay is very low. "
+                 "This may lead to missing votes in a consensus.");
+      }
+    } else {
+      REJECT("V3AuthDistDelay is way too low.");
+    }
+  }
+
+  if (options->V3AuthNIntervalsValid < 2)
+    REJECT("V3AuthNIntervalsValid must be at least 2.");
+
+  if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL) {
+    if (options->TestingTorNetwork) {
+      if (options->V3AuthVotingInterval < MIN_VOTE_INTERVAL_TESTING) {
+        /* Unreachable, covered by earlier checks */
+        REJECT("V3AuthVotingInterval is insanely low."); /* LCOV_EXCL_LINE */
+      } else {
+        COMPLAIN("V3AuthVotingInterval is very low. "
+                 "This may lead to failure to synchronise for a consensus.");
+      }
+    } else {
+      REJECT("V3AuthVotingInterval is insanely low.");
+    }
+  } else if (options->V3AuthVotingInterval > 24*60*60) {
+    REJECT("V3AuthVotingInterval is insanely high.");
+  } else if (((24*60*60) % options->V3AuthVotingInterval) != 0) {
+    COMPLAIN("V3AuthVotingInterval does not divide evenly into 24 hours.");
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the dirauth testing options
+ * in options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_dirauth_testing(const or_options_t *old_options,
+                                 or_options_t *options,
+                                 char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!authdir_mode(options))
+    return 0;
+
+  if (!authdir_mode_v3(options))
+    return 0;
+
+  if (options->TestingV3AuthInitialVotingInterval
+      < MIN_VOTE_INTERVAL_TESTING_INITIAL) {
+    REJECT("TestingV3AuthInitialVotingInterval is insanely low.");
+  } else if (((30*60) % options->TestingV3AuthInitialVotingInterval) != 0) {
+    REJECT("TestingV3AuthInitialVotingInterval does not divide evenly into "
+           "30 minutes.");
+  }
+
+  if (options->TestingV3AuthInitialVoteDelay < MIN_VOTE_SECONDS_TESTING) {
+    REJECT("TestingV3AuthInitialVoteDelay is way too low.");
+  }
+
+  if (options->TestingV3AuthInitialDistDelay < MIN_DIST_SECONDS_TESTING) {
+    REJECT("TestingV3AuthInitialDistDelay is way too low.");
+  }
+
+  if (options->TestingV3AuthInitialVoteDelay +
+      options->TestingV3AuthInitialDistDelay >=
+      options->TestingV3AuthInitialVotingInterval) {
+    REJECT("TestingV3AuthInitialVoteDelay plus TestingV3AuthInitialDistDelay "
+           "must be less than TestingV3AuthInitialVotingInterval");
+  }
+
+  if (options->TestingV3AuthVotingStartOffset >
+      MIN(options->TestingV3AuthInitialVotingInterval,
+          options->V3AuthVotingInterval)) {
+    REJECT("TestingV3AuthVotingStartOffset is higher than the voting "
+           "interval.");
+  } else if (options->TestingV3AuthVotingStartOffset < 0) {
+    REJECT("TestingV3AuthVotingStartOffset must be non-negative.");
+  }
+
+  return 0;
+}
+
+/**
+ * Return true if changing the configuration from <b>old</b> to <b>new</b>
+ * affects the timing of the voting subsystem
+ */
+static int
+options_transition_affects_dirauth_timing(const or_options_t *old_options,
+                                          const or_options_t *new_options)
+{
+  tor_assert(old_options);
+  tor_assert(new_options);
+
+  if (authdir_mode_v3(old_options) != authdir_mode_v3(new_options))
+    return 1;
+  if (! authdir_mode_v3(new_options))
+    return 0;
+
+  YES_IF_CHANGED_INT(V3AuthVotingInterval);
+  YES_IF_CHANGED_INT(V3AuthVoteDelay);
+  YES_IF_CHANGED_INT(V3AuthDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVotingInterval);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialVoteDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthInitialDistDelay);
+  YES_IF_CHANGED_INT(TestingV3AuthVotingStartOffset);
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirauth actions based on it. All of
+ * the things we do should survive being done repeatedly.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_dirauth(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* We may need to reschedule some dirauth stuff if our status changed. */
+  if (old_options) {
+    if (options_transition_affects_dirauth_timing(old_options, options)) {
+      dirauth_sched_recalculate_timing(options, time(NULL));
+      reschedule_dirvote(options);
+    }
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirauth mtbf actions based on it.
+ * All of the things we do should survive being done repeatedly.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * Must be called immediately after a successful or_state_load().
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_dirauth_mtbf(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  const or_options_t *options = get_options();
+  int running_tor = options->command == CMD_RUN_TOR;
+
+  if (!authdir_mode(options))
+    return 0;
+
+  /* Load dirauth state */
+  if (running_tor) {
+    rep_hist_load_mtbf_data(time(NULL));
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirauth statistics actions based
+ * on it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Sets <b>*print_notice_out</b> if we enabled stats, and need to print
+ * a stats log using options_act_relay_stats_msg().
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_dirauth_stats(const or_options_t *old_options,
+                          bool *print_notice_out)
+{
+  if (BUG(!print_notice_out))
+    return -1;
+
+  const or_options_t *options = get_options();
+
+  if (authdir_mode_bridge(options)) {
+    time_t now = time(NULL);
+    int print_notice = 0;
+
+    if (!old_options || !authdir_mode_bridge(old_options)) {
+      rep_hist_desc_stats_init(now);
+      print_notice = 1;
+    }
+    if (print_notice)
+      *print_notice_out = 1;
+  }
+
+  /* If we used to have statistics enabled but we just disabled them,
+     stop gathering them.  */
+  if (old_options && authdir_mode_bridge(old_options) &&
+      !authdir_mode_bridge(options))
+    rep_hist_desc_stats_term();
+
+  return 0;
+}
+
+/**
+ * Make any necessary modifications to a dirauth_options_t that occur
+ * before validation.  On success return 0; on failure return -1 and
+ * set *<b>msg_out</b> to a newly allocated error string.
+ **/
+static int
+dirauth_options_pre_normalize(void *arg, char **msg_out)
+{
+  dirauth_options_t *options = arg;
+  (void)msg_out;
+
+  if (!options->RecommendedClientVersions)
+    options->RecommendedClientVersions =
+      config_lines_dup(options->RecommendedVersions);
+  if (!options->RecommendedServerVersions)
+    options->RecommendedServerVersions =
+      config_lines_dup(options->RecommendedVersions);
+
+  if (config_ensure_bandwidth_cap(&options->AuthDirFastGuarantee,
+                           "AuthDirFastGuarantee", msg_out) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->AuthDirGuardBWGuarantee,
+                                  "AuthDirGuardBWGuarantee", msg_out) < 0)
+    return -1;
+
+  return 0;
+}
+
+/**
+ * Check whether a dirauth_options_t is correct.
+ *
+ * On success return 0; on failure return -1 and set *<b>msg_out</b> to a
+ * newly allocated error string.
+ **/
+static int
+dirauth_options_validate(const void *arg, char **msg)
+{
+  const dirauth_options_t *options = arg;
+
+  if (options->VersioningAuthoritativeDirectory &&
+      (!options->RecommendedClientVersions ||
+       !options->RecommendedServerVersions)) {
+      REJECT("Versioning authoritative dir servers must set "
+           "Recommended*Versions.");
+  }
+
+  char *t;
+  /* Call these functions to produce warnings only. */
+  t = format_recommended_version_list(options->RecommendedClientVersions, 1);
+  tor_free(t);
+  t = format_recommended_version_list(options->RecommendedServerVersions, 1);
+  tor_free(t);
+
+  if (options->TestingAuthDirTimeToLearnReachability > 2*60*60) {
+    COMPLAIN("TestingAuthDirTimeToLearnReachability is insanely high.");
+  }
+
+  return 0;
+}
+
+/* Declare the options field table for dirauth_options */
+#define CONF_CONTEXT TABLE
+#include "feature/dirauth/dirauth_options.inc"
+#undef CONF_CONTEXT
+
+/** Magic number for dirauth_options_t. */
+#define DIRAUTH_OPTIONS_MAGIC 0x41757448
+
+/**
+ * Declare the configuration options for the dirauth module.
+ **/
+const config_format_t dirauth_options_fmt = {
+  .size = sizeof(dirauth_options_t),
+  .magic = { "dirauth_options_t",
+             DIRAUTH_OPTIONS_MAGIC,
+             offsetof(dirauth_options_t, magic) },
+  .vars = dirauth_options_t_vars,
+
+  .pre_normalize_fn = dirauth_options_pre_normalize,
+  .validate_fn = dirauth_options_validate
+};
diff --git a/src/feature/dirauth/dirauth_config.h b/src/feature/dirauth/dirauth_config.h
new file mode 100644
index 0000000000..9042ff8779
--- /dev/null
+++ b/src/feature/dirauth/dirauth_config.h
@@ -0,0 +1,91 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_config.h
+ * @brief Header for feature/dirauth/dirauth_config.c
+ **/
+
+#ifndef TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H
+#define TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H
+
+struct or_options_t;
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+#include "lib/cc/torint.h"
+
+int options_validate_dirauth_mode(const struct or_options_t *old_options,
+                                  struct or_options_t *options,
+                                  char **msg);
+
+int options_validate_dirauth_schedule(const struct or_options_t *old_options,
+                                      struct or_options_t *options,
+                                      char **msg);
+
+int options_validate_dirauth_testing(const struct or_options_t *old_options,
+                                     struct or_options_t *options,
+                                     char **msg);
+
+int options_act_dirauth(const struct or_options_t *old_options);
+int options_act_dirauth_mtbf(const struct or_options_t *old_options);
+int options_act_dirauth_stats(const struct or_options_t *old_options,
+                              bool *print_notice_out);
+
+bool dirauth_should_reject_requests_under_load(void);
+
+extern const struct config_format_t dirauth_options_fmt;
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+/** When tor is compiled with the dirauth module disabled, it can't be
+ * configured as a directory authority.
+ *
+ * Returns -1 and sets msg to a newly allocated string, if AuthoritativeDir
+ * is set in options. Otherwise returns 0. */
+static inline int
+options_validate_dirauth_mode(const struct or_options_t *old_options,
+                              struct or_options_t *options,
+                              char **msg)
+{
+  (void)old_options;
+
+  /* Only check the primary option for now, #29211 will disable more
+   * options. */
+  if (options->AuthoritativeDir) {
+    /* REJECT() this configuration */
+    *msg = tor_strdup("This tor was built with dirauth mode disabled. "
+                      "It can not be configured with AuthoritativeDir 1.");
+    return -1;
+  }
+
+  return 0;
+}
+
+#define options_validate_dirauth_schedule(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_dirauth_testing(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+
+#define options_act_dirauth(old_options) \
+  (((void)(old_options)),0)
+#define options_act_dirauth_mtbf(old_options) \
+  (((void)(old_options)),0)
+
+static inline int
+options_act_dirauth_stats(const struct or_options_t *old_options,
+                          bool *print_notice_out)
+{
+  (void)old_options;
+  *print_notice_out = 0;
+  return 0;
+}
+
+#define dirauth_should_reject_requests_under_load() (false)
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(TOR_FEATURE_DIRAUTH_DIRAUTH_CONFIG_H) */
diff --git a/src/feature/dirauth/dirauth_options.inc b/src/feature/dirauth/dirauth_options.inc
new file mode 100644
index 0000000000..2aa07a6c88
--- /dev/null
+++ b/src/feature/dirauth/dirauth_options.inc
@@ -0,0 +1,105 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options.inc
+ * @brief Declare configuration options for the crypto_ops module.
+ **/
+
+/** Holds configuration about our directory authority options. */
+BEGIN_CONF_STRUCT(dirauth_options_t)
+
+/** If non-zero, always vote the Fast flag for any relay advertising
+ * this amount of capacity or more. */
+CONF_VAR(AuthDirFastGuarantee, MEMUNIT, 0, "100 KB")
+
+/** If non-zero, this advertised capacity or more is always sufficient
+ * to satisfy the bandwidth requirement for the Guard flag. */
+CONF_VAR(AuthDirGuardBWGuarantee, MEMUNIT, 0, "2 MB")
+
+/** Boolean: are we on IPv6?  */
+CONF_VAR(AuthDirHasIPv6Connectivity, BOOL, 0, "0")
+
+/** True iff we should list bad exits, and vote for all other exits as
+ * good. */
+CONF_VAR(AuthDirListBadExits, BOOL, 0, "0")
+
+/** Do not permit more than this number of servers per IP address. */
+CONF_VAR(AuthDirMaxServersPerAddr, POSINT, 0, "2")
+
+/** Boolean: Do we enforce key-pinning? */
+CONF_VAR(AuthDirPinKeys, BOOL, 0, "1")
+
+/** Bool (default: 1): Switch for the shared random protocol. Only
+ * relevant to a directory authority. If off, the authority won't
+ * participate in the protocol. If on (default), a flag is added to the
+ * vote indicating participation. */
+CONF_VAR(AuthDirSharedRandomness, BOOL, 0, "1")
+
+/** Bool (default: 1): When testing routerinfos as a directory authority,
+ * do we enforce Ed25519 identity match? */
+/* NOTE: remove this option someday. */
+CONF_VAR(AuthDirTestEd25519LinkKeys, BOOL, 0, "1")
+
+/** Authority only: key=value pairs that we add to our networkstatus
+ * consensus vote on the 'params' line. */
+CONF_VAR(ConsensusParams, LINELIST, 0, NULL)
+
+/** Authority only: minimum number of measured bandwidths we must see
+ * before we only believe measured bandwidths to assign flags. */
+CONF_VAR(MinMeasuredBWsForAuthToIgnoreAdvertised, INT, 0, "500")
+
+/** As directory authority, accept hidden service directories after what
+ * time? */
+CONF_VAR(MinUptimeHidServDirectoryV2, INTERVAL, 0, "96 hours")
+
+/** Which versions of tor should we tell users to run? */
+CONF_VAR(RecommendedVersions, LINELIST, 0, NULL)
+
+/** Which versions of tor should we tell users to run on clients? */
+CONF_VAR(RecommendedClientVersions, LINELIST, 0, NULL)
+
+/** Which versions of tor should we tell users to run on relays? */
+CONF_VAR(RecommendedServerVersions, LINELIST, 0, NULL)
+
+/** If an authority has been around for less than this amount of time, it
+ * does not believe its reachability information is accurate.  Only
+ * altered on testing networks. */
+CONF_VAR(TestingAuthDirTimeToLearnReachability, INTERVAL, 0, "30 minutes")
+
+  /** Relays in a testing network which should be voted Exit
+   * regardless of exit policy. */
+CONF_VAR(TestingDirAuthVoteExit, ROUTERSET, 0, NULL)
+CONF_VAR(TestingDirAuthVoteExitIsStrict, BOOL, 0, "0")
+
+/** Relays in a testing network which should be voted Guard
+ * regardless of uptime and bandwidth. */
+CONF_VAR(TestingDirAuthVoteGuard, ROUTERSET, 0, NULL)
+CONF_VAR(TestingDirAuthVoteGuardIsStrict, BOOL, 0, "0")
+
+/** Relays in a testing network which should be voted HSDir
+ * regardless of uptime and DirPort. */
+CONF_VAR(TestingDirAuthVoteHSDir, ROUTERSET, 0, NULL)
+CONF_VAR(TestingDirAuthVoteHSDirIsStrict, BOOL, 0, "0")
+
+/** Minimum value for the Exit flag threshold on testing networks. */
+CONF_VAR(TestingMinExitFlagThreshold, MEMUNIT, 0, "0")
+
+/** Minimum value for the Fast flag threshold on testing networks. */
+CONF_VAR(TestingMinFastFlagThreshold, MEMUNIT, 0, "0")
+
+/** Boolean: is this an authoritative directory that's willing to recommend
+ * versions? */
+CONF_VAR(VersioningAuthoritativeDirectory, BOOL, 0, "0")
+
+/** Boolean: Under bandwidth pressure, if set to 1, the authority will always
+ * answer directory requests from relays but will start sending 503 error code
+ * for the other connections. If set to 0, all connections are considered the
+ * same and the authority will try to answer them all regardless of bandwidth
+ * pressure or not. */
+CONF_VAR(AuthDirRejectRequestsUnderLoad, BOOL, 0, "1")
+
+END_CONF_STRUCT(dirauth_options_t)
diff --git a/src/feature/dirauth/dirauth_options_st.h b/src/feature/dirauth/dirauth_options_st.h
new file mode 100644
index 0000000000..02a498c054
--- /dev/null
+++ b/src/feature/dirauth/dirauth_options_st.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options_st.h
+ * @brief Structure dirauth_options_t to hold directory authority options.
+ **/
+
+#ifndef TOR_FEATURE_DIRAUTH_DIRAUTH_OPTIONS_ST_H
+#define TOR_FEATURE_DIRAUTH_DIRAUTH_OPTIONS_ST_H
+
+#include "lib/conf/confdecl.h"
+#include "feature/nodelist/routerset.h"
+
+#define CONF_CONTEXT STRUCT
+#include "feature/dirauth/dirauth_options.inc"
+#undef CONF_CONTEXT
+
+typedef struct dirauth_options_t dirauth_options_t;
+
+#endif /* !defined(TOR_FEATURE_DIRAUTH_DIRAUTH_OPTIONS_ST_H) */
diff --git a/src/feature/dirauth/dirauth_periodic.c b/src/feature/dirauth/dirauth_periodic.c
new file mode 100644
index 0000000000..19e51c5a05
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.c
@@ -0,0 +1,168 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_periodic.c
+ * @brief Peridoic events for directory authorities.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/or_options_st.h"
+#include "core/mainloop/netstatus.h"
+#include "feature/dirauth/reachability.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/bridgeauth.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/authmode.h"
+
+#include "core/mainloop/periodic.h"
+
+#ifndef COCCI
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+#endif /* !defined(COCCI) */
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: if we're an authority, check on our authority
+ * certificate (the one that authenticates our authority signing key).
+ */
+static int
+check_authority_cert_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  (void)options;
+  /* 1e. Periodically, if we're a v3 authority, we check whether our cert is
+   * close to expiring and warn the admin if it is. */
+  v3_authority_check_key_expiry();
+#define CHECK_V3_CERTIFICATE_INTERVAL (5*60)
+  return CHECK_V3_CERTIFICATE_INTERVAL;
+}
+
+DECLARE_EVENT(check_authority_cert, DIRAUTH, 0);
+
+/**
+ * Scheduled callback: Run directory-authority voting functionality.
+ *
+ * The schedule is a bit complicated here, so dirvote_act() manages the
+ * schedule itself.
+ **/
+static int
+dirvote_callback(time_t now, const or_options_t *options)
+{
+  if (!authdir_mode_v3(options)) {
+    tor_assert_nonfatal_unreached();
+    return 3600;
+  }
+
+  time_t next = dirvote_act(options, now);
+  if (BUG(next == TIME_MAX)) {
+    /* This shouldn't be returned unless we called dirvote_act() without
+     * being an authority.  If it happens, maybe our configuration will
+     * fix itself in an hour or so? */
+    return 3600;
+  }
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(dirvote, DIRAUTH, FL(NEED_NET));
+
+/** Reschedule the directory-authority voting event.  Run this whenever the
+ * schedule has changed. */
+void
+reschedule_dirvote(const or_options_t *options)
+{
+  if (authdir_mode_v3(options)) {
+    periodic_event_reschedule(&dirvote_event);
+  }
+}
+
+/**
+ * Periodic callback: if we're an authority, record our measured stability
+ * information from rephist in an mtbf file.
+ */
+static int
+save_stability_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options)) {
+    if (rep_hist_record_mtbf_data(now, 1)<0) {
+      log_warn(LD_GENERAL, "Couldn't store mtbf data.");
+    }
+  }
+#define SAVE_STABILITY_INTERVAL (30*60)
+  return SAVE_STABILITY_INTERVAL;
+}
+
+DECLARE_EVENT(save_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're an authority, make sure we test
+ * the routers on the network for reachability.
+ */
+static int
+launch_reachability_tests_callback(time_t now, const or_options_t *options)
+{
+  if (authdir_mode_tests_reachability(options) &&
+      !net_is_disabled()) {
+    /* try to determine reachability of the other Tor relays */
+    dirserv_test_reachability(now);
+  }
+  return REACHABILITY_TEST_INTERVAL;
+}
+
+DECLARE_EVENT(launch_reachability_tests, AUTHORITIES, FL(NEED_NET));
+
+/**
+ * Periodic callback: if we're an authority, discount the stability
+ * information (and other rephist information) that's older.
+ */
+static int
+downrate_stability_callback(time_t now, const or_options_t *options)
+{
+  (void)options;
+  /* 1d. Periodically, we discount older stability information so that new
+   * stability info counts more, and save the stability information to disk as
+   * appropriate. */
+  time_t next = rep_hist_downrate_old_runs(now);
+  return safe_timer_diff(now, next);
+}
+
+DECLARE_EVENT(downrate_stability, AUTHORITIES, 0);
+
+/**
+ * Periodic callback: if we're the bridge authority, write a networkstatus
+ * file to disk.
+ */
+static int
+write_bridge_ns_callback(time_t now, const or_options_t *options)
+{
+  if (options->BridgeAuthoritativeDir) {
+    bridgeauth_dump_bridge_status_to_file(now);
+#define BRIDGE_STATUSFILE_INTERVAL (30*60)
+     return BRIDGE_STATUSFILE_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(write_bridge_ns, BRIDGEAUTH, 0);
+
+void
+dirauth_register_periodic_events(void)
+{
+  periodic_events_register(&downrate_stability_event);
+  periodic_events_register(&launch_reachability_tests_event);
+  periodic_events_register(&save_stability_event);
+  periodic_events_register(&check_authority_cert_event);
+  periodic_events_register(&dirvote_event);
+  periodic_events_register(&write_bridge_ns_event);
+}
diff --git a/src/feature/dirauth/dirauth_periodic.h b/src/feature/dirauth/dirauth_periodic.h
new file mode 100644
index 0000000000..ccdda92a77
--- /dev/null
+++ b/src/feature/dirauth/dirauth_periodic.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_periodic.h
+ * @brief Header for dirauth_periodic.c
+ **/
+
+#ifndef DIRVOTE_PERIODIC_H
+#define DIRVOTE_PERIODIC_H
+
+#ifdef HAVE_MODULE_DIRAUTH
+
+void dirauth_register_periodic_events(void);
+void reschedule_dirvote(const or_options_t *options);
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+static inline void
+reschedule_dirvote(const or_options_t *options)
+{
+  (void)options;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(DIRVOTE_PERIODIC_H) */
diff --git a/src/feature/dirauth/dirauth_stub.c b/src/feature/dirauth/dirauth_stub.c
new file mode 100644
index 0000000000..9f48ce14fd
--- /dev/null
+++ b/src/feature/dirauth/dirauth_stub.c
@@ -0,0 +1,34 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_stub.c
+ * @brief Stub declarations for use when dirauth module is disabled.
+ **/
+
+#include "orconfig.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confdecl.h"
+#include "lib/subsys/subsys.h"
+
+/* Declare the options field table for dirauth_options */
+#define CONF_CONTEXT STUB_TABLE
+#include "feature/dirauth/dirauth_options.inc"
+#undef CONF_CONTEXT
+
+static const config_format_t dirauth_options_stub_fmt = {
+  .vars = dirauth_options_t_vars,
+};
+
+const struct subsys_fns_t sys_dirauth = {
+  .name = "dirauth",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = false,
+  .level = DIRAUTH_SUBSYS_LEVEL,
+
+  .options_format = &dirauth_options_stub_fmt
+};
diff --git a/src/feature/dirauth/dirauth_sys.c b/src/feature/dirauth/dirauth_sys.c
new file mode 100644
index 0000000000..07c5743877
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.c
@@ -0,0 +1,71 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_sys.c
+ * @brief Directory authority subsystem declarations
+ **/
+
+#include "core/or/or.h"
+
+#define DIRAUTH_SYS_PRIVATE
+#include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/dirauth/dirauth_periodic.h"
+#include "feature/dirauth/keypin.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/dirauth/dirauth_config.h"
+
+#include "feature/dirauth/dirauth_options_st.h"
+
+#include "lib/subsys/subsys.h"
+
+static const dirauth_options_t *global_dirauth_options;
+
+static int
+subsys_dirauth_initialize(void)
+{
+  dirauth_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_dirauth_shutdown(void)
+{
+  dirserv_free_fingerprint_list();
+  dirvote_free_all();
+  dirserv_clear_measured_bw_cache();
+  keypin_close_journal();
+  global_dirauth_options = NULL;
+}
+
+const dirauth_options_t *
+dirauth_get_options(void)
+{
+  tor_assert(global_dirauth_options);
+  return global_dirauth_options;
+}
+
+STATIC int
+dirauth_set_options(void *arg)
+{
+  dirauth_options_t *opts = arg;
+  global_dirauth_options = opts;
+  return 0;
+}
+
+const struct subsys_fns_t sys_dirauth = {
+  .name = "dirauth",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = DIRAUTH_SUBSYS_LEVEL,
+  .initialize = subsys_dirauth_initialize,
+  .shutdown = subsys_dirauth_shutdown,
+
+  .options_format = &dirauth_options_fmt,
+  .set_options = dirauth_set_options,
+};
diff --git a/src/feature/dirauth/dirauth_sys.h b/src/feature/dirauth/dirauth_sys.h
new file mode 100644
index 0000000000..c512b91b33
--- /dev/null
+++ b/src/feature/dirauth/dirauth_sys.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_sys.h
+ * @brief Header for dirauth_sys.c
+ **/
+
+#ifndef DIRAUTH_SYS_H
+#define DIRAUTH_SYS_H
+
+struct dirauth_options_t;
+const struct dirauth_options_t *dirauth_get_options(void);
+
+extern const struct subsys_fns_t sys_dirauth;
+
+/**
+ * Subsystem level for the directory-authority system.
+ *
+ * Defined here so that it can be shared between the real and stub
+ * definitions.
+ **/
+#define DIRAUTH_SUBSYS_LEVEL 70
+
+#ifdef DIRAUTH_SYS_PRIVATE
+STATIC int dirauth_set_options(void *arg);
+#endif
+
+#endif /* !defined(DIRAUTH_SYS_H) */
diff --git a/src/feature/dirauth/dircollate.c b/src/feature/dirauth/dircollate.c
index 7992e3a85f..2657f53853 100644
--- a/src/feature/dirauth/dircollate.c
+++ b/src/feature/dirauth/dircollate.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,8 +32,8 @@ static void dircollator_collate_by_ed25519(dircollator_t *dc);
 
 /** Hashtable entry mapping a pair of digests (actually an ed25519 key and an
  * RSA SHA1 digest) to an array of vote_routerstatus_t. */
-typedef struct ddmap_entry_s {
-  HT_ENTRY(ddmap_entry_s) node;
+typedef struct ddmap_entry_t {
+  HT_ENTRY(ddmap_entry_t) node;
   /** A SHA1-RSA1024 identity digest and Ed25519 identity key,
    * concatenated.  (If there is no ed25519 identity key, there is no
    * entry in this table.) */
@@ -89,10 +89,10 @@ ddmap_entry_set_digests(ddmap_entry_t *ent,
   memcpy(ent->d + DIGEST_LEN, ed25519, DIGEST256_LEN);
 }
 
-HT_PROTOTYPE(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash,
-             ddmap_entry_eq)
-HT_GENERATE2(double_digest_map, ddmap_entry_s, node, ddmap_entry_hash,
-             ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_)
+HT_PROTOTYPE(double_digest_map, ddmap_entry_t, node, ddmap_entry_hash,
+             ddmap_entry_eq);
+HT_GENERATE2(double_digest_map, ddmap_entry_t, node, ddmap_entry_hash,
+             ddmap_entry_eq, 0.6, tor_reallocarray, tor_free_);
 
 /** Helper: add a single vote_routerstatus_t <b>vrs</b> to the collator
  * <b>dc</b>, indexing it by its RSA key digest, and by the 2-tuple of its RSA
@@ -324,4 +324,3 @@ dircollator_get_votes_for_router(dircollator_t *dc, int idx)
   return digestmap_get(dc->by_collated_rsa_sha1,
                        smartlist_get(dc->all_rsa_sha1_lst, idx));
 }
-
diff --git a/src/feature/dirauth/dircollate.h b/src/feature/dirauth/dircollate.h
index 754a094817..90c6bddad5 100644
--- a/src/feature/dirauth/dircollate.h
+++ b/src/feature/dirauth/dircollate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
 
-typedef struct dircollator_s dircollator_t;
+typedef struct dircollator_t dircollator_t;
 
 dircollator_t *dircollator_new(int n_votes, int n_authorities);
 void dircollator_free_(dircollator_t *obj);
@@ -30,11 +30,11 @@ vote_routerstatus_t **dircollator_get_votes_for_router(dircollator_t *dc,
                                                        int idx);
 
 #ifdef DIRCOLLATE_PRIVATE
-struct ddmap_entry_s;
-typedef HT_HEAD(double_digest_map, ddmap_entry_s) double_digest_map_t;
+struct ddmap_entry_t;
+typedef HT_HEAD(double_digest_map, ddmap_entry_t) double_digest_map_t;
 /** A dircollator keeps track of all the routerstatus entries in a
  * set of networkstatus votes, and matches them by an appropriate rule. */
-struct dircollator_s {
+struct dircollator_t {
   /** True iff we have run the collation algorithm. */
   int is_collated;
   /** The total number of votes that we received. */
diff --git a/src/feature/dirauth/dirvote.c b/src/feature/dirauth/dirvote.c
index 9e01cee42a..d9fbd2a7ce 100644
--- a/src/feature/dirauth/dirvote.c
+++ b/src/feature/dirauth/dirvote.c
@@ -1,11 +1,12 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DIRVOTE_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "core/or/tor_version_st.h"
@@ -28,6 +29,7 @@
 #include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -35,15 +37,17 @@
 #include "feature/stats/rephist.h"
 #include "feature/client/entrynodes.h" /* needed for guardfraction methods */
 #include "feature/nodelist/torcert.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random_state.h"
+#include "feature/dirauth/dirauth_sys.h"
 
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/dircache/cached_dir_st.h"
 #include "feature/dirclient/dir_server_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/document_signature_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -60,6 +64,9 @@
 #include "lib/encoding/confline.h"
 #include "lib/crypt_ops/crypto_format.h"
 
+/* Algorithm to use for the bandwidth file digest. */
+#define DIGEST_ALG_BW_FILE DIGEST_SHA256
+
 /**
  * \file dirvote.c
  * \brief Functions to compute directory consensus, and schedule voting.
@@ -216,7 +223,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                           networkstatus_t *v3_ns)
 {
   smartlist_t *chunks = smartlist_new();
-  char *packages = NULL;
   char fingerprint[FINGERPRINT_LEN+1];
   char digest[DIGEST_LEN];
   uint32_t addr;
@@ -242,19 +248,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
                                                 v3_ns->server_versions);
   protocols_lines = format_protocols_lines_for_vote(v3_ns);
 
-  if (v3_ns->package_lines) {
-    smartlist_t *tmp = smartlist_new();
-    SMARTLIST_FOREACH(v3_ns->package_lines, const char *, p,
-                      if (validate_recommended_package_line(p))
-                        smartlist_add_asprintf(tmp, "package %s\n", p));
-    smartlist_sort_strings(tmp);
-    packages = smartlist_join_strings(tmp, "", 0, NULL);
-    SMARTLIST_FOREACH(tmp, char *, cp, tor_free(cp));
-    smartlist_free(tmp);
-  } else {
-    packages = tor_strdup("");
-  }
-
     /* Get shared random commitments/reveals line(s). */
   shared_random_vote_str = sr_get_string_for_vote();
 
@@ -268,6 +261,7 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     char *flag_thresholds = dirserv_get_flag_thresholds_line();
     char *params;
     char *bw_headers_line = NULL;
+    char *bw_file_digest = NULL;
     authority_cert_t *cert = v3_ns->cert;
     char *methods =
       make_consensus_method_list(MIN_SUPPORTED_CONSENSUS_METHOD,
@@ -307,43 +301,68 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
       tor_free(bw_file_headers);
     }
 
-    smartlist_add_asprintf(chunks,
-                 "network-status-version 3\n"
-                 "vote-status %s\n"
-                 "consensus-methods %s\n"
-                 "published %s\n"
-                 "valid-after %s\n"
-                 "fresh-until %s\n"
-                 "valid-until %s\n"
-                 "voting-delay %d %d\n"
-                 "%s%s" /* versions */
-                 "%s" /* protocols */
-                 "%s" /* packages */
-                 "known-flags %s\n"
-                 "flag-thresholds %s\n"
-                 "params %s\n"
-                 "%s" /* bandwidth file headers */
-                 "dir-source %s %s %s %s %d %d\n"
-                 "contact %s\n"
-                 "%s" /* shared randomness information */
-                 ,
-                 v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
-                 methods,
-                 published, va, fu, vu,
-                 v3_ns->vote_seconds, v3_ns->dist_seconds,
-                 client_versions_line,
-                 server_versions_line,
-                 protocols_lines,
-                 packages,
-                 flags,
-                 flag_thresholds,
-                 params,
-                 bw_headers_line ? bw_headers_line : "",
-                 voter->nickname, fingerprint, voter->address,
-                 fmt_addr32(addr), voter->dir_port, voter->or_port,
-                 voter->contact,
-                 shared_random_vote_str ?
-                           shared_random_vote_str : "");
+    /* Create bandwidth-file-digest if applicable.
+     * v3_ns->b64_digest_bw_file will contain the digest when V3BandwidthsFile
+     * is configured and the bandwidth file could be read, even if it was not
+     * parseable.
+     */
+    if (!tor_digest256_is_zero((const char *)v3_ns->bw_file_digest256)) {
+      /* Encode the digest. */
+      char b64_digest_bw_file[BASE64_DIGEST256_LEN+1] = {0};
+      digest256_to_base64(b64_digest_bw_file,
+                          (const char *)v3_ns->bw_file_digest256);
+      /* "bandwidth-file-digest" 1*(SP algorithm "=" digest) NL */
+      char *digest_algo_b64_digest_bw_file = NULL;
+      tor_asprintf(&digest_algo_b64_digest_bw_file, "%s=%s",
+                   crypto_digest_algorithm_get_name(DIGEST_ALG_BW_FILE),
+                   b64_digest_bw_file);
+      /* No need for tor_strdup(""), format_line_if_present does it. */
+      bw_file_digest = format_line_if_present(
+          "bandwidth-file-digest", digest_algo_b64_digest_bw_file);
+      tor_free(digest_algo_b64_digest_bw_file);
+    }
+
+    const char *ip_str = fmt_addr32(addr);
+
+    if (ip_str[0]) {
+      smartlist_add_asprintf(chunks,
+                   "network-status-version 3\n"
+                   "vote-status %s\n"
+                   "consensus-methods %s\n"
+                   "published %s\n"
+                   "valid-after %s\n"
+                   "fresh-until %s\n"
+                   "valid-until %s\n"
+                   "voting-delay %d %d\n"
+                   "%s%s" /* versions */
+                   "%s" /* protocols */
+                   "known-flags %s\n"
+                   "flag-thresholds %s\n"
+                   "params %s\n"
+                   "%s" /* bandwidth file headers */
+                   "%s" /* bandwidth file digest */
+                   "dir-source %s %s %s %s %d %d\n"
+                   "contact %s\n"
+                   "%s" /* shared randomness information */
+                   ,
+                   v3_ns->type == NS_TYPE_VOTE ? "vote" : "opinion",
+                   methods,
+                   published, va, fu, vu,
+                   v3_ns->vote_seconds, v3_ns->dist_seconds,
+                   client_versions_line,
+                   server_versions_line,
+                   protocols_lines,
+                   flags,
+                   flag_thresholds,
+                   params,
+                   bw_headers_line ? bw_headers_line : "",
+                   bw_file_digest ? bw_file_digest: "",
+                   voter->nickname, fingerprint, voter->address,
+                   ip_str, voter->dir_port, voter->or_port,
+                   voter->contact,
+                   shared_random_vote_str ?
+                             shared_random_vote_str : "");
+    }
 
     tor_free(params);
     tor_free(flags);
@@ -351,6 +370,10 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     tor_free(methods);
     tor_free(shared_random_vote_str);
     tor_free(bw_headers_line);
+    tor_free(bw_file_digest);
+
+    if (ip_str[0] == '\0')
+      goto err;
 
     if (!tor_digest_is_zero(voter->legacy_id_digest)) {
       char fpbuf[HEX_DIGEST_LEN+1];
@@ -369,7 +392,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
     rsf = routerstatus_format_entry(&vrs->status,
                                     vrs->version, vrs->protocols,
                                     NS_V3_VOTE,
-                                    ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
                                     vrs);
     if (rsf)
       smartlist_add(chunks, rsf);
@@ -412,7 +434,8 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
 
   {
     networkstatus_t *v;
-    if (!(v = networkstatus_parse_vote_from_string(status, NULL,
+    if (!(v = networkstatus_parse_vote_from_string(status, strlen(status),
+                                                   NULL,
                                                    v3_ns->type))) {
       log_err(LD_BUG,"Generated a networkstatus %s we couldn't parse: "
               "<<%s>>",
@@ -430,7 +453,6 @@ format_networkstatus_vote(crypto_pk_t *private_signing_key,
   tor_free(client_versions_line);
   tor_free(server_versions_line);
   tor_free(protocols_lines);
-  tor_free(packages);
 
   SMARTLIST_FOREACH(chunks, char *, cp, tor_free(cp));
   smartlist_free(chunks);
@@ -872,7 +894,7 @@ dirvote_get_intermediate_param_value(const smartlist_t *param_list,
       int ok;
       value = (int32_t)
         tor_parse_long(integer_str, 10, INT32_MIN, INT32_MAX, &ok, NULL);
-      if (BUG(! ok))
+      if (BUG(!ok))
         return default_val;
       ++n_found;
     }
@@ -1525,14 +1547,11 @@ networkstatus_compute_consensus(smartlist_t *votes,
     consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD;
   }
 
-  if (consensus_method >= MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE) {
+  {
     /* It's smarter to initialize these weights to 1, so that later on,
      * we can't accidentally divide by zero. */
     G = M = E = D = 1;
     T = 4;
-  } else {
-    /* ...but originally, they were set to zero. */
-    G = M = E = D = T = 0;
   }
 
   /* Compute medians of time-related things, and figure out how many
@@ -2233,7 +2252,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         /* Okay!! Now we can write the descriptor... */
         /*     First line goes into "buf". */
         buf = routerstatus_format_entry(&rs_out, NULL, NULL,
-                                        rs_format, consensus_method, NULL);
+                                        rs_format, NULL);
         if (buf)
           smartlist_add(chunks, buf);
       }
@@ -2253,8 +2272,7 @@ networkstatus_compute_consensus(smartlist_t *votes,
         smartlist_add_strdup(chunks, chosen_version);
       }
       smartlist_add_strdup(chunks, "\n");
-      if (chosen_protocol_list &&
-          consensus_method >= MIN_METHOD_FOR_RS_PROTOCOLS) {
+      if (chosen_protocol_list) {
         smartlist_add_asprintf(chunks, "pr %s\n", chosen_protocol_list);
       }
       /*     Now the weight line. */
@@ -2409,7 +2427,8 @@ networkstatus_compute_consensus(smartlist_t *votes,
 
   {
     networkstatus_t *c;
-    if (!(c = networkstatus_parse_vote_from_string(result, NULL,
+    if (!(c = networkstatus_parse_vote_from_string(result, strlen(result),
+                                                   NULL,
                                                    NS_TYPE_CONSENSUS))) {
       log_err(LD_BUG, "Generated a networkstatus consensus we couldn't "
               "parse.");
@@ -2516,9 +2535,12 @@ compute_consensus_package_lines(smartlist_t *votes)
  * any new signatures in <b>src_voter_list</b> that should be added to
  * <b>target</b>. (A signature should be added if we have no signature for that
  * voter in <b>target</b> yet, or if we have no verifiable signature and the
- * new signature is verifiable.)  Return the number of signatures added or
- * changed, or -1 if the document signed by <b>sigs</b> isn't the same
- * document as <b>target</b>. */
+ * new signature is verifiable.)
+ *
+ * Return the number of signatures added or changed, or -1 if the document
+ * signatures are invalid. Sets *<b>msg_out</b> to a string constant
+ * describing the signature status.
+ */
 STATIC int
 networkstatus_add_detached_signatures(networkstatus_t *target,
                                       ns_detached_signatures_t *sigs,
@@ -2567,7 +2589,7 @@ networkstatus_add_detached_signatures(networkstatus_t *target,
       return -1;
     }
     for (alg = DIGEST_SHA1; alg < N_COMMON_DIGEST_ALGORITHMS; ++alg) {
-      if (!tor_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
+      if (!fast_mem_is_zero(digests->d[alg], DIGEST256_LEN)) {
         if (fast_memeq(target->digests.d[alg], digests->d[alg],
                        DIGEST256_LEN)) {
           ++n_matches;
@@ -2763,7 +2785,7 @@ networkstatus_get_detached_signatures(smartlist_t *consensuses)
       char d[HEX_DIGEST256_LEN+1];
       const char *alg_name =
         crypto_digest_algorithm_get_name(alg);
-      if (tor_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
+      if (fast_mem_is_zero(ns->digests.d[alg], DIGEST256_LEN))
         continue;
       base16_encode(d, sizeof(d), ns->digests.d[alg], DIGEST256_LEN);
       smartlist_add_asprintf(elements, "additional-digest %s %s %s\n",
@@ -2839,7 +2861,7 @@ dirvote_act(const or_options_t *options, time_t now)
                "Mine is %s.",
                keys, hex_str(c->cache_info.identity_digest, DIGEST_LEN));
     tor_free(keys);
-    voting_schedule_recalculate_timing(options, now);
+    dirauth_sched_recalculate_timing(options, now);
   }
 
 #define IF_TIME_FOR_NEXT_ACTION(when_field, done_field) \
@@ -2885,7 +2907,7 @@ dirvote_act(const or_options_t *options, time_t now)
                 networkstatus_get_latest_consensus_by_flavor(FLAV_NS));
     /* XXXX We will want to try again later if we haven't got enough
      * signatures yet.  Implement this if it turns out to ever happen. */
-    voting_schedule_recalculate_timing(options, now);
+    dirauth_sched_recalculate_timing(options, now);
     return voting_schedule.voting_starts;
   } ENDIF
 
@@ -2952,7 +2974,7 @@ dirvote_perform_vote(void)
   if (!contents)
     return -1;
 
-  pending_vote = dirvote_add_vote(contents, &msg, &status);
+  pending_vote = dirvote_add_vote(contents, 0, &msg, &status);
   tor_free(contents);
   if (!pending_vote) {
     log_warn(LD_DIR, "Couldn't store my own vote! (I told myself, '%s'.)",
@@ -3108,13 +3130,45 @@ list_v3_auth_ids(void)
   return keys;
 }
 
+/* Check the voter information <b>vi</b>, and  assert that at least one
+ * signature is good. Asserts on failure. */
+static void
+assert_any_sig_good(const networkstatus_voter_info_t *vi)
+{
+  int any_sig_good = 0;
+  SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig,
+                    if (sig->good_signature)
+                      any_sig_good = 1);
+  tor_assert(any_sig_good);
+}
+
+/* Add <b>cert</b> to our list of known authority certificates. */
+static void
+add_new_cert_if_needed(const struct authority_cert_t *cert)
+{
+  tor_assert(cert);
+  if (!authority_cert_get_by_digests(cert->cache_info.identity_digest,
+                                     cert->signing_key_digest)) {
+    /* Hey, it's a new cert! */
+    trusted_dirs_load_certs_from_string(
+                               cert->cache_info.signed_descriptor_body,
+                               TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/,
+                               NULL);
+    if (!authority_cert_get_by_digests(cert->cache_info.identity_digest,
+                                       cert->signing_key_digest)) {
+      log_warn(LD_BUG, "We added a cert, but still couldn't find it.");
+    }
+  }
+}
+
 /** Called when we have received a networkstatus vote in <b>vote_body</b>.
  * Parse and validate it, and on success store it as a pending vote (which we
  * then return).  Return NULL on failure.  Sets *<b>msg_out</b> and
  * *<b>status_out</b> to an HTTP response and status code.  (V3 authority
  * only) */
 pending_vote_t *
-dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+dirvote_add_vote(const char *vote_body, time_t time_posted,
+                 const char **msg_out, int *status_out)
 {
   networkstatus_t *vote;
   networkstatus_voter_info_t *vi;
@@ -3132,7 +3186,8 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
   *msg_out = NULL;
 
  again:
-  vote = networkstatus_parse_vote_from_string(vote_body, &end_of_vote,
+  vote = networkstatus_parse_vote_from_string(vote_body, strlen(vote_body),
+                                              &end_of_vote,
                                               NS_TYPE_VOTE);
   if (!end_of_vote)
     end_of_vote = vote_body + strlen(vote_body);
@@ -3144,13 +3199,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
   }
   tor_assert(smartlist_len(vote->voters) == 1);
   vi = get_voter(vote);
-  {
-    int any_sig_good = 0;
-    SMARTLIST_FOREACH(vi->sigs, document_signature_t *, sig,
-                      if (sig->good_signature)
-                        any_sig_good = 1);
-    tor_assert(any_sig_good);
-  }
+  assert_any_sig_good(vi);
   ds = trusteddirserver_get_by_v3_auth_digest(vi->identity_digest);
   if (!ds) {
     char *keys = list_v3_auth_ids();
@@ -3163,19 +3212,7 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
     *msg_out = "Vote not from a recognized v3 authority";
     goto err;
   }
-  tor_assert(vote->cert);
-  if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest,
-                                     vote->cert->signing_key_digest)) {
-    /* Hey, it's a new cert! */
-    trusted_dirs_load_certs_from_string(
-                               vote->cert->cache_info.signed_descriptor_body,
-                               TRUSTED_DIRS_CERTS_SRC_FROM_VOTE, 1 /*flush*/,
-                               NULL);
-    if (!authority_cert_get_by_digests(vote->cert->cache_info.identity_digest,
-                                       vote->cert->signing_key_digest)) {
-      log_warn(LD_BUG, "We added a cert, but still couldn't find it.");
-    }
-  }
+  add_new_cert_if_needed(vote->cert);
 
   /* Is it for the right period? */
   if (vote->valid_after != voting_schedule.interval_starts) {
@@ -3188,6 +3225,23 @@ dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
     goto err;
   }
 
+  /* Check if we received it, as a post, after the cutoff when we
+   * start asking other dir auths for it. If we do, the best plan
+   * is to discard it, because using it greatly increases the chances
+   * of a split vote for this round (some dir auths got it in time,
+   * some didn't). */
+  if (time_posted && time_posted > voting_schedule.fetch_missing_votes) {
+    char tbuf1[ISO_TIME_LEN+1], tbuf2[ISO_TIME_LEN+1];
+    format_iso_time(tbuf1, time_posted);
+    format_iso_time(tbuf2, voting_schedule.fetch_missing_votes);
+    log_warn(LD_DIR, "Rejecting posted vote from %s received at %s; "
+             "our cutoff for received votes is %s. Check your clock, "
+             "CPU load, and network load. Also check the authority that "
+             "posted the vote.", vi->address, tbuf1, tbuf2);
+    *msg_out = "Posted vote received too late, would be dangerous to count it";
+    goto err;
+  }
+
   /* Fetch any new router descriptors we just learned about */
   update_consensus_router_descriptor_downloads(time(NULL), 1, vote);
 
@@ -3390,7 +3444,9 @@ dirvote_compute_consensuses(void)
                  flavor_name);
         continue;
       }
-      consensus = networkstatus_parse_vote_from_string(consensus_body, NULL,
+      consensus = networkstatus_parse_vote_from_string(consensus_body,
+                                                       strlen(consensus_body),
+                                                       NULL,
                                                        NS_TYPE_CONSENSUS);
       if (!consensus) {
         log_warn(LD_DIR, "Couldn't parse %s consensus we generated!",
@@ -3529,7 +3585,7 @@ dirvote_add_signatures_to_pending_consensus(
      * just in case we break detached signature processing at some point. */
     {
       networkstatus_t *v = networkstatus_parse_vote_from_string(
-                                             pc->body, NULL,
+                                             pc->body, strlen(pc->body), NULL,
                                              NS_TYPE_CONSENSUS);
       tor_assert(v);
       networkstatus_vote_free(v);
@@ -3550,6 +3606,14 @@ dirvote_add_signatures_to_pending_consensus(
   return r;
 }
 
+/** Helper: we just got the <b>detached_signatures_body</b> sent to us as
+ * signatures on the currently pending consensus.  Add them to the pending
+ * consensus (if we have one).
+ *
+ * Set *<b>msg</b> to a string constant describing the status, regardless of
+ * success or failure.
+ *
+ * Return negative on failure, nonnegative on success. */
 static int
 dirvote_add_signatures_to_all_pending_consensuses(
                        const char *detached_signatures_body,
@@ -3612,7 +3676,12 @@ dirvote_add_signatures_to_all_pending_consensuses(
 /** Helper: we just got the <b>detached_signatures_body</b> sent to us as
  * signatures on the currently pending consensus.  Add them to the pending
  * consensus (if we have one); otherwise queue them until we have a
- * consensus.  Return negative on failure, nonnegative on success. */
+ * consensus.
+ *
+ * Set *<b>msg</b> to a string constant describing the status, regardless of
+ * success or failure.
+ *
+ * Return negative on failure, nonnegative on success. */
 int
 dirvote_add_signatures(const char *detached_signatures_body,
                        const char *source,
@@ -3654,7 +3723,9 @@ dirvote_publish_consensus(void)
       continue;
     }
 
-    if (networkstatus_set_current_consensus(pending->body, name, 0, NULL))
+    if (networkstatus_set_current_consensus(pending->body,
+                                            strlen(pending->body),
+                                            name, 0, NULL))
       log_warn(LD_DIR, "Error publishing %s consensus", name);
     else
       log_notice(LD_DIR, "Published %s consensus", name);
@@ -3784,15 +3855,16 @@ dirvote_create_microdescriptor(const routerinfo_t *ri, int consensus_method)
     smartlist_add_asprintf(chunks, "ntor-onion-key %s", kbuf);
   }
 
-  /* We originally put a lines in the micrdescriptors, but then we worked out
-   * that we needed them in the microdesc consensus. See #20916. */
-  if (consensus_method < MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC &&
-      !tor_addr_is_null(&ri->ipv6_addr) && ri->ipv6_orport)
-    smartlist_add_asprintf(chunks, "a %s\n",
-                           fmt_addrport(&ri->ipv6_addr, ri->ipv6_orport));
-
-  if (family)
-    smartlist_add_asprintf(chunks, "family %s\n", family);
+  if (family) {
+    if (consensus_method < MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS) {
+      smartlist_add_asprintf(chunks, "family %s\n", family);
+    } else {
+      const uint8_t *id = (const uint8_t *)ri->cache_info.identity_digest;
+      char *canonical_family = nodefamily_canonicalize(family, id, 0);
+      smartlist_add_asprintf(chunks, "family %s\n", canonical_family);
+      tor_free(canonical_family);
+    }
+  }
 
   if (summary && strcmp(summary, "reject 1-65535"))
     smartlist_add_asprintf(chunks, "p %s\n", summary);
@@ -3869,8 +3941,7 @@ dirvote_format_microdesc_vote_line(char *out_buf, size_t out_buf_len,
                                ",");
   tor_assert(microdesc_consensus_methods);
 
-  if (digest256_to_base64(d64, md->digest)<0)
-    goto out;
+  digest256_to_base64(d64, md->digest);
 
   if (tor_snprintf(out_buf, out_buf_len, "m %s sha256=%s\n",
                    microdesc_consensus_methods, d64)<0)
@@ -3889,8 +3960,10 @@ static const struct consensus_method_range_t {
   int low;
   int high;
 } microdesc_consensus_methods[] = {
-  {MIN_SUPPORTED_CONSENSUS_METHOD, MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC - 1},
-  {MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC, MAX_SUPPORTED_CONSENSUS_METHOD},
+  {MIN_SUPPORTED_CONSENSUS_METHOD,
+   MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS - 1},
+  {MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS,
+   MAX_SUPPORTED_CONSENSUS_METHOD},
   {-1, -1}
 };
 
@@ -4199,7 +4272,7 @@ compare_routerinfo_by_ip_and_bw_(const void **a, const void **b)
 static digestmap_t *
 get_possible_sybil_list(const smartlist_t *routers)
 {
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
   digestmap_t *omit_as_sybil;
   smartlist_t *routers_by_ip = smartlist_new();
   uint32_t last_addr;
@@ -4364,6 +4437,23 @@ clear_status_flags_on_sybil(routerstatus_t *rs)
    * forget to add it to this clause. */
 }
 
+/** Space-separated list of all the flags that we will always vote on. */
+const char DIRVOTE_UNIVERSAL_FLAGS[] =
+  "Authority "
+  "Exit "
+  "Fast "
+  "Guard "
+  "HSDir "
+  "Stable "
+  "StaleDesc "
+  "V2Dir "
+  "Valid";
+/** Space-separated list of all flags that we may or may not vote on,
+ * depending on our configuration. */
+const char DIRVOTE_OPTIONAL_FLAGS[] =
+  "BadExit "
+  "Running";
+
 /** Return a new networkstatus_t* containing our current opinion. (For v3
  * authorities) */
 networkstatus_t *
@@ -4371,6 +4461,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
                                         authority_cert_t *cert)
 {
   const or_options_t *options = get_options();
+  const dirauth_options_t *d_options = dirauth_get_options();
   networkstatus_t *v3_out = NULL;
   uint32_t addr;
   char *hostname = NULL, *client_versions = NULL, *server_versions = NULL;
@@ -4378,7 +4469,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   smartlist_t *routers, *routerstatuses;
   char identity_digest[DIGEST_LEN];
   char signing_key_digest[DIGEST_LEN];
-  int listbadexits = options->AuthDirListBadExits;
+  const int listbadexits = d_options->AuthDirListBadExits;
   routerlist_t *rl = router_get_routerlist();
   time_t now = time(NULL);
   time_t cutoff = now - ROUTER_MAX_AGE_TO_PUBLISH;
@@ -4388,6 +4479,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   const int vote_on_reachability = running_long_enough_to_decide_unreachable();
   smartlist_t *microdescriptors = NULL;
   smartlist_t *bw_file_headers = NULL;
+  uint8_t bw_file_digest256[DIGEST256_LEN] = {0};
 
   tor_assert(private_key);
   tor_assert(cert);
@@ -4409,11 +4501,16 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     hostname = tor_dup_ip(addr);
   }
 
-  if (options->VersioningAuthoritativeDir) {
+  if (!hostname) {
+    log_err(LD_BUG, "Failed to determine hostname AND duplicate address");
+    return NULL;
+  }
+
+  if (d_options->VersioningAuthoritativeDirectory) {
     client_versions =
-      format_recommended_version_list(options->RecommendedClientVersions, 0);
+      format_recommended_version_list(d_options->RecommendedClientVersions, 0);
     server_versions =
-      format_recommended_version_list(options->RecommendedServerVersions, 0);
+      format_recommended_version_list(d_options->RecommendedServerVersions, 0);
   }
 
   contact = get_options()->ContactInfo;
@@ -4425,7 +4522,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
    * set_routerstatus_from_routerinfo() see up-to-date bandwidth info.
    */
   if (options->V3BandwidthsFile) {
-    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL);
+    dirserv_read_measured_bandwidths(options->V3BandwidthsFile, NULL, NULL,
+                                     NULL);
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4479,8 +4577,8 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
       vrs = tor_malloc_zero(sizeof(vote_routerstatus_t));
       rs = &vrs->status;
-      set_routerstatus_from_routerinfo(rs, node, ri, now,
-                                       listbadexits);
+      dirauth_set_routerstatus_from_routerinfo(rs, node, ri, now,
+                                               listbadexits);
 
       if (ri->cache_info.signing_key_cert) {
         memcpy(vrs->ed25519_id,
@@ -4530,7 +4628,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     /* Only set bw_file_headers when V3BandwidthsFile is configured */
     bw_file_headers = smartlist_new();
     dirserv_read_measured_bandwidths(options->V3BandwidthsFile,
-                                     routerstatuses, bw_file_headers);
+                                     routerstatuses, bw_file_headers,
+                                     bw_file_digest256);
+
   } else {
     /*
      * No bandwidths file; clear the measured bandwidth cache in case we had
@@ -4557,7 +4657,7 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     else
       last_consensus_interval = options->TestingV3AuthInitialVotingInterval;
     v3_out->valid_after =
-      voting_schedule_get_start_of_next_interval(now,
+      voting_sched_get_start_of_interval_after(now,
                                    (int)last_consensus_interval,
                                    options->TestingV3AuthVotingStartOffset);
     format_iso_time(tbuf, v3_out->valid_after);
@@ -4579,17 +4679,14 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
 
   /* These are hardwired, to avoid disaster. */
   v3_out->recommended_relay_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 Microdesc=1-2 Relay=2");
+    tor_strdup(DIRVOTE_RECOMMEND_RELAY_PROTO);
   v3_out->recommended_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 Microdesc=1-2 Relay=2");
-  v3_out->required_client_protocols =
-    tor_strdup("Cons=1-2 Desc=1-2 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=4 Microdesc=1-2 Relay=2");
+    tor_strdup(DIRVOTE_RECOMMEND_CLIENT_PROTO);
+
   v3_out->required_relay_protocols =
-    tor_strdup("Cons=1 Desc=1 DirCache=1 HSDir=1 HSIntro=3 HSRend=1 "
-               "Link=3-4 Microdesc=1 Relay=1-2");
+    tor_strdup(DIRVOTE_REQUIRE_RELAY_PROTO);
+  v3_out->required_client_protocols =
+    tor_strdup(DIRVOTE_REQUIRE_CLIENT_PROTO);
 
   /* We are not allowed to vote to require anything we don't have. */
   tor_assert(protover_all_supported(v3_out->required_relay_protocols, NULL));
@@ -4601,18 +4698,9 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
   tor_assert_nonfatal(protover_all_supported(
                                v3_out->recommended_client_protocols, NULL));
 
-  v3_out->package_lines = smartlist_new();
-  {
-    config_line_t *cl;
-    for (cl = get_options()->RecommendedPackages; cl; cl = cl->next) {
-      if (validate_recommended_package_line(cl->value))
-        smartlist_add_strdup(v3_out->package_lines, cl->value);
-    }
-  }
-
   v3_out->known_flags = smartlist_new();
   smartlist_split_string(v3_out->known_flags,
-                         "Authority Exit Fast Guard Stable V2Dir Valid HSDir",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          0, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
   if (vote_on_reachability)
     smartlist_add_strdup(v3_out->known_flags, "Running");
@@ -4620,13 +4708,17 @@ dirserv_generate_networkstatus_vote_obj(crypto_pk_t *private_key,
     smartlist_add_strdup(v3_out->known_flags, "BadExit");
   smartlist_sort_strings(v3_out->known_flags);
 
-  if (options->ConsensusParams) {
+  if (d_options->ConsensusParams) {
+    config_line_t *paramline = d_options->ConsensusParams;
     v3_out->net_params = smartlist_new();
-    smartlist_split_string(v3_out->net_params,
-                           options->ConsensusParams, NULL, 0, 0);
+    for ( ; paramline; paramline = paramline->next) {
+      smartlist_split_string(v3_out->net_params,
+                             paramline->value, NULL, 0, 0);
+    }
     smartlist_sort_strings(v3_out->net_params);
   }
   v3_out->bw_file_headers = bw_file_headers;
+  memcpy(v3_out->bw_file_digest256, bw_file_digest256, DIGEST256_LEN);
 
   voter = tor_malloc_zero(sizeof(networkstatus_voter_info_t));
   voter->nickname = tor_strdup(options->Nickname);
diff --git a/src/feature/dirauth/dirvote.h b/src/feature/dirauth/dirvote.h
index 02d88d19d1..a9b356b387 100644
--- a/src/feature/dirauth/dirvote.h
+++ b/src/feature/dirauth/dirvote.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,35 +49,17 @@
 #define MIN_VOTE_INTERVAL_TESTING_INITIAL \
                 ((MIN_VOTE_SECONDS_TESTING)+(MIN_DIST_SECONDS_TESTING)+1)
 
-/* A placeholder for routerstatus_format_entry() when the consensus method
- * argument is not applicable. */
-#define ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD 0
-
 /** The lowest consensus method that we currently support. */
-#define MIN_SUPPORTED_CONSENSUS_METHOD 25
+#define MIN_SUPPORTED_CONSENSUS_METHOD 28
 
 /** The highest consensus method that we currently support. */
-#define MAX_SUPPORTED_CONSENSUS_METHOD 28
-
-/** Lowest consensus method where authorities vote on required/recommended
- * protocols. */
-#define MIN_METHOD_FOR_RECOMMENDED_PROTOCOLS 25
-
-/** Lowest consensus method where authorities add protocols to routerstatus
- * entries. */
-#define MIN_METHOD_FOR_RS_PROTOCOLS 25
-
-/** Lowest consensus method where authorities initialize bandwidth weights to 1
- * instead of 0. See #14881 */
-#define MIN_METHOD_FOR_INIT_BW_WEIGHTS_ONE 26
+#define MAX_SUPPORTED_CONSENSUS_METHOD 29
 
-/** Lowest consensus method where the microdesc consensus contains relay IPv6
- * addresses. See #23826 and #20916. */
-#define MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS 27
-
-/** Lowest consensus method where microdescriptors do not contain relay IPv6
- * addresses. See #23828 and #20916. */
-#define MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC 28
+/**
+ * Lowest consensus method where microdescriptor lines are put in canonical
+ * form for improved compressibility and ease of storage. See proposal 298.
+ **/
+#define MIN_METHOD_FOR_CANONICAL_FAMILIES_IN_MICRODESCS 29
 
 /** Default bandwidth to clip unmeasured bandwidths to using method >=
  * MIN_METHOD_TO_CLIP_UNMEASURED_BW.  (This is not a consensus method; do not
@@ -92,6 +74,9 @@
 /** Maximum size of a line in a vote. */
 #define MAX_BW_FILE_HEADERS_LINE_LEN 1024
 
+extern const char DIRVOTE_UNIVERSAL_FLAGS[];
+extern const char DIRVOTE_OPTIONAL_FLAGS[];
+
 /*
  * Public API. Used outside of the dirauth subsystem.
  *
@@ -109,6 +94,7 @@ void dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
 
 /* Storing signatures and votes functions */
 struct pending_vote_t * dirvote_add_vote(const char *vote_body,
+                                         time_t time_posted,
                                          const char **msg_out,
                                          int *status_out);
 int dirvote_add_signatures(const char *detached_signatures_body,
@@ -119,7 +105,7 @@ struct config_line_t;
 char *format_recommended_version_list(const struct config_line_t *line,
                                       int warn);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline time_t
 dirvote_act(const or_options_t *options, time_t now)
@@ -157,9 +143,13 @@ dirvote_dirreq_get_status_vote(const char *url, smartlist_t *items,
 }
 
 static inline struct pending_vote_t *
-dirvote_add_vote(const char *vote_body, const char **msg_out, int *status_out)
+dirvote_add_vote(const char *vote_body,
+                 time_t time_posted,
+                 const char **msg_out,
+                 int *status_out)
 {
   (void) vote_body;
+  (void) time_posted;
   /* If the dirauth module is disabled, this should NEVER be called else we
    * failed to safeguard the dirauth module. */
   tor_assert_nonfatal_unreached();
@@ -177,14 +167,14 @@ dirvote_add_signatures(const char *detached_signatures_body,
 {
   (void) detached_signatures_body;
   (void) source;
-  (void) msg_out;
+  *msg_out = "No directory authority support";
   /* If the dirauth module is disabled, this should NEVER be called else we
    * failed to safeguard the dirauth module. */
   tor_assert_nonfatal_unreached();
   return 0;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Item access */
 MOCK_DECL(const char*, dirvote_get_pending_consensus,
@@ -245,6 +235,64 @@ char *networkstatus_get_detached_signatures(smartlist_t *consensuses);
 STATIC microdesc_t *dirvote_create_microdescriptor(const routerinfo_t *ri,
                                                    int consensus_method);
 
+/** The recommended relay protocols for this authority's votes.
+ * Recommending a new protocol causes old tor versions to log a warning.
+ */
+#define DIRVOTE_RECOMMEND_RELAY_PROTO           \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "DirCache=2 "                                 \
+  "HSDir=2 "                                    \
+  "HSIntro=4 "                                  \
+  "HSRend=2 "                                   \
+  "Link=4-5 "                                   \
+  "LinkAuth=3 "                                 \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
+/** The recommended client protocols for this authority's votes.
+ * Recommending a new protocol causes old tor versions to log a warning.
+ */
+#define DIRVOTE_RECOMMEND_CLIENT_PROTO          \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "DirCache=2 "                                 \
+  "HSDir=2 "                                    \
+  "HSIntro=4 "                                  \
+  "HSRend=2 "                                   \
+  "Link=4-5 "                                   \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
+/** The required relay protocols for this authority's votes.
+ * WARNING: Requiring a new protocol causes old tor versions to shut down.
+ *          Requiring the wrong protocols can break the tor network.
+ * See Proposal 303: When and how to remove support for protocol versions.
+ */
+#define DIRVOTE_REQUIRE_RELAY_PROTO             \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "DirCache=2 "                                 \
+  "HSDir=2 "                                    \
+  "HSIntro=4 "                                  \
+  "HSRend=2 "                                   \
+  "Link=4-5 "                                   \
+  "LinkAuth=3 "                                 \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
+/** The required relay protocols for this authority's votes.
+ * WARNING: Requiring a new protocol causes old tor versions to shut down.
+ *          Requiring the wrong protocols can break the tor network.
+ * See Proposal 303: When and how to remove support for protocol versions.
+ */
+#define DIRVOTE_REQUIRE_CLIENT_PROTO            \
+  "Cons=2 "                                     \
+  "Desc=2 "                                     \
+  "Link=4 "                                     \
+  "Microdesc=2 "                                \
+  "Relay=2"
+
 #endif /* defined(DIRVOTE_PRIVATE) */
 
 #endif /* !defined(TOR_DIRVOTE_H) */
diff --git a/src/feature/dirauth/dsigs_parse.c b/src/feature/dirauth/dsigs_parse.c
index d88176fee9..d0bb931814 100644
--- a/src/feature/dirauth/dsigs_parse.c
+++ b/src/feature/dirauth/dsigs_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -127,7 +127,7 @@ networkstatus_parse_detached_signatures(const char *s, const char *eos)
     }
     digests = detached_get_digests(sigs, flavor);
     tor_assert(digests);
-    if (!tor_mem_is_zero(digests->d[alg], digest_length)) {
+    if (!fast_mem_is_zero(digests->d[alg], digest_length)) {
       log_warn(LD_DIR, "Multiple digests for %s with %s on detached "
                "signatures document", flavor, algname);
       continue;
diff --git a/src/feature/dirauth/dsigs_parse.h b/src/feature/dirauth/dsigs_parse.h
index fec51ba488..b25e3e0b28 100644
--- a/src/feature/dirauth/dsigs_parse.h
+++ b/src/feature/dirauth/dsigs_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ void ns_detached_signatures_free_(ns_detached_signatures_t *s);
 #define ns_detached_signatures_free(s) \
   FREE_AND_NULL(ns_detached_signatures_t, ns_detached_signatures_free_, (s))
 
-#endif
+#endif /* !defined(TOR_DSIGS_PARSE_H) */
diff --git a/src/feature/dirauth/feature_dirauth.md b/src/feature/dirauth/feature_dirauth.md
new file mode 100644
index 0000000000..b152b94894
--- /dev/null
+++ b/src/feature/dirauth/feature_dirauth.md
@@ -0,0 +1,9 @@
+@dir /feature/dirauth
+@brief feature/dirauth: Directory authority implementation.
+
+This module handles running Tor as a directory authority.
+
+The directory protocol is specified in
+[dir-spec.txt](https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt).
+
+
diff --git a/src/feature/dirauth/guardfraction.c b/src/feature/dirauth/guardfraction.c
index d1a7f194d4..b84f804f5f 100644
--- a/src/feature/dirauth/guardfraction.c
+++ b/src/feature/dirauth/guardfraction.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -188,7 +188,7 @@ guardfraction_file_parse_inputs_line(const char *inputs_line,
  *
  *      guardfraction-file-version 1
  *      written-at <date and time>
- *      n-inputs <number of consesuses parsed> <number of days considered>
+ *      n-inputs <number of consensuses parsed> <number of days considered>
  *
  *      guard-seen <fpr 1> <guardfraction percentage> <consensus appearances>
  *      guard-seen <fpr 2> <guardfraction percentage> <consensus appearances>
diff --git a/src/feature/dirauth/guardfraction.h b/src/feature/dirauth/guardfraction.h
index 72404907a4..c10fd9b7bb 100644
--- a/src/feature/dirauth/guardfraction.h
+++ b/src/feature/dirauth/guardfraction.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,9 +16,9 @@
 STATIC int
 dirserv_read_guardfraction_file_from_str(const char *guardfraction_file_str,
                                       smartlist_t *vote_routerstatuses);
-#endif /* defined(DIRSERV_PRIVATE) */
+#endif
 
 int dirserv_read_guardfraction_file(const char *fname,
                                  smartlist_t *vote_routerstatuses);
 
-#endif
+#endif /* !defined(TOR_GUARDFRACTION_H) */
diff --git a/src/feature/dirauth/include.am b/src/feature/dirauth/include.am
new file mode 100644
index 0000000000..e26f120d4e
--- /dev/null
+++ b/src/feature/dirauth/include.am
@@ -0,0 +1,54 @@
+
+# The Directory Authority module.
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+MODULE_DIRAUTH_SOURCES =					\
+	src/feature/dirauth/authmode.c				\
+	src/feature/dirauth/bridgeauth.c			\
+	src/feature/dirauth/bwauth.c				\
+	src/feature/dirauth/dirauth_config.c			\
+	src/feature/dirauth/dirauth_periodic.c			\
+	src/feature/dirauth/dirauth_sys.c			\
+	src/feature/dirauth/dircollate.c			\
+	src/feature/dirauth/dirvote.c				\
+	src/feature/dirauth/dsigs_parse.c			\
+	src/feature/dirauth/guardfraction.c			\
+	src/feature/dirauth/keypin.c				\
+	src/feature/dirauth/process_descs.c			\
+	src/feature/dirauth/reachability.c			\
+	src/feature/dirauth/recommend_pkg.c			\
+	src/feature/dirauth/shared_random.c			\
+	src/feature/dirauth/shared_random_state.c		\
+	src/feature/dirauth/voteflags.c				\
+	src/feature/dirauth/voting_schedule.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dirauth/authmode.h			\
+	src/feature/dirauth/bridgeauth.h		\
+	src/feature/dirauth/bwauth.h			\
+	src/feature/dirauth/dirauth_config.h		\
+	src/feature/dirauth/dirauth_options.inc		\
+	src/feature/dirauth/dirauth_options_st.h	\
+	src/feature/dirauth/dirauth_periodic.h		\
+	src/feature/dirauth/dirauth_sys.h		\
+	src/feature/dirauth/dircollate.h		\
+	src/feature/dirauth/dirvote.h			\
+	src/feature/dirauth/dsigs_parse.h		\
+	src/feature/dirauth/guardfraction.h		\
+	src/feature/dirauth/keypin.h			\
+	src/feature/dirauth/ns_detached_signatures_st.h	\
+	src/feature/dirauth/reachability.h		\
+	src/feature/dirauth/recommend_pkg.h		\
+	src/feature/dirauth/process_descs.h		\
+	src/feature/dirauth/shared_random.h		\
+	src/feature/dirauth/shared_random_state.h	\
+	src/feature/dirauth/vote_microdesc_hash_st.h	\
+	src/feature/dirauth/voteflags.h			\
+	src/feature/dirauth/voting_schedule.h
+
+if BUILD_MODULE_DIRAUTH
+LIBTOR_APP_A_SOURCES += $(MODULE_DIRAUTH_SOURCES)
+else
+LIBTOR_APP_A_STUB_SOURCES += src/feature/dirauth/dirauth_stub.c
+endif
diff --git a/src/feature/dirauth/keypin.c b/src/feature/dirauth/keypin.c
index 06cb9ba1ff..5072a58573 100644
--- a/src/feature/dirauth/keypin.c
+++ b/src/feature/dirauth/keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,8 +15,6 @@
 #include "lib/cc/torint.h"
 #include "lib/crypt_ops/crypto_digest.h"
 #include "lib/crypt_ops/crypto_format.h"
-#include "lib/crypt_ops/crypto_format.h"
-#include "lib/ctime/di_ops.h"
 #include "lib/ctime/di_ops.h"
 #include "lib/encoding/binascii.h"
 #include "lib/encoding/time_fmt.h"
@@ -120,14 +118,14 @@ return (unsigned) siphash24g(a->ed25519_key, sizeof(a->ed25519_key));
 }
 
 HT_PROTOTYPE(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa,
-               keypin_ents_eq_rsa)
+             keypin_ents_eq_rsa);
 HT_GENERATE2(rsamap, keypin_ent_st, rsamap_node, keypin_ent_hash_rsa,
-               keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_)
+             keypin_ents_eq_rsa, 0.6, tor_reallocarray, tor_free_);
 
 HT_PROTOTYPE(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed,
-               keypin_ents_eq_ed)
+             keypin_ents_eq_ed);
 HT_GENERATE2(edmap, keypin_ent_st, edmap_node, keypin_ent_hash_ed,
-               keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_)
+             keypin_ents_eq_ed, 0.6, tor_reallocarray, tor_free_);
 
 /**
  * Check whether we already have an entry in the key pinning table for a
@@ -438,7 +436,7 @@ keypin_load_journal_impl(const char *data, size_t size)
   tor_log(severity, LD_DIRSERV,
           "Loaded %d entries from keypin journal. "
           "Found %d corrupt lines (ignored), %d duplicates (harmless), "
-          "and %d conflicts (resolved in favor or more recent entry).",
+          "and %d conflicts (resolved in favor of more recent entry).",
           n_entries, n_corrupt_lines, n_duplicates, n_conflicts);
 
   return 0;
diff --git a/src/feature/dirauth/keypin.h b/src/feature/dirauth/keypin.h
index 722b6ca5fc..881f010f0e 100644
--- a/src/feature/dirauth/keypin.h
+++ b/src/feature/dirauth/keypin.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file keypin.h
+ * @brief Header for keypin.c
+ **/
+
 #ifndef TOR_KEYPIN_H
 #define TOR_KEYPIN_H
 
@@ -11,10 +16,25 @@ int keypin_check_and_add(const uint8_t *rsa_id_digest,
                          const int replace_existing_entry);
 int keypin_check(const uint8_t *rsa_id_digest,
                  const uint8_t *ed25519_id_key);
+int keypin_close_journal(void);
 
+#ifdef HAVE_MODULE_DIRAUTH
 int keypin_open_journal(const char *fname);
-int keypin_close_journal(void);
 int keypin_load_journal(const char *fname);
+#else
+static inline int
+keypin_open_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+static inline int
+keypin_load_journal(const char *fname)
+{
+  (void)fname;
+  return 0;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 void keypin_clear(void);
 int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
 
@@ -25,6 +45,8 @@ int keypin_check_lone_rsa(const uint8_t *rsa_id_digest);
 
 #ifdef KEYPIN_PRIVATE
 
+#include "ext/ht.h"
+
 /**
  * In-memory representation of a key-pinning table entry.
  */
@@ -44,4 +66,3 @@ MOCK_DECL(STATIC void, keypin_add_entry_to_map, (keypin_ent_t *ent));
 #endif /* defined(KEYPIN_PRIVATE) */
 
 #endif /* !defined(TOR_KEYPIN_H) */
-
diff --git a/src/feature/dirauth/ns_detached_signatures_st.h b/src/feature/dirauth/ns_detached_signatures_st.h
index 0f92be2f0d..f409431ec1 100644
--- a/src/feature/dirauth/ns_detached_signatures_st.h
+++ b/src/feature/dirauth/ns_detached_signatures_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file ns_detached_signatures_st.h
+ * @brief Detached consensus signatures structure.
+ **/
+
 #ifndef NS_DETACHED_SIGNATURES_ST_H
 #define NS_DETACHED_SIGNATURES_ST_H
 
@@ -18,5 +23,4 @@ struct ns_detached_signatures_t {
                          * document_signature_t */
 };
 
-#endif
-
+#endif /* !defined(NS_DETACHED_SIGNATURES_ST_H) */
diff --git a/src/feature/dirauth/process_descs.c b/src/feature/dirauth/process_descs.c
index 21b8e239ec..5025d0ae39 100644
--- a/src/feature/dirauth/process_descs.c
+++ b/src/feature/dirauth/process_descs.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,17 +12,21 @@
  * them make those decisions.
  **/
 
+#define PROCESS_DESCS_PRIVATE
+
 #include "core/or/or.h"
 #include "feature/dirauth/process_descs.h"
 
 #include "app/config/config.h"
 #include "core/or/policies.h"
 #include "core/or/versions.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/dirauth/keypin.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
+#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo.h"
@@ -32,46 +36,28 @@
 #include "feature/relay/router.h"
 
 #include "core/or/tor_version_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/extrainfo_st.h"
 #include "feature/nodelist/node_st.h"
+#include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/vote_routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
+#include "lib/crypt_ops/crypto_format.h"
 
 /** How far in the future do we allow a router to get? (seconds) */
 #define ROUTER_ALLOW_SKEW (60*60*12)
 
 static void directory_remove_invalid(void);
-struct authdir_config_t;
 static was_router_added_t dirserv_add_extrainfo(extrainfo_t *ei,
                                                 const char **msg);
 static uint32_t
-dirserv_get_status_impl(const char *fp, const char *nickname,
-                        uint32_t addr, uint16_t or_port,
-                        const char *platform, const char **msg,
-                        int severity);
-
-/*                 1  Historically used to indicate Named */
-#define FP_INVALID 2  /**< Believed invalid. */
-#define FP_REJECT  4  /**< We will not publish this router. */
-/*                 8  Historically used to avoid using this as a dir. */
-#define FP_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
-/*                 32 Historically used to indicade Unnamed */
-
-/** Target of status_by_digest map. */
-typedef uint32_t router_status_t;
-
-static void add_fingerprint_to_dir(const char *fp,
-                                   struct authdir_config_t *list,
-                                   router_status_t add_status);
-
-/** List of nickname-\>identity fingerprint mappings for all the routers
- * that we name.  Used to prevent router impersonation. */
-typedef struct authdir_config_t {
-  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
-  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
-} authdir_config_t;
+dirserv_get_status_impl(const char *id_digest,
+                        const ed25519_public_key_t *ed25519_public_key,
+                        const char *nickname, uint32_t addr, uint16_t or_port,
+                        const char *platform, const char **msg, int severity);
 
 /** Should be static; exposed for testing. */
 static authdir_config_t *fingerprint_list = NULL;
@@ -83,20 +69,39 @@ authdir_config_new(void)
   authdir_config_t *list = tor_malloc_zero(sizeof(authdir_config_t));
   list->fp_by_name = strmap_new();
   list->status_by_digest = digestmap_new();
+  list->status_by_digest256 = digest256map_new();
   return list;
 }
 
+#ifdef TOR_UNIT_TESTS
+
+/** Initialize fingerprint_list to a new authdir_config_t. Used for tests. */
+void
+authdir_init_fingerprint_list(void)
+{
+  fingerprint_list = authdir_config_new();
+}
+
+/* Return the current fingerprint_list. Used for tests. */
+authdir_config_t *
+authdir_return_fingerprint_list(void)
+{
+  return fingerprint_list;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Add the fingerprint <b>fp</b> to the smartlist of fingerprint_entry_t's
  * <b>list</b>, or-ing the currently set status flags with
  * <b>add_status</b>.
  */
-/* static */ void
-add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
-                       router_status_t add_status)
+int
+add_rsa_fingerprint_to_dir(const char *fp, authdir_config_t *list,
+                           rtr_flags_t add_status)
 {
   char *fingerprint;
   char d[DIGEST_LEN];
-  router_status_t *status;
+  rtr_flags_t *status;
   tor_assert(fp);
   tor_assert(list);
 
@@ -107,24 +112,52 @@ add_fingerprint_to_dir(const char *fp, authdir_config_t *list,
     log_warn(LD_DIRSERV, "Couldn't decode fingerprint \"%s\"",
              escaped(fp));
     tor_free(fingerprint);
-    return;
+    return -1;
   }
 
   status = digestmap_get(list->status_by_digest, d);
   if (!status) {
-    status = tor_malloc_zero(sizeof(router_status_t));
+    status = tor_malloc_zero(sizeof(rtr_flags_t));
     digestmap_set(list->status_by_digest, d, status);
   }
 
   tor_free(fingerprint);
   *status |= add_status;
-  return;
+  return 0;
+}
+
+/** Add the ed25519 key <b>edkey</b> to the smartlist of fingerprint_entry_t's
+ * <b>list</b>, or-ing the currently set status flags with <b>add_status</b>.
+ * Return -1 if we were unable to decode the key, else return 0.
+ */
+int
+add_ed25519_to_dir(const ed25519_public_key_t *edkey, authdir_config_t *list,
+                   rtr_flags_t add_status)
+{
+  rtr_flags_t *status;
+
+  tor_assert(edkey);
+  tor_assert(list);
+
+  if (ed25519_validate_pubkey(edkey) < 0) {
+    log_warn(LD_DIRSERV, "Invalid ed25519 key \"%s\"", ed25519_fmt(edkey));
+    return -1;
+  }
+
+  status = digest256map_get(list->status_by_digest256, edkey->pubkey);
+  if (!status) {
+    status = tor_malloc_zero(sizeof(rtr_flags_t));
+    digest256map_set(list->status_by_digest256, edkey->pubkey, status);
+  }
+
+  *status |= add_status;
+  return 0;
 }
 
 /** Add the fingerprint for this OR to the global list of recognized
  * identity key fingerprints. */
 int
-dirserv_add_own_fingerprint(crypto_pk_t *pk)
+dirserv_add_own_fingerprint(crypto_pk_t *pk, const ed25519_public_key_t *edkey)
 {
   char fp[FINGERPRINT_LEN+1];
   if (crypto_pk_get_fingerprint(pk, fp, 0)<0) {
@@ -133,7 +166,14 @@ dirserv_add_own_fingerprint(crypto_pk_t *pk)
   }
   if (!fingerprint_list)
     fingerprint_list = authdir_config_new();
-  add_fingerprint_to_dir(fp, fingerprint_list, 0);
+  if (add_rsa_fingerprint_to_dir(fp, fingerprint_list, 0) < 0) {
+    log_err(LD_BUG, "Error adding RSA fingerprint");
+    return -1;
+  }
+  if (add_ed25519_to_dir(edkey, fingerprint_list, 0) < 0) {
+    log_err(LD_BUG, "Error adding ed25519 key");
+    return -1;
+  }
   return 0;
 }
 
@@ -174,27 +214,46 @@ dirserv_load_fingerprint_file(void)
   fingerprint_list_new = authdir_config_new();
 
   for (list=front; list; list=list->next) {
-    char digest_tmp[DIGEST_LEN];
-    router_status_t add_status = 0;
+    rtr_flags_t add_status = 0;
     nickname = list->key; fingerprint = list->value;
     tor_strstrip(fingerprint, " "); /* remove spaces */
-    if (strlen(fingerprint) != HEX_DIGEST_LEN ||
-        base16_decode(digest_tmp, sizeof(digest_tmp),
-                      fingerprint, HEX_DIGEST_LEN) != sizeof(digest_tmp)) {
-      log_notice(LD_CONFIG,
-                 "Invalid fingerprint (nickname '%s', "
-                 "fingerprint %s). Skipping.",
-                 nickname, fingerprint);
-      continue;
-    }
+
+   /* Determine what we should do with the relay with the nickname field. */
     if (!strcasecmp(nickname, "!reject")) {
-        add_status = FP_REJECT;
+        add_status = RTR_REJECT;
     } else if (!strcasecmp(nickname, "!badexit")) {
-        add_status = FP_BADEXIT;
+        add_status = RTR_BADEXIT;
     } else if (!strcasecmp(nickname, "!invalid")) {
-        add_status = FP_INVALID;
+        add_status = RTR_INVALID;
+    }
+
+    /* Check if fingerprint is RSA or ed25519 by verifying it. */
+    int ed25519_not_ok = -1, rsa_not_ok = -1;
+
+    /* Attempt to add the RSA key. */
+    if (strlen(fingerprint) == HEX_DIGEST_LEN) {
+      rsa_not_ok = add_rsa_fingerprint_to_dir(fingerprint,
+                                              fingerprint_list_new,
+                                              add_status);
+    }
+
+    /* Check ed25519 key. We check the size to prevent buffer overflows.
+     * If valid, attempt to add it, */
+    ed25519_public_key_t ed25519_pubkey_tmp;
+    if (strlen(fingerprint) == BASE64_DIGEST256_LEN) {
+      if (!digest256_from_base64((char *) ed25519_pubkey_tmp.pubkey,
+                                 fingerprint)) {
+        ed25519_not_ok = add_ed25519_to_dir(&ed25519_pubkey_tmp,
+                                            fingerprint_list_new, add_status);
+      }
+    }
+
+    /* If both keys are invalid (or missing), log and skip. */
+    if (ed25519_not_ok && rsa_not_ok) {
+      log_warn(LD_CONFIG, "Invalid fingerprint (nickname '%s', "
+               "fingerprint %s). Skipping.", nickname, fingerprint);
+      continue;
     }
-    add_fingerprint_to_dir(fingerprint, fingerprint_list_new, add_status);
   }
 
   config_free_lines(front);
@@ -216,30 +275,42 @@ dirserv_load_fingerprint_file(void)
 
 #define DISABLE_DISABLING_ED25519
 
-/** Check whether <b>router</b> has a nickname/identity key combination that
- * we recognize from the fingerprint list, or an IP we automatically act on
- * according to our configuration.  Return the appropriate router status.
+/** Check whether <b>router</b> has:
+ * - a nickname/identity key combination that we recognize from the fingerprint
+ *   list,
+ * - an IP we automatically act on according to our configuration,
+ * - an appropriate version, and
+ * - matching pinned keys.
+ *
+ * Return the appropriate router status.
  *
- * If the status is 'FP_REJECT' and <b>msg</b> is provided, set
- * *<b>msg</b> to an explanation of why. */
+ * If the status is 'RTR_REJECT' and <b>msg</b> is provided, set
+ * *<b>msg</b> to a string constant explaining why. */
 uint32_t
 dirserv_router_get_status(const routerinfo_t *router, const char **msg,
                           int severity)
 {
   char d[DIGEST_LEN];
-  const int key_pinning = get_options()->AuthDirPinKeys;
+  const int key_pinning = dirauth_get_options()->AuthDirPinKeys;
+  uint32_t r;
+  ed25519_public_key_t *signing_key = NULL;
 
   if (crypto_pk_get_digest(router->identity_pkey, d)) {
     log_warn(LD_BUG,"Error computing fingerprint");
     if (msg)
       *msg = "Bug: Error computing fingerprint";
-    return FP_REJECT;
+    return RTR_REJECT;
   }
 
-  /* Check for the more usual versions to reject a router first. */
-  const uint32_t r = dirserv_get_status_impl(d, router->nickname,
-                                             router->addr, router->or_port,
-                                             router->platform, msg, severity);
+  /* First, check for the more common reasons to reject a router. */
+  if (router->cache_info.signing_key_cert) {
+    /* This has an ed25519 identity key. */
+    signing_key = &router->cache_info.signing_key_cert->signing_key;
+  }
+  r = dirserv_get_status_impl(d, signing_key, router->nickname, router->addr,
+                              router->or_port, router->platform, msg,
+                              severity);
+
   if (r)
     return r;
 
@@ -254,7 +325,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
            "key.", router_describe(router));
     if (msg)
       *msg = "Missing ntor curve25519 onion key. Please upgrade!";
-    return FP_REJECT;
+    return RTR_REJECT;
   }
 
   if (router->cache_info.signing_key_cert) {
@@ -270,7 +341,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
         if (msg) {
           *msg = "Ed25519 identity key or RSA identity key has changed.";
         }
-        return FP_REJECT;
+        return RTR_REJECT;
       }
     }
   } else {
@@ -287,7 +358,7 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
         if (msg) {
           *msg = "Ed25519 identity key has disappeared.";
         }
-        return FP_REJECT;
+        return RTR_REJECT;
       }
 #endif /* defined(DISABLE_DISABLING_ED25519) */
     }
@@ -299,31 +370,74 @@ dirserv_router_get_status(const routerinfo_t *router, const char **msg,
 /** Return true if there is no point in downloading the router described by
  * <b>rs</b> because this directory would reject it. */
 int
-dirserv_would_reject_router(const routerstatus_t *rs)
+dirserv_would_reject_router(const routerstatus_t *rs,
+                            const vote_routerstatus_t *vrs)
 {
   uint32_t res;
+  struct ed25519_public_key_t pk;
+  memcpy(&pk.pubkey, vrs->ed25519_id, ED25519_PUBKEY_LEN);
 
-  res = dirserv_get_status_impl(rs->identity_digest, rs->nickname,
-                                rs->addr, rs->or_port,
-                                NULL, NULL, LOG_DEBUG);
+  res = dirserv_get_status_impl(rs->identity_digest, &pk, rs->nickname,
+                                rs->addr, rs->or_port, NULL, NULL, LOG_DEBUG);
 
-  return (res & FP_REJECT) != 0;
+  return (res & RTR_REJECT) != 0;
+}
+
+/**
+ * Check whether the platform string in <b>platform</b> describes a platform
+ * that, as a directory authority, we want to reject.  If it does, return
+ * true, and set *<b>msg</b> (if present) to a rejection message.  Otherwise
+ * return false.
+ */
+STATIC bool
+dirserv_rejects_tor_version(const char *platform,
+                            const char **msg)
+{
+  if (!platform)
+    return false;
+
+  static const char please_upgrade_string[] =
+    "Tor version is insecure or unsupported. Please upgrade!";
+
+  /* Versions before Tor 0.3.5 are unsupported.
+   *
+   * Also, reject unstable versions of 0.3.5, since (as of this writing)
+   * they are almost none of the network. */
+  if (!tor_version_as_new_as(platform,"0.3.5.7")) {
+    if (msg)
+      *msg = please_upgrade_string;
+    return true;
+  }
+
+  /* Series between Tor 0.3.6 and 0.4.1.4-rc inclusive are unsupported.
+   * Reject them. 0.3.6.0-alpha-dev only existed for a short time, before
+   * it was renamed to 0.4.0.0-alpha-dev. */
+  if (tor_version_as_new_as(platform,"0.3.6.0-alpha-dev") &&
+      !tor_version_as_new_as(platform,"0.4.1.5")) {
+    if (msg) {
+      *msg = please_upgrade_string;
+    }
+    return true;
+  }
+
+  return false;
 }
 
 /** Helper: As dirserv_router_get_status, but takes the router fingerprint
- * (hex, no spaces), nickname, address (used for logging only), IP address, OR
- * port and platform (logging only) as arguments.
+ * (hex, no spaces), ed25519 key, nickname, address (used for logging only),
+ * IP address, OR port and platform (logging only) as arguments.
  *
  * Log messages at 'severity'. (There's not much point in
  * logging that we're rejecting servers we'll not download.)
  */
 static uint32_t
-dirserv_get_status_impl(const char *id_digest, const char *nickname,
-                        uint32_t addr, uint16_t or_port,
+dirserv_get_status_impl(const char *id_digest,
+                        const ed25519_public_key_t *ed25519_public_key,
+                        const char *nickname, uint32_t addr, uint16_t or_port,
                         const char *platform, const char **msg, int severity)
 {
   uint32_t result = 0;
-  router_status_t *status_by_digest;
+  rtr_flags_t *status_by_digest;
 
   if (!fingerprint_list)
     fingerprint_list = authdir_config_new();
@@ -338,27 +452,13 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
       if (msg) {
         *msg = "Malformed platform string.";
       }
-      return FP_REJECT;
+      return RTR_REJECT;
     }
   }
 
-  /* Versions before Tor 0.2.4.18-rc are too old to support, and are
-   * missing some important security fixes too. Disable them. */
-  if (platform && !tor_version_as_new_as(platform,"0.2.4.18-rc")) {
-    if (msg)
-      *msg = "Tor version is insecure or unsupported. Please upgrade!";
-    return FP_REJECT;
-  }
-
-  /* Tor 0.2.9.x where x<5 suffers from bug #20499, where relays don't
-   * keep their consensus up to date so they make bad guards.
-   * The simple fix is to just drop them from the network. */
-  if (platform &&
-      tor_version_as_new_as(platform,"0.2.9.0-alpha") &&
-      !tor_version_as_new_as(platform,"0.2.9.5-alpha")) {
-    if (msg)
-      *msg = "Tor version contains bug 20499. Please upgrade!";
-    return FP_REJECT;
+  /* Check whether the version is obsolete, broken, insecure, etc... */
+  if (platform && dirserv_rejects_tor_version(platform, msg)) {
+    return RTR_REJECT;
   }
 
   status_by_digest = digestmap_get(fingerprint_list->status_by_digest,
@@ -366,23 +466,30 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
   if (status_by_digest)
     result |= *status_by_digest;
 
-  if (result & FP_REJECT) {
+  if (ed25519_public_key) {
+    status_by_digest = digest256map_get(fingerprint_list->status_by_digest256,
+                                        ed25519_public_key->pubkey);
+    if (status_by_digest)
+      result |= *status_by_digest;
+  }
+
+  if (result & RTR_REJECT) {
     if (msg)
-      *msg = "Fingerprint is marked rejected -- if you think this is a "
-             "mistake please set a valid email address in ContactInfo and "
-             "send an email to bad-relays@lists.torproject.org mentioning "
-             "your fingerprint(s)?";
-    return FP_REJECT;
-  } else if (result & FP_INVALID) {
+      *msg = "Fingerprint and/or ed25519 identity is marked rejected -- if "
+             "you think this is a mistake please set a valid email address "
+             "in ContactInfo and send an email to "
+             "bad-relays@lists.torproject.org mentioning your fingerprint(s)?";
+    return RTR_REJECT;
+  } else if (result & RTR_INVALID) {
     if (msg)
-      *msg = "Fingerprint is marked invalid";
+      *msg = "Fingerprint and/or ed25519 identity is marked invalid";
   }
 
   if (authdir_policy_badexit_address(addr, or_port)) {
     log_fn(severity, LD_DIRSERV,
            "Marking '%s' as bad exit because of address '%s'",
                nickname, fmt_addr32(addr));
-    result |= FP_BADEXIT;
+    result |= RTR_BADEXIT;
   }
 
   if (!authdir_policy_permits_address(addr, or_port)) {
@@ -393,13 +500,13 @@ dirserv_get_status_impl(const char *id_digest, const char *nickname,
              "mistake please set a valid email address in ContactInfo and "
              "send an email to bad-relays@lists.torproject.org mentioning "
              "your address(es) and fingerprint(s)?";
-    return FP_REJECT;
+    return RTR_REJECT;
   }
   if (!authdir_policy_valid_address(addr, or_port)) {
     log_fn(severity, LD_DIRSERV,
            "Not marking '%s' valid because of address '%s'",
                nickname, fmt_addr32(addr));
-    result |= FP_INVALID;
+    result |= RTR_INVALID;
   }
 
   return result;
@@ -414,6 +521,7 @@ dirserv_free_fingerprint_list(void)
 
   strmap_free(fingerprint_list->fp_by_name, tor_free_);
   digestmap_free(fingerprint_list->status_by_digest, tor_free_);
+  digest256map_free(fingerprint_list->status_by_digest256, tor_free_);
   tor_free(fingerprint_list);
 }
 
@@ -423,27 +531,40 @@ dirserv_free_fingerprint_list(void)
 
 /** Return -1 if <b>ri</b> has a private or otherwise bad address,
  * unless we're configured to not care. Return 0 if all ok. */
-static int
+STATIC int
 dirserv_router_has_valid_address(routerinfo_t *ri)
 {
   tor_addr_t addr;
+
   if (get_options()->DirAllowPrivateAddresses)
     return 0; /* whatever it is, we're fine with it */
+
   tor_addr_from_ipv4h(&addr, ri->addr);
+  if (tor_addr_is_null(&addr) || tor_addr_is_internal(&addr, 0)) {
+    log_info(LD_DIRSERV,
+             "Router %s published internal IPv4 address. Refusing.",
+             router_describe(ri));
+    return -1; /* it's a private IP, we should reject it */
+  }
 
-  if (tor_addr_is_internal(&addr, 0)) {
+  /* We only check internal v6 on non-null addresses because we do not require
+   * IPv6 and null IPv6 is normal. */
+  if (!tor_addr_is_null(&ri->ipv6_addr) &&
+      tor_addr_is_internal(&ri->ipv6_addr, 0)) {
     log_info(LD_DIRSERV,
-             "Router %s published internal IP address. Refusing.",
+             "Router %s published internal IPv6 address. Refusing.",
              router_describe(ri));
     return -1; /* it's a private IP, we should reject it */
   }
+
   return 0;
 }
 
 /** Check whether we, as a directory server, want to accept <b>ri</b>.  If so,
  * set its is_valid,running fields and return 0.  Otherwise, return -1.
  *
- * If the router is rejected, set *<b>msg</b> to an explanation of why.
+ * If the router is rejected, set *<b>msg</b> to a string constant explining
+ * why.
  *
  * If <b>complain</b> then explain at log-level 'notice' why we refused
  * a descriptor; else explain at log-level 'info'.
@@ -457,7 +578,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
   int severity = (complain && ri->contact_info) ? LOG_NOTICE : LOG_INFO;
   uint32_t status = dirserv_router_get_status(ri, msg, severity);
   tor_assert(msg);
-  if (status & FP_REJECT)
+  if (status & RTR_REJECT)
     return -1; /* msg is already set. */
 
   /* Is there too much clock skew? */
@@ -493,7 +614,7 @@ authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
     return -1;
   }
 
-  *valid_out = ! (status & FP_INVALID);
+  *valid_out = ! (status & RTR_INVALID);
 
   return 0;
 }
@@ -505,8 +626,8 @@ void
 dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                  uint32_t authstatus)
 {
-  node->is_valid = (authstatus & FP_INVALID) ? 0 : 1;
-  node->is_bad_exit = (authstatus & FP_BADEXIT) ? 1 : 0;
+  node->is_valid = (authstatus & RTR_INVALID) ? 0 : 1;
+  node->is_bad_exit = (authstatus & RTR_BADEXIT) ? 1 : 0;
 }
 
 /** True iff <b>a</b> is more severe than <b>b</b>. */
@@ -519,7 +640,8 @@ WRA_MORE_SEVERE(was_router_added_t a, was_router_added_t b)
 /** As for dirserv_add_descriptor(), but accepts multiple documents, and
  * returns the most severe error that occurred for any one of them. */
 was_router_added_t
-dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
                                  const char *source,
                                  const char **msg)
 {
@@ -534,7 +656,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   int general = purpose == ROUTER_PURPOSE_GENERAL;
   tor_assert(msg);
 
-  r=ROUTER_ADDED_SUCCESSFULLY; /*Least severe return value. */
+  r=ROUTER_ADDED_SUCCESSFULLY; /* Least severe return value. */
+
+  if (!string_is_utf8_no_bom(desc, desclen)) {
+    *msg = "descriptor(s) or extrainfo(s) not valid UTF-8 or had BOM.";
+    return ROUTER_AUTHDIR_REJECTS;
+  }
 
   format_iso_time(time_buf, now);
   if (tor_snprintf(annotation_buf, sizeof(annotation_buf),
@@ -545,14 +672,12 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
                    !general ? router_purpose_to_string(purpose) : "",
                    !general ? "\n" : "")<0) {
     *msg = "Couldn't format annotations";
-    /* XXX Not cool: we return -1 below, but (was_router_added_t)-1 is
-     * ROUTER_BAD_EI, which isn't what's gone wrong here. :( */
-    return -1;
+    return ROUTER_AUTHDIR_BUG_ANNOTATIONS;
   }
 
   s = desc;
   list = smartlist_new();
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 0, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 0, 0,
                                      annotation_buf, NULL)) {
     SMARTLIST_FOREACH(list, routerinfo_t *, ri, {
         msg_out = NULL;
@@ -568,7 +693,7 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
   smartlist_clear(list);
 
   s = desc;
-  if (!router_parse_list_from_string(&s, NULL, list, SAVED_NOWHERE, 1, 0,
+  if (!router_parse_list_from_string(&s, s+desclen, list, SAVED_NOWHERE, 1, 0,
                                      NULL, NULL)) {
     SMARTLIST_FOREACH(list, extrainfo_t *, ei, {
         msg_out = NULL;
@@ -605,7 +730,8 @@ dirserv_add_multiple_descriptors(const char *desc, uint8_t purpose,
  * That means the caller must not access <b>ri</b> after this function
  * returns, since it might have been freed.
  *
- * Return the status of the operation.
+ * Return the status of the operation, and set *<b>msg</b> to a string
+ * constant describing the status.
  *
  * This function is only called when fresh descriptors are posted, not when
  * we re-load the cache.
@@ -618,7 +744,7 @@ dirserv_add_descriptor(routerinfo_t *ri, const char **msg, const char *source)
   char *desc, *nickname;
   const size_t desclen = ri->cache_info.signed_descriptor_len +
       ri->cache_info.annotations_len;
-  const int key_pinning = get_options()->AuthDirPinKeys;
+  const int key_pinning = dirauth_get_options()->AuthDirPinKeys;
   *msg = NULL;
 
   /* If it's too big, refuse it now. Otherwise we'll cache it all over the
@@ -816,21 +942,21 @@ directory_remove_invalid(void)
       continue;
     r = dirserv_router_get_status(ent, &msg, LOG_INFO);
     description = router_describe(ent);
-    if (r & FP_REJECT) {
+    if (r & RTR_REJECT) {
       log_info(LD_DIRSERV, "Router %s is now rejected: %s",
                description, msg?msg:"");
       routerlist_remove(rl, ent, 0, time(NULL));
       continue;
     }
-    if (bool_neq((r & FP_INVALID), !node->is_valid)) {
+    if (bool_neq((r & RTR_INVALID), !node->is_valid)) {
       log_info(LD_DIRSERV, "Router '%s' is now %svalid.", description,
-               (r&FP_INVALID) ? "in" : "");
-      node->is_valid = (r&FP_INVALID)?0:1;
+               (r&RTR_INVALID) ? "in" : "");
+      node->is_valid = (r&RTR_INVALID)?0:1;
     }
-    if (bool_neq((r & FP_BADEXIT), node->is_bad_exit)) {
+    if (bool_neq((r & RTR_BADEXIT), node->is_bad_exit)) {
       log_info(LD_DIRSERV, "Router '%s' is now a %s exit", description,
-               (r & FP_BADEXIT) ? "bad" : "good");
-      node->is_bad_exit = (r&FP_BADEXIT) ? 1: 0;
+               (r & RTR_BADEXIT) ? "bad" : "good");
+      node->is_bad_exit = (r&RTR_BADEXIT) ? 1: 0;
     }
   } SMARTLIST_FOREACH_END(node);
 
diff --git a/src/feature/dirauth/process_descs.h b/src/feature/dirauth/process_descs.h
index ae2d6ad25d..1461ab697d 100644
--- a/src/feature/dirauth/process_descs.h
+++ b/src/feature/dirauth/process_descs.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,27 +12,155 @@
 #ifndef TOR_RECV_UPLOADS_H
 #define TOR_RECV_UPLOADS_H
 
-int dirserv_load_fingerprint_file(void);
+// for was_router_added_t.
+#include "feature/nodelist/routerlist.h"
+
+#include "lib/crypt_ops/crypto_ed25519.h"
+
+struct authdir_config_t;
+
+/** Target of status_by_digest map. */
+typedef uint32_t rtr_flags_t;
+
+int add_rsa_fingerprint_to_dir(const char *fp, struct authdir_config_t *list,
+                               rtr_flags_t add_status);
+
+int add_ed25519_to_dir(const ed25519_public_key_t *edkey,
+                       struct authdir_config_t *list,
+                       rtr_flags_t add_status);
+
+/** List of nickname-\>identity fingerprint mappings for all the routers
+ * that we name.  Used to prevent router impersonation. */
+typedef struct authdir_config_t {
+  strmap_t *fp_by_name; /**< Map from lc nickname to fingerprint. */
+  digestmap_t *status_by_digest; /**< Map from digest to router_status_t. */
+  digest256map_t *status_by_digest256; /**< Map from digest256 to
+                                        * router_status_t. */
+} authdir_config_t;
+
+#if defined(PROCESS_DESCS_PRIVATE) || defined(TOR_UNIT_TESTS)
+
+/*                 1  Historically used to indicate Named */
+#define RTR_INVALID 2  /**< Believed invalid. */
+#define RTR_REJECT  4  /**< We will not publish this router. */
+/*                 8  Historically used to avoid using this as a dir. */
+#define RTR_BADEXIT 16 /**< We'll tell clients not to use this as an exit. */
+/*                 32 Historically used to indicade Unnamed */
+
+#endif /* defined(PROCESS_DESCS_PRIVATE) || defined(TOR_UNIT_TESTS) */
+
+#ifdef TOR_UNIT_TESTS
+
+void authdir_init_fingerprint_list(void);
+
+authdir_config_t *authdir_return_fingerprint_list(void);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
 void dirserv_free_fingerprint_list(void);
-int dirserv_add_own_fingerprint(crypto_pk_t *pk);
 
+#ifdef HAVE_MODULE_DIRAUTH
+int dirserv_load_fingerprint_file(void);
 enum was_router_added_t dirserv_add_multiple_descriptors(
-                                     const char *desc, uint8_t purpose,
+                                     const char *desc, size_t desclen,
+                                     uint8_t purpose,
                                      const char *source,
                                      const char **msg);
 enum was_router_added_t dirserv_add_descriptor(routerinfo_t *ri,
                                                const char **msg,
                                                const char *source);
 
+int dirserv_would_reject_router(const routerstatus_t *rs,
+                                const vote_routerstatus_t *vrs);
 int authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
                                    int complain,
                                    int *valid_out);
+int dirserv_add_own_fingerprint(crypto_pk_t *pk,
+                                const ed25519_public_key_t *edkey);
 uint32_t dirserv_router_get_status(const routerinfo_t *router,
                                    const char **msg,
                                    int severity);
 void dirserv_set_node_flags_from_authoritative_status(node_t *node,
                                                       uint32_t authstatus);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+static inline int
+dirserv_load_fingerprint_file(void)
+{
+  return 0;
+}
+static inline enum was_router_added_t
+dirserv_add_multiple_descriptors(const char *desc, size_t desclen,
+                                 uint8_t purpose,
+                                 const char *source,
+                                 const char **msg)
+{
+  (void)desc;
+  (void)desclen;
+  (void)purpose;
+  (void)source;
+  *msg = "No directory authority support";
+  return (enum was_router_added_t)0;
+}
+static inline enum was_router_added_t
+dirserv_add_descriptor(routerinfo_t *ri,
+                       const char **msg,
+                       const char *source)
+{
+  (void)ri;
+  (void)source;
+  *msg = "No directory authority support";
+  return (enum was_router_added_t)0;
+}
+static inline int
+dirserv_would_reject_router(const routerstatus_t *rs,
+                            const vote_routerstatus_t *vrs)
+{
+  (void)rs;
+  (void)vrs;
+  return 0;
+}
+static inline int
+authdir_wants_to_reject_router(routerinfo_t *ri, const char **msg,
+                               int complain,
+                               int *valid_out)
+{
+  (void)ri;
+  (void)complain;
+  *msg = "No directory authority support";
+  *valid_out = 0;
+  return 0;
+}
+static inline int
+dirserv_add_own_fingerprint(crypto_pk_t *pk, const ed25519_public_key_t *edkey)
+{
+  (void)pk;
+  (void)edkey;
+  return 0;
+}
+static inline uint32_t
+dirserv_router_get_status(const routerinfo_t *router,
+                          const char **msg,
+                          int severity)
+{
+  (void)router;
+  (void)severity;
+  if (msg)
+    *msg = "No directory authority support";
+  return 0;
+}
+static inline void
+dirserv_set_node_flags_from_authoritative_status(node_t *node,
+                                                 uint32_t authstatus)
+{
+  (void)node;
+  (void)authstatus;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-int dirserv_would_reject_router(const routerstatus_t *rs);
+#ifdef TOR_UNIT_TESTS
+STATIC int dirserv_router_has_valid_address(routerinfo_t *ri);
+STATIC bool dirserv_rejects_tor_version(const char *platform,
+                                        const char **msg);
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_RECV_UPLOADS_H) */
diff --git a/src/feature/dirauth/reachability.c b/src/feature/dirauth/reachability.c
index 883b692cbb..65fa27ed80 100644
--- a/src/feature/dirauth/reachability.c
+++ b/src/feature/dirauth/reachability.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,7 @@
 #include "core/or/channeltls.h"
 #include "core/or/command.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo.h"
@@ -24,6 +25,7 @@
 #include "feature/nodelist/torcert.h"
 #include "feature/stats/rephist.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerlist_st.h"
@@ -53,7 +55,7 @@ dirserv_orconn_tls_done(const tor_addr_t *addr,
 
   ri = node->ri;
 
-  if (get_options()->AuthDirTestEd25519LinkKeys &&
+  if (dirauth_get_options()->AuthDirTestEd25519LinkKeys &&
       node_supports_ed25519_link_authentication(node, 1) &&
       ri->cache_info.signing_key_cert) {
     /* We allow the node to have an ed25519 key if we haven't been told one in
@@ -125,7 +127,7 @@ dirserv_should_launch_reachability_test(const routerinfo_t *ri,
 void
 dirserv_single_reachability_test(time_t now, routerinfo_t *router)
 {
-  const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
   channel_t *chan = NULL;
   const node_t *node = NULL;
   tor_addr_t router_addr;
@@ -136,7 +138,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
   node = node_get_by_id(router->cache_info.identity_digest);
   tor_assert(node);
 
-  if (options->AuthDirTestEd25519LinkKeys &&
+  if (dirauth_options->AuthDirTestEd25519LinkKeys &&
       node_supports_ed25519_link_authentication(node, 1) &&
       router->cache_info.signing_key_cert) {
     ed_id_key = &router->cache_info.signing_key_cert->signing_key;
@@ -154,7 +156,7 @@ dirserv_single_reachability_test(time_t now, routerinfo_t *router)
   if (chan) command_setup_channel(chan);
 
   /* Possible IPv6. */
-  if (get_options()->AuthDirHasIPv6Connectivity == 1 &&
+  if (dirauth_get_options()->AuthDirHasIPv6Connectivity == 1 &&
       !tor_addr_is_null(&router->ipv6_addr)) {
     char addrstr[TOR_ADDR_BUF_LEN];
     log_debug(LD_OR, "Testing reachability of %s at %s:%u.",
diff --git a/src/feature/dirauth/reachability.h b/src/feature/dirauth/reachability.h
index 5a938673ff..19448a67f3 100644
--- a/src/feature/dirauth/reachability.h
+++ b/src/feature/dirauth/reachability.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,13 +24,27 @@
 #define REACHABILITY_TEST_CYCLE_PERIOD \
   (REACHABILITY_TEST_INTERVAL*REACHABILITY_MODULO_PER_TEST)
 
+#ifdef HAVE_MODULE_DIRAUTH
+void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
+void dirserv_test_reachability(time_t now);
+
+int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
+                                            const routerinfo_t *ri_old);
 void dirserv_orconn_tls_done(const tor_addr_t *addr,
                              uint16_t or_port,
                              const char *digest_rcvd,
                              const struct ed25519_public_key_t *ed_id_rcvd);
-int dirserv_should_launch_reachability_test(const routerinfo_t *ri,
-                                            const routerinfo_t *ri_old);
-void dirserv_single_reachability_test(time_t now, routerinfo_t *router);
-void dirserv_test_reachability(time_t now);
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+#define dirserv_single_reachability_test(now, router) \
+  (((void)(now)),((void)(router)))
+#define dirserv_test_reachability(now) \
+  (((void)(now)))
+
+#define dirserv_should_launch_reachability_test(ri, ri_old) \
+  (((void)(ri)),((void)(ri_old)),0)
+#define dirserv_orconn_tls_done(addr, or_port, digest_rcvd, ed_id_rcvd) \
+  (((void)(addr)),((void)(or_port)),((void)(digest_rcvd)), \
+   ((void)(ed_id_rcvd)))
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
-#endif
+#endif /* !defined(TOR_REACHABILITY_H) */
diff --git a/src/feature/dirauth/recommend_pkg.c b/src/feature/dirauth/recommend_pkg.c
index 0456ff8463..84254566c6 100644
--- a/src/feature/dirauth/recommend_pkg.c
+++ b/src/feature/dirauth/recommend_pkg.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirauth/recommend_pkg.h b/src/feature/dirauth/recommend_pkg.h
index 8200d78f72..dcd9f8be8a 100644
--- a/src/feature/dirauth/recommend_pkg.h
+++ b/src/feature/dirauth/recommend_pkg.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,18 @@
 #ifndef TOR_RECOMMEND_PKG_H
 #define TOR_RECOMMEND_PKG_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 int validate_recommended_package_line(const char *line);
 
-#endif
+#else
+
+static inline int
+validate_recommended_package_line(const char *line)
+{
+  (void) line;
+  return 0;
+}
+
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+#endif /* !defined(TOR_RECOMMEND_PKG_H) */
diff --git a/src/feature/dirauth/shared_random.c b/src/feature/dirauth/shared_random.c
index 34b2283250..fd55008242 100644
--- a/src/feature/dirauth/shared_random.c
+++ b/src/feature/dirauth/shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -90,7 +90,7 @@
 #include "core/or/or.h"
 #include "feature/dirauth/shared_random.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/nodelist/networkstatus.h"
@@ -99,29 +99,31 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/dirauth/shared_random_state.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirauth/dirauth_sys.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 
-/* String prefix of shared random values in votes/consensuses. */
+/** String prefix of shared random values in votes/consensuses. */
 static const char previous_srv_str[] = "shared-rand-previous-value";
 static const char current_srv_str[] = "shared-rand-current-value";
 static const char commit_ns_str[] = "shared-rand-commit";
 static const char sr_flag_ns_str[] = "shared-rand-participate";
 
-/* The value of the consensus param AuthDirNumSRVAgreements found in the
+/** The value of the consensus param AuthDirNumSRVAgreements found in the
  * vote. This is set once the consensus creation subsystem requests the
  * SRV(s) that should be put in the consensus. We use this value to decide
  * if we keep or not an SRV. */
 static int32_t num_srv_agreements_from_vote;
 
-/* Return a heap allocated copy of the SRV <b>orig</b>. */
-STATIC sr_srv_t *
-srv_dup(const sr_srv_t *orig)
+/** Return a heap allocated copy of the SRV <b>orig</b>. */
+sr_srv_t *
+sr_srv_dup(const sr_srv_t *orig)
 {
   sr_srv_t *duplicate = NULL;
 
@@ -135,7 +137,7 @@ srv_dup(const sr_srv_t *orig)
   return duplicate;
 }
 
-/* Allocate a new commit object and initializing it with <b>rsa_identity</b>
+/** Allocate a new commit object and initializing it with <b>rsa_identity</b>
  * that MUST be provided. The digest algorithm is set to the default one
  * that is supported. The rest is uninitialized. This never returns NULL. */
 static sr_commit_t *
@@ -153,7 +155,7 @@ commit_new(const char *rsa_identity)
   return commit;
 }
 
-/* Issue a log message describing <b>commit</b>. */
+/** Issue a log message describing <b>commit</b>. */
 static void
 commit_log(const sr_commit_t *commit)
 {
@@ -166,7 +168,7 @@ commit_log(const sr_commit_t *commit)
             commit->reveal_ts, safe_str(commit->encoded_reveal));
 }
 
-/* Make sure that the commitment and reveal information in <b>commit</b>
+/** Make sure that the commitment and reveal information in <b>commit</b>
  * match. If they match return 0, return -1 otherwise. This function MUST be
  * used everytime we receive a new reveal value. Furthermore, the commit
  * object MUST have a reveal value and the hash of the reveal value. */
@@ -220,15 +222,15 @@ verify_commit_and_reveal(const sr_commit_t *commit)
   return -1;
 }
 
-/* Return true iff the commit contains an encoded reveal value. */
+/** Return true iff the commit contains an encoded reveal value. */
 STATIC int
 commit_has_reveal_value(const sr_commit_t *commit)
 {
-  return !tor_mem_is_zero(commit->encoded_reveal,
+  return !fast_mem_is_zero(commit->encoded_reveal,
                           sizeof(commit->encoded_reveal));
 }
 
-/* Parse the encoded commit. The format is:
+/** Parse the encoded commit. The format is:
  *    base64-encode( TIMESTAMP || H(REVEAL) )
  *
  * If successfully decoded and parsed, commit is updated and 0 is returned.
@@ -283,7 +285,7 @@ commit_decode(const char *encoded, sr_commit_t *commit)
   return -1;
 }
 
-/* Parse the b64 blob at <b>encoded</b> containing reveal information and
+/** Parse the b64 blob at <b>encoded</b> containing reveal information and
  * store the information in-place in <b>commit</b>. Return 0 on success else
  * a negative value. */
 STATIC int
@@ -333,7 +335,7 @@ reveal_decode(const char *encoded, sr_commit_t *commit)
   return -1;
 }
 
-/* Encode a reveal element using a given commit object to dst which is a
+/** Encode a reveal element using a given commit object to dst which is a
  * buffer large enough to put the base64-encoded reveal construction. The
  * format is as follow:
  *     REVEAL = base64-encode( TIMESTAMP || H(RN) )
@@ -362,7 +364,7 @@ reveal_encode(const sr_commit_t *commit, char *dst, size_t len)
   return ret;
 }
 
-/* Encode the given commit object to dst which is a buffer large enough to
+/** Encode the given commit object to dst which is a buffer large enough to
  * put the base64-encoded commit. The format is as follow:
  *     COMMIT = base64-encode( TIMESTAMP || H(H(RN)) )
  * Return base64 encoded length on success else a negative value.
@@ -388,14 +390,14 @@ commit_encode(const sr_commit_t *commit, char *dst, size_t len)
   return base64_encode(dst, len, buf, sizeof(buf), 0);
 }
 
-/* Cleanup both our global state and disk state. */
+/** Cleanup both our global state and disk state. */
 static void
 sr_cleanup(void)
 {
   sr_state_free_all();
 }
 
-/* Using <b>commit</b>, return a newly allocated string containing the commit
+/** Using <b>commit</b>, return a newly allocated string containing the commit
  * information that should be used during SRV calculation. It's the caller
  * responsibility to free the memory. Return NULL if this is not a commit to be
  * used for SRV calculation. */
@@ -414,7 +416,7 @@ get_srv_element_from_commit(const sr_commit_t *commit)
   return element;
 }
 
-/* Return a srv object that is built with the construction:
+/** Return a srv object that is built with the construction:
  *    SRV = SHA3-256("shared-random" | INT_8(reveal_num) |
  *                   INT_4(version) | HASHED_REVEALS | previous_SRV)
  * This function cannot fail. */
@@ -456,7 +458,7 @@ generate_srv(const char *hashed_reveals, uint64_t reveal_num,
   return srv;
 }
 
-/* Compare reveal values and return the result. This should exclusively be
+/** Compare reveal values and return the result. This should exclusively be
  * used by smartlist_sort(). */
 static int
 compare_reveal_(const void **_a, const void **_b)
@@ -466,7 +468,7 @@ compare_reveal_(const void **_a, const void **_b)
                      sizeof(a->hashed_reveal));
 }
 
-/* Given <b>commit</b> give the line that we should place in our votes.
+/** Given <b>commit</b> give the line that we should place in our votes.
  * It's the responsibility of the caller to free the string. */
 static char *
 get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
@@ -486,7 +488,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   {
     /* Send a reveal value for this commit if we have one. */
     const char *reveal_str = commit->encoded_reveal;
-    if (tor_mem_is_zero(commit->encoded_reveal,
+    if (fast_mem_is_zero(commit->encoded_reveal,
                         sizeof(commit->encoded_reveal))) {
       reveal_str = "";
     }
@@ -506,7 +508,7 @@ get_vote_line_from_commit(const sr_commit_t *commit, sr_phase_t phase)
   return vote_line;
 }
 
-/* Return a heap allocated string that contains the given <b>srv</b> string
+/** Return a heap allocated string that contains the given <b>srv</b> string
  * representation formatted for a networkstatus document using the
  * <b>key</b> as the start of the line. This doesn't return NULL. */
 static char *
@@ -524,7 +526,7 @@ srv_to_ns_string(const sr_srv_t *srv, const char *key)
   return srv_str;
 }
 
-/* Given the previous SRV and the current SRV, return a heap allocated
+/** Given the previous SRV and the current SRV, return a heap allocated
  * string with their data that could be put in a vote or a consensus. Caller
  * must free the returned string.  Return NULL if no SRVs were provided. */
 static char *
@@ -557,7 +559,7 @@ get_ns_str_from_sr_values(const sr_srv_t *prev_srv, const sr_srv_t *cur_srv)
   return srv_str;
 }
 
-/* Return 1 iff the two commits have the same commitment values. This
+/** Return 1 iff the two commits have the same commitment values. This
  * function does not care about reveal values. */
 STATIC int
 commitments_are_the_same(const sr_commit_t *commit_one,
@@ -572,7 +574,7 @@ commitments_are_the_same(const sr_commit_t *commit_one,
   return 1;
 }
 
-/* We just received a commit from the vote of authority with
+/** We just received a commit from the vote of authority with
  * <b>identity_digest</b>. Return 1 if this commit is authorititative that
  * is, it belongs to the authority that voted it. Else return 0 if not. */
 STATIC int
@@ -586,7 +588,7 @@ commit_is_authoritative(const sr_commit_t *commit,
                     sizeof(commit->rsa_identity));
 }
 
-/* Decide if the newly received <b>commit</b> should be kept depending on
+/** Decide if the newly received <b>commit</b> should be kept depending on
  * the current phase and state of the protocol. The <b>voter_key</b> is the
  * RSA identity key fingerprint of the authority's vote from which the
  * commit comes from. The <b>phase</b> is the phase we should be validating
@@ -705,7 +707,7 @@ should_keep_commit(const sr_commit_t *commit, const char *voter_key,
   return 0;
 }
 
-/* We are in reveal phase and we found a valid and verified <b>commit</b> in
+/** We are in reveal phase and we found a valid and verified <b>commit</b> in
  * a vote that contains reveal values that we could use. Update the commit
  * we have in our state. Never call this with an unverified commit. */
 STATIC void
@@ -726,7 +728,7 @@ save_commit_during_reveal_phase(const sr_commit_t *commit)
   sr_state_copy_reveal_info(saved_commit, commit);
 }
 
-/* Save <b>commit</b> to our persistent state. Depending on the current
+/** Save <b>commit</b> to our persistent state. Depending on the current
  * phase, different actions are taken. Steals reference of <b>commit</b>.
  * The commit object MUST be valid and verified before adding it to the
  * state. */
@@ -751,7 +753,7 @@ save_commit_to_state(sr_commit_t *commit)
   }
 }
 
-/* Return 1 if we should we keep an SRV voted by <b>n_agreements</b> auths.
+/** Return 1 if we should we keep an SRV voted by <b>n_agreements</b> auths.
  * Return 0 if we should ignore it. */
 static int
 should_keep_srv(int n_agreements)
@@ -781,7 +783,7 @@ should_keep_srv(int n_agreements)
   return 1;
 }
 
-/* Helper: compare two DIGEST256_LEN digests. */
+/** Helper: compare two DIGEST256_LEN digests. */
 static int
 compare_srvs_(const void **_a, const void **_b)
 {
@@ -789,7 +791,7 @@ compare_srvs_(const void **_a, const void **_b)
   return tor_memcmp(a->value, b->value, sizeof(a->value));
 }
 
-/* Return the most frequent member of the sorted list of DIGEST256_LEN
+/** Return the most frequent member of the sorted list of DIGEST256_LEN
  * digests in <b>sl</b> with the count of that most frequent element. */
 static sr_srv_t *
 smartlist_get_most_frequent_srv(const smartlist_t *sl, int *count_out)
@@ -806,7 +808,7 @@ compare_srv_(const void **_a, const void **_b)
                      sizeof(a->value));
 }
 
-/* Using a list of <b>votes</b>, return the SRV object from them that has
+/** Using a list of <b>votes</b>, return the SRV object from them that has
  * been voted by the majority of dirauths. If <b>current</b> is set, we look
  * for the current SRV value else the previous one. The returned pointer is
  * an object located inside a vote. NULL is returned if no appropriate value
@@ -868,7 +870,7 @@ get_majority_srv_from_votes(const smartlist_t *votes, int current)
   return the_srv;
 }
 
-/* Free a commit object. */
+/** Free a commit object. */
 void
 sr_commit_free_(sr_commit_t *commit)
 {
@@ -880,7 +882,7 @@ sr_commit_free_(sr_commit_t *commit)
   tor_free(commit);
 }
 
-/* Generate the commitment/reveal value for the protocol run starting at
+/** Generate the commitment/reveal value for the protocol run starting at
  * <b>timestamp</b>. <b>my_rsa_cert</b> is our authority RSA certificate. */
 sr_commit_t *
 sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
@@ -937,7 +939,8 @@ sr_generate_our_commit(time_t timestamp, const authority_cert_t *my_rsa_cert)
   return NULL;
 }
 
-/* Compute the shared random value based on the active commits in our state. */
+/** Compute the shared random value based on the active commits in our
+ * state. */
 void
 sr_compute_srv(void)
 {
@@ -1010,7 +1013,7 @@ sr_compute_srv(void)
   tor_free(reveals);
 }
 
-/* Parse a commit from a vote or from our disk state and return a newly
+/** Parse a commit from a vote or from our disk state and return a newly
  * allocated commit object. NULL is returned on error.
  *
  * The commit's data is in <b>args</b> and the order matters very much:
@@ -1082,7 +1085,7 @@ sr_parse_commit(const smartlist_t *args)
   return NULL;
 }
 
-/* Called when we are done parsing a vote by <b>voter_key</b> that might
+/** Called when we are done parsing a vote by <b>voter_key</b> that might
  * contain some useful <b>commits</b>. Find if any of them should be kept
  * and update our state accordingly. Once done, the list of commitments will
  * be empty. */
@@ -1120,7 +1123,7 @@ sr_handle_received_commits(smartlist_t *commits, crypto_pk_t *voter_key)
   } SMARTLIST_FOREACH_END(commit);
 }
 
-/* Return a heap-allocated string containing commits that should be put in
+/** Return a heap-allocated string containing commits that should be put in
  * the votes. It's the responsibility of the caller to free the string.
  * This always return a valid string, either empty or with line(s). */
 char *
@@ -1129,7 +1132,7 @@ sr_get_string_for_vote(void)
   char *vote_str = NULL;
   digestmap_t *state_commits;
   smartlist_t *chunks = smartlist_new();
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
 
   /* Are we participating in the protocol? */
   if (!options->AuthDirSharedRandomness) {
@@ -1178,7 +1181,7 @@ sr_get_string_for_vote(void)
   return vote_str;
 }
 
-/* Return a heap-allocated string that should be put in the consensus and
+/** Return a heap-allocated string that should be put in the consensus and
  * contains the shared randomness values. It's the responsibility of the
  * caller to free the string. NULL is returned if no SRV(s) available.
  *
@@ -1194,7 +1197,7 @@ sr_get_string_for_consensus(const smartlist_t *votes,
                             int32_t num_srv_agreements)
 {
   char *srv_str;
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
 
   tor_assert(votes);
 
@@ -1222,7 +1225,7 @@ sr_get_string_for_consensus(const smartlist_t *votes,
   return NULL;
 }
 
-/* We just computed a new <b>consensus</b>. Update our state with the SRVs
+/** We just computed a new <b>consensus</b>. Update our state with the SRVs
  * from the consensus (might be NULL as well). Register the SRVs in our SR
  * state and prepare for the upcoming protocol round. */
 void
@@ -1253,15 +1256,15 @@ sr_act_post_consensus(const networkstatus_t *consensus)
      * decided by the majority. */
     sr_state_unset_fresh_srv();
     /* Set the SR values from the given consensus. */
-    sr_state_set_previous_srv(srv_dup(consensus->sr_info.previous_srv));
-    sr_state_set_current_srv(srv_dup(consensus->sr_info.current_srv));
+    sr_state_set_previous_srv(sr_srv_dup(consensus->sr_info.previous_srv));
+    sr_state_set_current_srv(sr_srv_dup(consensus->sr_info.current_srv));
   }
 
   /* Prepare our state so that it's ready for the next voting period. */
-  sr_state_update(voting_schedule_get_next_valid_after_time());
+  sr_state_update(dirauth_sched_get_next_valid_after_time());
 }
 
-/* Initialize shared random subsystem. This MUST be called early in the boot
+/** Initialize shared random subsystem. This MUST be called early in the boot
  * process of tor. Return 0 on success else -1 on error. */
 int
 sr_init(int save_to_disk)
@@ -1269,7 +1272,7 @@ sr_init(int save_to_disk)
   return sr_state_init(save_to_disk, 1);
 }
 
-/* Save our state to disk and cleanup everything. */
+/** Save our state to disk and cleanup everything. */
 void
 sr_save_and_cleanup(void)
 {
@@ -1279,7 +1282,7 @@ sr_save_and_cleanup(void)
 
 #ifdef TOR_UNIT_TESTS
 
-/* Set the global value of number of SRV agreements so the test can play
+/** Set the global value of number of SRV agreements so the test can play
  * along by calling specific functions that don't parse the votes prior for
  * the AuthDirNumSRVAgreements value. */
 void
diff --git a/src/feature/dirauth/shared_random.h b/src/feature/dirauth/shared_random.h
index 25d95ebbc7..c4e259dcdb 100644
--- a/src/feature/dirauth/shared_random.h
+++ b/src/feature/dirauth/shared_random.h
@@ -1,86 +1,88 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SHARED_RANDOM_H
 #define TOR_SHARED_RANDOM_H
 
-/*
- * This file contains ABI/API of the shared random protocol defined in
+/**
+ * \file shared_random.h
+ *
+ * \brief This file contains ABI/API of the shared random protocol defined in
  * proposal #250. Every public functions and data structure are namespaced
  * with "sr_" which stands for shared random.
  */
 
 #include "core/or/or.h"
 
-/* Protocol version */
+/** Protocol version */
 #define SR_PROTO_VERSION  1
-/* Default digest algorithm. */
+/** Default digest algorithm. */
 #define SR_DIGEST_ALG DIGEST_SHA3_256
-/* Invariant token in the SRV calculation. */
+/** Invariant token in the SRV calculation. */
 #define SR_SRV_TOKEN "shared-random"
-/* Don't count the NUL terminated byte even though the TOKEN has it. */
+/** Don't count the NUL terminated byte even though the TOKEN has it. */
 #define SR_SRV_TOKEN_LEN (sizeof(SR_SRV_TOKEN) - 1)
 
-/* Length of the random number (in bytes). */
+/** Length of the random number (in bytes). */
 #define SR_RANDOM_NUMBER_LEN 32
-/* Size of a decoded commit value in a vote or state. It's a hash and a
+/** Size of a decoded commit value in a vote or state. It's a hash and a
  * timestamp. It adds up to 40 bytes. */
 #define SR_COMMIT_LEN (sizeof(uint64_t) + DIGEST256_LEN)
-/* Size of a decoded reveal value from a vote or state. It's a 64 bit
+/** Size of a decoded reveal value from a vote or state. It's a 64 bit
  * timestamp and the hashed random number. This adds up to 40 bytes. */
 #define SR_REVEAL_LEN (sizeof(uint64_t) + DIGEST256_LEN)
-/* Size of SRV message length. The construction is has follow:
+/** Size of SRV message length. The construction is has follow:
  *  "shared-random" | INT_8(reveal_num) | INT_4(version) | PREV_SRV */
 #define SR_SRV_MSG_LEN \
   (SR_SRV_TOKEN_LEN + sizeof(uint64_t) + sizeof(uint32_t) + DIGEST256_LEN)
 
-/* Length of base64 encoded commit NOT including the NUL terminated byte.
+/** Length of base64 encoded commit NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
 #define SR_COMMIT_BASE64_LEN (BASE64_LEN(SR_COMMIT_LEN))
-/* Length of base64 encoded reveal NOT including the NUL terminated byte.
+/** Length of base64 encoded reveal NOT including the NUL terminated byte.
  * Formula is taken from base64_encode_size. This adds up to 56 bytes. */
 #define SR_REVEAL_BASE64_LEN (BASE64_LEN(SR_REVEAL_LEN))
-/* Length of base64 encoded shared random value. It's 32 bytes long so 44
+/** Length of base64 encoded shared random value. It's 32 bytes long so 44
  * bytes from the base64_encode_size formula. That includes the '='
  * character at the end. */
 #define SR_SRV_VALUE_BASE64_LEN (BASE64_LEN(DIGEST256_LEN))
 
-/* Assert if commit valid flag is not set. */
+/** Assert if commit valid flag is not set. */
 #define ASSERT_COMMIT_VALID(c) tor_assert((c)->valid)
 
-/* Protocol phase. */
+/** Protocol phase. */
 typedef enum {
-  /* Commitment phase */
+  /** Commitment phase */
   SR_PHASE_COMMIT  = 1,
-  /* Reveal phase */
+  /** Reveal phase */
   SR_PHASE_REVEAL  = 2,
 } sr_phase_t;
 
-/* A shared random value (SRV). */
+/** A shared random value (SRV). */
 typedef struct sr_srv_t {
-  /* The number of reveal values used to derive this SRV. */
+  /** The number of reveal values used to derive this SRV. */
   uint64_t num_reveals;
-  /* The actual value. This is the stored result of SHA3-256. */
+  /** The actual value. This is the stored result of SHA3-256. */
   uint8_t value[DIGEST256_LEN];
 } sr_srv_t;
 
-/* A commit (either ours or from another authority). */
+/** A commit (either ours or from another authority). */
 typedef struct sr_commit_t {
-  /* Hashing algorithm used. */
+  /** Hashing algorithm used. */
   digest_algorithm_t alg;
-  /* Indicate if this commit has been verified thus valid. */
+  /** Indicate if this commit has been verified thus valid. */
   unsigned int valid:1;
 
   /* Commit owner info */
 
-  /* The RSA identity key of the authority and its base16 representation,
+  /** The RSA identity key of the authority and its base16 representation,
    * which includes the NUL terminated byte. */
   char rsa_identity[DIGEST_LEN];
   char rsa_identity_hex[HEX_DIGEST_LEN + 1];
 
   /* Commitment information */
 
-  /* Timestamp of reveal. Correspond to TIMESTAMP. */
+  /** Timestamp of reveal. Correspond to TIMESTAMP. */
   uint64_t reveal_ts;
   /* H(REVEAL) as found in COMMIT message. */
   char hashed_reveal[DIGEST256_LEN];
@@ -89,13 +91,13 @@ typedef struct sr_commit_t {
 
   /* Reveal information */
 
-  /* H(RN) which is what we used as the random value for this commit. We
+  /** H(RN) which is what we used as the random value for this commit. We
    * don't use the raw bytes since those are sent on the network thus
    * avoiding possible information leaks of our PRNG. */
   uint8_t random_number[SR_RANDOM_NUMBER_LEN];
-  /* Timestamp of commit. Correspond to TIMESTAMP. */
+  /** Timestamp of commit. Correspond to TIMESTAMP. */
   uint64_t commit_ts;
-  /* This is the whole reveal message. We use it during verification */
+  /** This is the whole reveal message. We use it during verification */
   char encoded_reveal[SR_REVEAL_BASE64_LEN + 1];
 } sr_commit_t;
 
@@ -110,7 +112,7 @@ int sr_init(int save_to_disk);
 void sr_save_and_cleanup(void);
 void sr_act_post_consensus(const networkstatus_t *consensus);
 
-#else /* HAVE_MODULE_DIRAUTH */
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
 
 static inline int
 sr_init(int save_to_disk)
@@ -131,7 +133,7 @@ sr_act_post_consensus(const networkstatus_t *consensus)
   (void) consensus;
 }
 
-#endif /* HAVE_MODULE_DIRAUTH */
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
 /* Public methods used only by dirauth code. */
 
@@ -154,6 +156,7 @@ const char *sr_commit_get_rsa_fpr(const sr_commit_t *commit)
 void sr_compute_srv(void);
 sr_commit_t *sr_generate_our_commit(time_t timestamp,
                                     const authority_cert_t *my_rsa_cert);
+sr_srv_t *sr_srv_dup(const sr_srv_t *orig);
 
 #ifdef SHARED_RANDOM_PRIVATE
 
@@ -172,7 +175,6 @@ STATIC sr_srv_t *get_majority_srv_from_votes(const smartlist_t *votes,
                                              int current);
 
 STATIC void save_commit_to_state(sr_commit_t *commit);
-STATIC sr_srv_t *srv_dup(const sr_srv_t *orig);
 STATIC int commitments_are_the_same(const sr_commit_t *commit_one,
                                     const sr_commit_t *commit_two);
 STATIC int commit_is_authoritative(const sr_commit_t *commit,
@@ -191,4 +193,3 @@ void set_num_srv_agreements(int32_t value);
 #endif /* TOR_UNIT_TESTS */
 
 #endif /* !defined(TOR_SHARED_RANDOM_H) */
-
diff --git a/src/feature/dirauth/shared_random_state.c b/src/feature/dirauth/shared_random_state.c
index b3e4a4ef92..07bc757506 100644
--- a/src/feature/dirauth/shared_random_state.c
+++ b/src/feature/dirauth/shared_random_state.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/nodelist/networkstatus.h"
@@ -20,23 +20,24 @@
 #include "feature/dirauth/shared_random.h"
 #include "feature/hs_common/shared_random_client.h"
 #include "feature/dirauth/shared_random_state.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "lib/encoding/confline.h"
+#include "lib/version/torversion.h"
 
 #include "app/config/or_state_st.h"
 
-/* Default filename of the shared random state on disk. */
+/** Default filename of the shared random state on disk. */
 static const char default_fname[] = "sr-state";
 
-/* String representation of a protocol phase. */
+/** String representation of a protocol phase. */
 static const char *phase_str[] = { "unknown", "commit", "reveal" };
 
-/* Our shared random protocol state. There is only one possible state per
+/** Our shared random protocol state. There is only one possible state per
  * protocol run so this is the global state which is reset at every run once
  * the shared random value has been computed. */
 static sr_state_t *sr_state = NULL;
 
-/* Representation of our persistent state on disk. The sr_state above
+/** Representation of our persistent state on disk. The sr_state above
  * contains the data parsed from this state. When we save to disk, we
  * translate the sr_state to this sr_disk_state. */
 static sr_disk_state_t *sr_disk_state = NULL;
@@ -50,24 +51,18 @@ static const char dstate_cur_srv_key[] = "SharedRandCurrentValue";
  * members with CONF_CHECK_VAR_TYPE. */
 DUMMY_TYPECHECK_INSTANCE(sr_disk_state_t);
 
-/* These next two are duplicates or near-duplicates from config.c */
-#define VAR(name, conftype, member, initvalue)                              \
-  { name, CONFIG_TYPE_ ## conftype, offsetof(sr_disk_state_t, member),      \
-      initvalue CONF_TEST_MEMBERS(sr_disk_state_t, conftype, member) }
-/* As VAR, but the option name and member name are the same. */
-#define V(member, conftype, initvalue) \
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(sr_disk_state_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
   VAR(#member, conftype, member, initvalue)
-/* Our persistent state magic number. */
-#define SR_DISK_STATE_MAGIC 0x98AB1254
 
-static int
-disk_state_validate_cb(void *old_state, void *state, void *default_state,
-                       int from_setconf, char **msg);
-static void disk_state_free_cb(void *);
+/** Our persistent state magic number. */
+#define SR_DISK_STATE_MAGIC 0x98AB1254
 
-/* Array of variables that are saved to disk as a persistent state. */
-static config_var_t state_vars[] = {
-  V(Version,                    UINT, "0"),
+/** Array of variables that are saved to disk as a persistent state. */
+// clang-format off
+static const config_var_t state_vars[] = {
+  V(Version,                    POSINT, "0"),
   V(TorVersion,                 STRING, NULL),
   V(ValidAfter,                 ISOTIME, NULL),
   V(ValidUntil,                 ISOTIME, NULL),
@@ -79,29 +74,45 @@ static config_var_t state_vars[] = {
   VAR("SharedRandCurrentValue", LINELIST_S, SharedRandValues, NULL),
   END_OF_CONFIG_VARS
 };
+// clang-format on
 
-/* "Extra" variable in the state that receives lines we can't parse. This
+/** "Extra" variable in the state that receives lines we can't parse. This
  * lets us preserve options from versions of Tor newer than us. */
-static config_var_t state_extra_var = {
-  "__extra", CONFIG_TYPE_LINELIST,
-  offsetof(sr_disk_state_t, ExtraLines), NULL
-  CONF_TEST_MEMBERS(sr_disk_state_t, LINELIST, ExtraLines)
+static const struct_member_t state_extra_var = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(sr_disk_state_t, ExtraLines),
 };
 
-/* Configuration format of sr_disk_state_t. */
+/** Configuration format of sr_disk_state_t. */
 static const config_format_t state_format = {
-  sizeof(sr_disk_state_t),
-  SR_DISK_STATE_MAGIC,
-  offsetof(sr_disk_state_t, magic_),
-  NULL,
-  NULL,
-  state_vars,
-  disk_state_validate_cb,
-  disk_state_free_cb,
-  &state_extra_var,
+  .size = sizeof(sr_disk_state_t),
+  .magic = {
+   "sr_disk_state_t",
+   SR_DISK_STATE_MAGIC,
+   offsetof(sr_disk_state_t, magic_),
+  },
+  .vars = state_vars,
+  .extra = &state_extra_var,
 };
 
-/* Return a string representation of a protocol phase. */
+/** Global configuration manager for the shared-random state file */
+static config_mgr_t *shared_random_state_mgr = NULL;
+
+/** Return the configuration manager for the shared-random state file. */
+static const config_mgr_t *
+get_srs_mgr(void)
+{
+  if (PREDICT_UNLIKELY(shared_random_state_mgr == NULL)) {
+    shared_random_state_mgr = config_mgr_new(&state_format);
+    config_mgr_freeze(shared_random_state_mgr);
+  }
+  return shared_random_state_mgr;
+}
+
+static void state_query_del_(sr_state_object_t obj_type, void *data);
+
+/** Return a string representation of a protocol phase. */
 STATIC const char *
 get_phase_str(sr_phase_t phase)
 {
@@ -119,7 +130,7 @@ get_phase_str(sr_phase_t phase)
 
   return the_string;
 }
-/* Return the time we should expire the state file created at <b>now</b>.
+/** Return the time we should expire the state file created at <b>now</b>.
  * We expire the state file in the beginning of the next protocol run. */
 STATIC time_t
 get_state_valid_until_time(time_t now)
@@ -130,7 +141,7 @@ get_state_valid_until_time(time_t now)
 
   voting_interval = get_voting_interval();
   /* Find the time the current round started. */
-  beginning_of_current_round = get_start_time_of_current_round();
+  beginning_of_current_round = dirauth_sched_get_cur_valid_after_time();
 
   /* Find how many rounds are left till the end of the protocol run */
   current_round = (now / voting_interval) % total_rounds;
@@ -150,7 +161,7 @@ get_state_valid_until_time(time_t now)
   return valid_until;
 }
 
-/* Given the consensus 'valid-after' time, return the protocol phase we should
+/** Given the consensus 'valid-after' time, return the protocol phase we should
  * be in. */
 STATIC sr_phase_t
 get_sr_protocol_phase(time_t valid_after)
@@ -170,7 +181,7 @@ get_sr_protocol_phase(time_t valid_after)
   }
 }
 
-/* Add the given <b>commit</b> to <b>state</b>. It MUST be a valid commit
+/** Add the given <b>commit</b> to <b>state</b>. It MUST be a valid commit
  * and there shouldn't be a commit from the same authority in the state
  * already else verification hasn't been done prior. This takes ownership of
  * the commit once in our state. */
@@ -195,7 +206,7 @@ commit_add_to_state(sr_commit_t *commit, sr_state_t *state)
   }
 }
 
-/* Helper: deallocate a commit object. (Used with digestmap_free(), which
+/** Helper: deallocate a commit object. (Used with digestmap_free(), which
  * requires a function pointer whose argument is void *). */
 static void
 commit_free_(void *p)
@@ -206,7 +217,7 @@ commit_free_(void *p)
 #define state_free(val) \
   FREE_AND_NULL(sr_state_t, state_free_, (val))
 
-/* Free a state that was allocated with state_new(). */
+/** Free a state that was allocated with state_new(). */
 static void
 state_free_(sr_state_t *state)
 {
@@ -220,7 +231,7 @@ state_free_(sr_state_t *state)
   tor_free(state);
 }
 
-/* Allocate an sr_state_t object and returns it. If no <b>fname</b>, the
+/** Allocate an sr_state_t object and returns it. If no <b>fname</b>, the
  * default file name is used. This function does NOT initialize the state
  * timestamp, phase or shared random value. NULL is never returned. */
 static sr_state_t *
@@ -239,7 +250,7 @@ state_new(const char *fname, time_t now)
   return new_state;
 }
 
-/* Set our global state pointer with the one given. */
+/** Set our global state pointer with the one given. */
 static void
 state_set(sr_state_t *state)
 {
@@ -253,34 +264,33 @@ state_set(sr_state_t *state)
 #define disk_state_free(val) \
   FREE_AND_NULL(sr_disk_state_t, disk_state_free_, (val))
 
-/* Free an allocated disk state. */
+/** Free an allocated disk state. */
 static void
 disk_state_free_(sr_disk_state_t *state)
 {
   if (state == NULL) {
     return;
   }
-  config_free(&state_format, state);
+  config_free(get_srs_mgr(), state);
 }
 
-/* Allocate a new disk state, initialize it and return it. */
+/** Allocate a new disk state, initialize it and return it. */
 static sr_disk_state_t *
 disk_state_new(time_t now)
 {
-  sr_disk_state_t *new_state = tor_malloc_zero(sizeof(*new_state));
+  sr_disk_state_t *new_state = config_new(get_srs_mgr());
 
-  new_state->magic_ = SR_DISK_STATE_MAGIC;
   new_state->Version = SR_PROTO_VERSION;
   new_state->TorVersion = tor_strdup(get_version());
   new_state->ValidUntil = get_state_valid_until_time(now);
   new_state->ValidAfter = now;
 
   /* Init config format. */
-  config_init(&state_format, new_state);
+  config_init(get_srs_mgr(), new_state);
   return new_state;
 }
 
-/* Set our global disk state with the given state. */
+/** Set our global disk state with the given state. */
 static void
 disk_state_set(sr_disk_state_t *state)
 {
@@ -291,7 +301,7 @@ disk_state_set(sr_disk_state_t *state)
   sr_disk_state = state;
 }
 
-/* Return -1 if the disk state is invalid (something in there that we can't or
+/** Return -1 if the disk state is invalid (something in there that we can't or
  * shouldn't use). Return 0 if everything checks out. */
 static int
 disk_state_validate(const sr_disk_state_t *state)
@@ -326,31 +336,7 @@ disk_state_validate(const sr_disk_state_t *state)
   return -1;
 }
 
-/* Validate the disk state (NOP for now). */
-static int
-disk_state_validate_cb(void *old_state, void *state, void *default_state,
-                       int from_setconf, char **msg)
-{
-  /* We don't use these; only options do. */
-  (void) from_setconf;
-  (void) default_state;
-  (void) old_state;
-
-  /* This is called by config_dump which is just before we are about to
-   * write it to disk. At that point, our global memory state has been
-   * copied to the disk state so it's fair to assume it's trustable. */
-  (void) state;
-  (void) msg;
-  return 0;
-}
-
-static void
-disk_state_free_cb(void *state)
-{
-  disk_state_free_(state);
-}
-
-/* Parse the Commit line(s) in the disk state and translate them to the
+/** Parse the Commit line(s) in the disk state and translate them to the
  * the memory state. Return 0 on success else -1 on error. */
 static int
 disk_state_parse_commits(sr_state_t *state,
@@ -405,7 +391,7 @@ disk_state_parse_commits(sr_state_t *state,
   return -1;
 }
 
-/* Parse a share random value line from the disk state and save it to dst
+/** Parse a share random value line from the disk state and save it to dst
  * which is an allocated srv object. Return 0 on success else -1. */
 static int
 disk_state_parse_srv(const char *value, sr_srv_t *dst)
@@ -440,7 +426,7 @@ disk_state_parse_srv(const char *value, sr_srv_t *dst)
   return ret;
 }
 
-/* Parse both SharedRandCurrentValue and SharedRandPreviousValue line from
+/** Parse both SharedRandCurrentValue and SharedRandPreviousValue line from
  * the state. Return 0 on success else -1. */
 static int
 disk_state_parse_sr_values(sr_state_t *state,
@@ -491,7 +477,7 @@ disk_state_parse_sr_values(sr_state_t *state,
   return -1;
 }
 
-/* Parse the given disk state and set a newly allocated state. On success,
+/** Parse the given disk state and set a newly allocated state. On success,
  * return that state else NULL. */
 static sr_state_t *
 disk_state_parse(const sr_disk_state_t *new_disk_state)
@@ -525,7 +511,7 @@ disk_state_parse(const sr_disk_state_t *new_disk_state)
   return NULL;
 }
 
-/* From a valid commit object and an allocated config line, set the line's
+/** From a valid commit object and an allocated config line, set the line's
  * value to the state string representation of a commit. */
 static void
 disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
@@ -535,7 +521,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
   tor_assert(commit);
   tor_assert(line);
 
-  if (!tor_mem_is_zero(commit->encoded_reveal,
+  if (!fast_mem_is_zero(commit->encoded_reveal,
                        sizeof(commit->encoded_reveal))) {
     /* Add extra whitespace so we can format the line correctly. */
     tor_asprintf(&reveal_str, " %s", commit->encoded_reveal);
@@ -552,7 +538,7 @@ disk_state_put_commit_line(const sr_commit_t *commit, config_line_t *line)
   }
 }
 
-/* From a valid srv object and an allocated config line, set the line's
+/** From a valid srv object and an allocated config line, set the line's
  * value to the state string representation of a shared random value. */
 static void
 disk_state_put_srv_line(const sr_srv_t *srv, config_line_t *line)
@@ -570,7 +556,7 @@ disk_state_put_srv_line(const sr_srv_t *srv, config_line_t *line)
   tor_asprintf(&line->value, "%" PRIu64 " %s", srv->num_reveals, encoded);
 }
 
-/* Reset disk state that is free allocated memory and zeroed the object. */
+/** Reset disk state that is free allocated memory and zeroed the object. */
 static void
 disk_state_reset(void)
 {
@@ -580,15 +566,16 @@ disk_state_reset(void)
   config_free_lines(sr_disk_state->ExtraLines);
   tor_free(sr_disk_state->TorVersion);
 
-  /* Clean up the struct */
-  memset(sr_disk_state, 0, sizeof(*sr_disk_state));
+  /* Clear other fields. */
+  sr_disk_state->ValidAfter = 0;
+  sr_disk_state->ValidUntil = 0;
+  sr_disk_state->Version = 0;
 
   /* Reset it with useful data */
-  sr_disk_state->magic_ = SR_DISK_STATE_MAGIC;
   sr_disk_state->TorVersion = tor_strdup(get_version());
 }
 
-/* Update our disk state based on our global SR state. */
+/** Update our disk state based on our global SR state. */
 static void
 disk_state_update(void)
 {
@@ -632,7 +619,7 @@ disk_state_update(void)
   } DIGESTMAP_FOREACH_END;
 }
 
-/* Load state from disk and put it into our disk state. If the state passes
+/** Load state from disk and put it into our disk state. If the state passes
  * validation, our global state will be updated with it. Return 0 on
  * success. On error, -EINVAL is returned if the state on disk did contained
  * something malformed or is unreadable. -ENOENT is returned indicating that
@@ -650,7 +637,7 @@ disk_state_load_from_disk(void)
   return ret;
 }
 
-/* Helper for disk_state_load_from_disk(). */
+/** Helper for disk_state_load_from_disk(). */
 STATIC int
 disk_state_load_from_disk_impl(const char *fname)
 {
@@ -679,7 +666,7 @@ disk_state_load_from_disk_impl(const char *fname)
     }
 
     disk_state = disk_state_new(time(NULL));
-    config_assign(&state_format, disk_state, lines, 0, &errmsg);
+    config_assign(get_srs_mgr(), disk_state, lines, 0, &errmsg);
     config_free_lines(lines);
     if (errmsg) {
       log_warn(LD_DIR, "SR: Reading state error: %s", errmsg);
@@ -712,7 +699,7 @@ disk_state_load_from_disk_impl(const char *fname)
   return ret;
 }
 
-/* Save the disk state to disk but before that update it from the current
+/** Save the disk state to disk but before that update it from the current
  * state so we always have the latest. Return 0 on success else -1. */
 static int
 disk_state_save_to_disk(void)
@@ -732,7 +719,7 @@ disk_state_save_to_disk(void)
   /* Make sure that our disk state is up to date with our memory state
    * before saving it to disk. */
   disk_state_update();
-  state = config_dump(&state_format, NULL, sr_disk_state, 0, 0);
+  state = config_dump(get_srs_mgr(), NULL, sr_disk_state, 0, 0);
   format_local_iso_time(tbuf, now);
   tor_asprintf(&content,
                "# Tor shared random state file last generated on %s "
@@ -756,7 +743,7 @@ disk_state_save_to_disk(void)
   return ret;
 }
 
-/* Reset our state to prepare for a new protocol run. Once this returns, all
+/** Reset our state to prepare for a new protocol run. Once this returns, all
  * commits in the state will be removed and freed. */
 STATIC void
 reset_state_for_new_protocol_run(time_t valid_after)
@@ -777,7 +764,7 @@ reset_state_for_new_protocol_run(time_t valid_after)
   sr_state_delete_commits();
 }
 
-/* This is the first round of the new protocol run starting at
+/** This is the first round of the new protocol run starting at
  * <b>valid_after</b>. Do the necessary housekeeping. */
 STATIC void
 new_protocol_run(time_t valid_after)
@@ -811,7 +798,7 @@ new_protocol_run(time_t valid_after)
   }
 }
 
-/* Return 1 iff the <b>next_phase</b> is a phase transition from the current
+/** Return 1 iff the <b>next_phase</b> is a phase transition from the current
  * phase that is it's different. */
 STATIC int
 is_phase_transition(sr_phase_t next_phase)
@@ -819,7 +806,7 @@ is_phase_transition(sr_phase_t next_phase)
   return sr_state->phase != next_phase;
 }
 
-/* Helper function: return a commit using the RSA fingerprint of the
+/** Helper function: return a commit using the RSA fingerprint of the
  * authority or NULL if no such commit is known. */
 static sr_commit_t *
 state_query_get_commit(const char *rsa_fpr)
@@ -828,11 +815,14 @@ state_query_get_commit(const char *rsa_fpr)
   return digestmap_get(sr_state->commits, rsa_fpr);
 }
 
-/* Helper function: This handles the GET state action using an
+/** Helper function: This handles the GET state action using an
  * <b>obj_type</b> and <b>data</b> needed for the action. */
 static void *
 state_query_get_(sr_state_object_t obj_type, const void *data)
 {
+  if (BUG(!sr_state))
+    return NULL;
+
   void *obj = NULL;
 
   switch (obj_type) {
@@ -860,24 +850,45 @@ state_query_get_(sr_state_object_t obj_type, const void *data)
   return obj;
 }
 
-/* Helper function: This handles the PUT state action using an
- * <b>obj_type</b> and <b>data</b> needed for the action. */
+/** Helper function: This handles the PUT state action using an
+ * <b>obj_type</b> and <b>data</b> needed for the action.
+ * PUT frees the previous data before replacing it, if needed. */
 static void
 state_query_put_(sr_state_object_t obj_type, void *data)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
     sr_commit_t *commit = data;
     tor_assert(commit);
+    /* commit_add_to_state() frees the old commit, if there is one */
     commit_add_to_state(commit, sr_state);
     break;
   }
   case SR_STATE_OBJ_CURSRV:
-    sr_state->current_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->current_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_CURSRV, NULL);
+      sr_state->current_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_PREVSRV:
-    sr_state->previous_srv = (sr_srv_t *) data;
+      /* Check if the new pointer is the same as the old one: if it is, it's
+       * probably a bug. The caller may have confused current and previous,
+       * or they may have forgotten to sr_srv_dup().
+       * Putting NULL multiple times is allowed. */
+    if (!BUG(data && sr_state->previous_srv == (sr_srv_t *) data)) {
+      /* We own the old SRV, so we need to free it.  */
+      state_query_del_(SR_STATE_OBJ_PREVSRV, NULL);
+      sr_state->previous_srv = (sr_srv_t *) data;
+    }
     break;
   case SR_STATE_OBJ_VALID_AFTER:
     sr_state->valid_after = *((time_t *) data);
@@ -892,11 +903,14 @@ state_query_put_(sr_state_object_t obj_type, void *data)
   }
 }
 
-/* Helper function: This handles the DEL_ALL state action using an
+/** Helper function: This handles the DEL_ALL state action using an
  * <b>obj_type</b> and <b>data</b> needed for the action. */
 static void
 state_query_del_all_(sr_state_object_t obj_type)
 {
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_COMMIT:
   {
@@ -907,7 +921,7 @@ state_query_del_all_(sr_state_object_t obj_type)
     } DIGESTMAP_FOREACH_END;
     break;
   }
-  /* The following object are _NOT_ suppose to be removed. */
+  /* The following objects are _NOT_ supposed to be removed. */
   case SR_STATE_OBJ_CURSRV:
   case SR_STATE_OBJ_PREVSRV:
   case SR_STATE_OBJ_PHASE:
@@ -918,13 +932,16 @@ state_query_del_all_(sr_state_object_t obj_type)
   }
 }
 
-/* Helper function: This handles the DEL state action using an
+/** Helper function: This handles the DEL state action using an
  * <b>obj_type</b> and <b>data</b> needed for the action. */
 static void
 state_query_del_(sr_state_object_t obj_type, void *data)
 {
   (void) data;
 
+  if (BUG(!sr_state))
+    return;
+
   switch (obj_type) {
   case SR_STATE_OBJ_PREVSRV:
     tor_free(sr_state->previous_srv);
@@ -941,7 +958,7 @@ state_query_del_(sr_state_object_t obj_type, void *data)
   }
 }
 
-/* Query state using an <b>action</b> for an object type <b>obj_type</b>.
+/** Query state using an <b>action</b> for an object type <b>obj_type</b>.
  * The <b>data</b> pointer needs to point to an object that the action needs
  * to use and if anything is required to be returned, it is stored in
  * <b>out</b>.
@@ -983,7 +1000,7 @@ state_query(sr_state_action_t action, sr_state_object_t obj_type,
   }
 }
 
-/* Delete the current SRV value from the state freeing it and the value is set
+/** Delete the current SRV value from the state freeing it and the value is set
  * to NULL meaning empty. */
 STATIC void
 state_del_current_srv(void)
@@ -991,7 +1008,7 @@ state_del_current_srv(void)
   state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_CURSRV, NULL, NULL);
 }
 
-/* Delete the previous SRV value from the state freeing it and the value is
+/** Delete the previous SRV value from the state freeing it and the value is
  * set to NULL meaning empty. */
 STATIC void
 state_del_previous_srv(void)
@@ -999,20 +1016,20 @@ state_del_previous_srv(void)
   state_query(SR_STATE_ACTION_DEL, SR_STATE_OBJ_PREVSRV, NULL, NULL);
 }
 
-/* Rotate SRV value by freeing the previous value, assigning the current
- * value to the previous one and nullifying the current one. */
+/** Rotate SRV value by setting the previous SRV to the current SRV, and
+ * clearing the current SRV. */
 STATIC void
 state_rotate_srv(void)
 {
   /* First delete previous SRV from the state. Object will be freed. */
   state_del_previous_srv();
-  /* Set previous SRV with the current one. */
-  sr_state_set_previous_srv(sr_state_get_current_srv());
-  /* Nullify the current srv. */
+  /* Set previous SRV to a copy of the current one. */
+  sr_state_set_previous_srv(sr_srv_dup(sr_state_get_current_srv()));
+  /* Free and NULL the current srv. */
   sr_state_set_current_srv(NULL);
 }
 
-/* Set valid after time in the our state. */
+/** Set valid after time in the our state. */
 void
 sr_state_set_valid_after(time_t valid_after)
 {
@@ -1020,16 +1037,19 @@ sr_state_set_valid_after(time_t valid_after)
               (void *) &valid_after, NULL);
 }
 
-/* Return the phase we are currently in according to our state. */
+/** Return the phase we are currently in according to our state. */
 sr_phase_t
 sr_state_get_phase(void)
 {
-  void *ptr;
+  void *ptr=NULL;
   state_query(SR_STATE_ACTION_GET, SR_STATE_OBJ_PHASE, NULL, &ptr);
+  tor_assert(ptr);
   return *(sr_phase_t *) ptr;
 }
 
-/* Return the previous SRV value from our state. Value CAN be NULL. */
+/** Return the previous SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_previous_srv(void)
 {
@@ -1039,7 +1059,7 @@ sr_state_get_previous_srv(void)
   return srv;
 }
 
-/* Set the current SRV value from our state. Value CAN be NULL. The srv
+/** Set the current SRV value from our state. Value CAN be NULL. The srv
  * object ownership is transferred to the state object. */
 void
 sr_state_set_previous_srv(const sr_srv_t *srv)
@@ -1048,7 +1068,9 @@ sr_state_set_previous_srv(const sr_srv_t *srv)
               NULL);
 }
 
-/* Return the current SRV value from our state. Value CAN be NULL. */
+/** Return the current SRV value from our state. Value CAN be NULL.
+ * The state object owns the SRV, so the calling code should not free the SRV.
+ * Use sr_srv_dup() if you want to keep a copy of the SRV. */
 const sr_srv_t *
 sr_state_get_current_srv(void)
 {
@@ -1058,7 +1080,7 @@ sr_state_get_current_srv(void)
   return srv;
 }
 
-/* Set the current SRV value from our state. Value CAN be NULL. The srv
+/** Set the current SRV value from our state. Value CAN be NULL. The srv
  * object ownership is transferred to the state object. */
 void
 sr_state_set_current_srv(const sr_srv_t *srv)
@@ -1067,7 +1089,7 @@ sr_state_set_current_srv(const sr_srv_t *srv)
               NULL);
 }
 
-/* Clean all the SRVs in our state. */
+/** Clean all the SRVs in our state. */
 void
 sr_state_clean_srvs(void)
 {
@@ -1076,7 +1098,7 @@ sr_state_clean_srvs(void)
   state_del_current_srv();
 }
 
-/* Return a pointer to the commits map from our state. CANNOT be NULL. */
+/** Return a pointer to the commits map from our state. CANNOT be NULL. */
 digestmap_t *
 sr_state_get_commits(void)
 {
@@ -1087,7 +1109,7 @@ sr_state_get_commits(void)
   return commits;
 }
 
-/* Update the current SR state as needed for the upcoming voting round at
+/** Update the current SR state as needed for the upcoming voting round at
  * <b>valid_after</b>. */
 void
 sr_state_update(time_t valid_after)
@@ -1151,7 +1173,7 @@ sr_state_update(time_t valid_after)
   }
 }
 
-/* Return commit object from the given authority digest <b>rsa_identity</b>.
+/** Return commit object from the given authority digest <b>rsa_identity</b>.
  * Return NULL if not found. */
 sr_commit_t *
 sr_state_get_commit(const char *rsa_identity)
@@ -1165,7 +1187,7 @@ sr_state_get_commit(const char *rsa_identity)
   return commit;
 }
 
-/* Add <b>commit</b> to the permanent state. The commit object ownership is
+/** Add <b>commit</b> to the permanent state. The commit object ownership is
  * transferred to the state so the caller MUST not free it. */
 void
 sr_state_add_commit(sr_commit_t *commit)
@@ -1180,14 +1202,14 @@ sr_state_add_commit(sr_commit_t *commit)
             sr_commit_get_rsa_fpr(commit));
 }
 
-/* Remove all commits from our state. */
+/** Remove all commits from our state. */
 void
 sr_state_delete_commits(void)
 {
   state_query(SR_STATE_ACTION_DEL_ALL, SR_STATE_OBJ_COMMIT, NULL, NULL);
 }
 
-/* Copy the reveal information from <b>commit</b> into <b>saved_commit</b>.
+/** Copy the reveal information from <b>commit</b> into <b>saved_commit</b>.
  * This <b>saved_commit</b> MUST come from our current SR state. Once modified,
  * the disk state is updated. */
 void
@@ -1208,7 +1230,7 @@ sr_state_copy_reveal_info(sr_commit_t *saved_commit, const sr_commit_t *commit)
             sr_commit_get_rsa_fpr(saved_commit));
 }
 
-/* Set the fresh SRV flag from our state. This doesn't need to trigger a
+/** Set the fresh SRV flag from our state. This doesn't need to trigger a
  * disk state synchronization so we directly change the state. */
 void
 sr_state_set_fresh_srv(void)
@@ -1216,7 +1238,7 @@ sr_state_set_fresh_srv(void)
   sr_state->is_srv_fresh = 1;
 }
 
-/* Unset the fresh SRV flag from our state. This doesn't need to trigger a
+/** Unset the fresh SRV flag from our state. This doesn't need to trigger a
  * disk state synchronization so we directly change the state. */
 void
 sr_state_unset_fresh_srv(void)
@@ -1224,14 +1246,14 @@ sr_state_unset_fresh_srv(void)
   sr_state->is_srv_fresh = 0;
 }
 
-/* Return the value of the fresh SRV flag. */
+/** Return the value of the fresh SRV flag. */
 unsigned int
 sr_state_srv_is_fresh(void)
 {
   return sr_state->is_srv_fresh;
 }
 
-/* Cleanup and free our disk and memory state. */
+/** Cleanup and free our disk and memory state. */
 void
 sr_state_free_all(void)
 {
@@ -1240,9 +1262,10 @@ sr_state_free_all(void)
   /* Nullify our global state. */
   sr_state = NULL;
   sr_disk_state = NULL;
+  config_mgr_free(shared_random_state_mgr);
 }
 
-/* Save our current state in memory to disk. */
+/** Save our current state in memory to disk. */
 void
 sr_state_save(void)
 {
@@ -1250,7 +1273,7 @@ sr_state_save(void)
   state_query(SR_STATE_ACTION_SAVE, 0, NULL, NULL);
 }
 
-/* Return 1 iff the state has been initialized that is it exists in memory.
+/** Return 1 iff the state has been initialized that is it exists in memory.
  * Return 0 otherwise. */
 int
 sr_state_is_initialized(void)
@@ -1258,7 +1281,7 @@ sr_state_is_initialized(void)
   return sr_state == NULL ? 0 : 1;
 }
 
-/* Initialize the disk and memory state.
+/** Initialize the disk and memory state.
  *
  * If save_to_disk is set to 1, the state is immediately saved to disk after
  * creation else it's not thus only kept in memory.
@@ -1310,7 +1333,7 @@ sr_state_init(int save_to_disk, int read_from_disk)
   /* We have a state in memory, let's make sure it's updated for the current
    * and next voting round. */
   {
-    time_t valid_after = voting_schedule_get_next_valid_after_time();
+    time_t valid_after = dirauth_sched_get_next_valid_after_time();
     sr_state_update(valid_after);
   }
   return 0;
@@ -1321,7 +1344,7 @@ sr_state_init(int save_to_disk, int read_from_disk)
 
 #ifdef TOR_UNIT_TESTS
 
-/* Set the current phase of the protocol. Used only by unit tests. */
+/** Set the current phase of the protocol. Used only by unit tests. */
 void
 set_sr_phase(sr_phase_t phase)
 {
@@ -1330,7 +1353,7 @@ set_sr_phase(sr_phase_t phase)
   sr_state->phase = phase;
 }
 
-/* Get the SR state. Used only by unit tests */
+/** Get the SR state. Used only by unit tests */
 sr_state_t *
 get_sr_state(void)
 {
diff --git a/src/feature/dirauth/shared_random_state.h b/src/feature/dirauth/shared_random_state.h
index 08f999f9d4..3a34bcc3e7 100644
--- a/src/feature/dirauth/shared_random_state.h
+++ b/src/feature/dirauth/shared_random_state.h
@@ -1,12 +1,17 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file shared_random_state.h
+ * @brief Header for shared_random_state.c
+ **/
+
 #ifndef TOR_SHARED_RANDOM_STATE_H
 #define TOR_SHARED_RANDOM_STATE_H
 
 #include "feature/dirauth/shared_random.h"
 
-/* Action that can be performed on the state for any objects. */
+/** Action that can be performed on the state for any objects. */
 typedef enum {
   SR_STATE_ACTION_GET     = 1,
   SR_STATE_ACTION_PUT     = 2,
@@ -15,52 +20,53 @@ typedef enum {
   SR_STATE_ACTION_SAVE    = 5,
 } sr_state_action_t;
 
-/* Object in the state that can be queried through the state API. */
+/** Object in the state that can be queried through the state API. */
 typedef enum {
-  /* Will return a single commit using an authority identity key. */
+  /** Will return a single commit using an authority identity key. */
   SR_STATE_OBJ_COMMIT,
-  /* Returns the entire list of commits from the state. */
+  /** Returns the entire list of commits from the state. */
   SR_STATE_OBJ_COMMITS,
-  /* Return the current SRV object pointer. */
+  /** Return the current SRV object pointer. */
   SR_STATE_OBJ_CURSRV,
-  /* Return the previous SRV object pointer. */
+  /** Return the previous SRV object pointer. */
   SR_STATE_OBJ_PREVSRV,
-  /* Return the phase. */
+  /** Return the phase. */
   SR_STATE_OBJ_PHASE,
-  /* Get or Put the valid after time. */
+  /** Get or Put the valid after time. */
   SR_STATE_OBJ_VALID_AFTER,
 } sr_state_object_t;
 
-/* State of the protocol. It's also saved on disk in fname. This data
+/** State of the protocol. It's also saved on disk in fname. This data
  * structure MUST be synchronized at all time with the one on disk. */
 typedef struct sr_state_t {
-  /* Filename of the state file on disk. */
+  /** Filename of the state file on disk. */
   char *fname;
-  /* Version of the protocol. */
+  /** Version of the protocol. */
   uint32_t version;
-  /* The valid-after of the voting period we have prepared the state for. */
+  /** The valid-after of the voting period we have prepared the state for. */
   time_t valid_after;
-  /* Until when is this state valid? */
+  /** Until when is this state valid? */
   time_t valid_until;
-  /* Protocol phase. */
+  /** Protocol phase. */
   sr_phase_t phase;
 
-  /* Number of runs completed. */
+  /** Number of runs completed. */
   uint64_t n_protocol_runs;
-  /* The number of commitment rounds we've performed in this protocol run. */
+  /** The number of commitment rounds we've performed in this protocol run. */
   unsigned int n_commit_rounds;
-  /* The number of reveal rounds we've performed in this protocol run. */
+  /** The number of reveal rounds we've performed in this protocol run. */
   unsigned int n_reveal_rounds;
 
-  /* A map of all the received commitments for this protocol run. This is
+  /** A map of all the received commitments for this protocol run. This is
    * indexed by authority RSA identity digest. */
   digestmap_t *commits;
 
-  /* Current and previous shared random value. */
+  /** Current shared random value. */
   sr_srv_t *previous_srv;
+  /** Previous shared random value. */
   sr_srv_t *current_srv;
 
-  /* Indicate if the state contains an SRV that was _just_ generated. This is
+  /** Indicate if the state contains an SRV that was _just_ generated. This is
    * used during voting so that we know whether to use the super majority rule
    * or not when deciding on keeping it for the consensus. It is _always_ set
    * to 0 post consensus.
@@ -73,22 +79,22 @@ typedef struct sr_state_t {
   unsigned int is_srv_fresh:1;
 } sr_state_t;
 
-/* Persistent state of the protocol, as saved to disk. */
+/** Persistent state of the protocol, as saved to disk. */
 typedef struct sr_disk_state_t {
   uint32_t magic_;
-  /* Version of the protocol. */
+  /** Version of the protocol. */
   int Version;
-  /* Version of our running tor. */
+  /** Version of our running tor. */
   char *TorVersion;
-  /* Creation time of this state */
+  /** Creation time of this state */
   time_t ValidAfter;
-  /* State valid until? */
+  /** State valid until? */
   time_t ValidUntil;
-  /* All commits seen that are valid. */
+  /** All commits seen that are valid. */
   struct config_line_t *Commit;
-  /* Previous and current shared random value. */
+  /** Previous and current shared random value. */
   struct config_line_t *SharedRandValues;
-  /* Extra Lines for configuration we might not know. */
+  /** Extra Lines for configuration we might not know. */
   struct config_line_t *ExtraLines;
 } sr_disk_state_t;
 
diff --git a/src/feature/dirauth/vote_microdesc_hash_st.h b/src/feature/dirauth/vote_microdesc_hash_st.h
index 92acdf1157..7f8ebf7fd7 100644
--- a/src/feature/dirauth/vote_microdesc_hash_st.h
+++ b/src/feature/dirauth/vote_microdesc_hash_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file vote_microdesc_hash_st.h
+ * @brief Microdescriptor-hash voting strcture.
+ **/
+
 #ifndef VOTE_MICRODESC_HASH_ST_H
 #define VOTE_MICRODESC_HASH_ST_H
 
@@ -18,5 +23,4 @@ struct vote_microdesc_hash_t {
   char *microdesc_hash_line;
 };
 
-#endif
-
+#endif /* !defined(VOTE_MICRODESC_HASH_ST_H) */
diff --git a/src/feature/dirauth/voteflags.c b/src/feature/dirauth/voteflags.c
index 54c70b989a..477eb6f0b7 100644
--- a/src/feature/dirauth/voteflags.c
+++ b/src/feature/dirauth/voteflags.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,7 @@
 #include "core/or/policies.h"
 #include "feature/dirauth/bwauth.h"
 #include "feature/dirauth/reachability.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
@@ -27,8 +28,10 @@
 #include "feature/relay/router.h"
 #include "feature/stats/rephist.h"
 
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerlist_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
 
 #include "lib/container/order.h"
@@ -95,7 +98,7 @@ real_uptime(const routerinfo_t *router, time_t now)
  */
 static int
 dirserv_thinks_router_is_unreliable(time_t now,
-                                    routerinfo_t *router,
+                                    const routerinfo_t *router,
                                     int need_uptime, int need_capacity)
 {
   if (need_uptime) {
@@ -144,7 +147,7 @@ router_is_active(const routerinfo_t *ri, const node_t *node, time_t now)
    * if TestingTorNetwork, and TestingMinExitFlagThreshold is non-zero */
   if (!ri->bandwidthcapacity) {
     if (get_options()->TestingTorNetwork) {
-      if (get_options()->TestingMinExitFlagThreshold > 0) {
+      if (dirauth_get_options()->TestingMinExitFlagThreshold > 0) {
         /* If we're in a TestingTorNetwork, and TestingMinExitFlagThreshold is,
          * then require bandwidthcapacity */
         return 0;
@@ -174,14 +177,14 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
   long uptime;
 
   /* If we haven't been running for at least
-   * get_options()->MinUptimeHidServDirectoryV2 seconds, we can't
+   * MinUptimeHidServDirectoryV2 seconds, we can't
    * have accurate data telling us a relay has been up for at least
    * that long. We also want to allow a bit of slack: Reachability
    * tests aren't instant. If we haven't been running long enough,
    * trust the relay. */
 
   if (get_uptime() >
-      get_options()->MinUptimeHidServDirectoryV2 * 1.1)
+      dirauth_get_options()->MinUptimeHidServDirectoryV2 * 1.1)
     uptime = MIN(rep_hist_get_uptime(router->cache_info.identity_digest, now),
                  real_uptime(router, now));
   else
@@ -190,7 +193,7 @@ dirserv_thinks_router_is_hs_dir(const routerinfo_t *router,
   return (router->wants_to_be_hs_dir &&
           router->supports_tunnelled_dir_requests &&
           node->is_stable && node->is_fast &&
-          uptime >= get_options()->MinUptimeHidServDirectoryV2 &&
+          uptime >= dirauth_get_options()->MinUptimeHidServDirectoryV2 &&
           router_is_active(router, node, now));
 }
 
@@ -213,9 +216,10 @@ router_counts_toward_thresholds(const node_t *node, time_t now,
     dirserv_has_measured_bw(node->identity);
   uint64_t min_bw_kb = ABSOLUTE_MIN_BW_VALUE_TO_CONSIDER_KB;
   const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
 
   if (options->TestingTorNetwork) {
-    min_bw_kb = (int64_t)options->TestingMinExitFlagThreshold / 1000;
+    min_bw_kb = (int64_t)dirauth_options->TestingMinExitFlagThreshold / 1000;
   }
 
   return node->ri && router_is_active(node->ri, node, now) &&
@@ -238,14 +242,15 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
   uint32_t *uptimes, *bandwidths_kb, *bandwidths_excluding_exits_kb;
   long *tks;
   double *mtbfs, *wfus;
-  smartlist_t *nodelist;
+  const smartlist_t *nodelist;
   time_t now = time(NULL);
   const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
 
   /* Require mbw? */
   int require_mbw =
     (dirserv_get_last_n_measured_bws() >
-     options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
+     dirauth_options->MinMeasuredBWsForAuthToIgnoreAdvertised) ? 1 : 0;
 
   /* initialize these all here, in case there are no routers */
   stable_uptime = 0;
@@ -337,7 +342,7 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
       ABSOLUTE_MIN_VALUE_FOR_FAST_FLAG,
       INT32_MAX);
     if (options->TestingTorNetwork) {
-      min_fast = (int32_t)options->TestingMinFastFlagThreshold;
+      min_fast = (int32_t)dirauth_options->TestingMinFastFlagThreshold;
     }
     max_fast = networkstatus_get_param(NULL, "FastFlagMaxThreshold",
                                        INT32_MAX, min_fast, INT32_MAX);
@@ -351,9 +356,11 @@ dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil)
   }
   /* Protect sufficiently fast nodes from being pushed out of the set
    * of Fast nodes. */
-  if (options->AuthDirFastGuarantee &&
-      fast_bandwidth_kb > options->AuthDirFastGuarantee/1000)
-    fast_bandwidth_kb = (uint32_t)options->AuthDirFastGuarantee/1000;
+  {
+    const uint64_t fast_opt = dirauth_get_options()->AuthDirFastGuarantee;
+    if (fast_opt && fast_bandwidth_kb > fast_opt / 1000)
+      fast_bandwidth_kb = (uint32_t)(fast_opt / 1000);
+  }
 
   /* Now that we have a time-known that 7/8 routers are known longer than,
    * fill wfus with the wfu of every such "familiar" router. */
@@ -427,7 +434,7 @@ dirserv_get_flag_thresholds_line(void)
 {
   char *result=NULL;
   const int measured_threshold =
-    get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
+    dirauth_get_options()->MinMeasuredBWsForAuthToIgnoreAdvertised;
   const int enough_measured_bw =
     dirserv_get_last_n_measured_bws() > measured_threshold;
 
@@ -454,8 +461,9 @@ dirserv_get_flag_thresholds_line(void)
 int
 running_long_enough_to_decide_unreachable(void)
 {
-  return time_of_process_start
-    + get_options()->TestingAuthDirTimeToLearnReachability < approx_time();
+  const dirauth_options_t *opts = dirauth_get_options();
+  return time_of_process_start +
+    opts->TestingAuthDirTimeToLearnReachability < approx_time();
 }
 
 /** Each server needs to have passed a reachability test no more
@@ -480,6 +488,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
    */
   int answer;
   const or_options_t *options = get_options();
+  const dirauth_options_t *dirauth_options = dirauth_get_options();
   node_t *node = node_get_mutable_by_id(router->cache_info.identity_digest);
   tor_assert(node);
 
@@ -506,7 +515,7 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
        IPv6 OR port since that'd kill all dual stack relays until a
        majority of the dir auths have IPv6 connectivity. */
     answer = (now < node->last_reachable + REACHABLE_TIMEOUT &&
-              (options->AuthDirHasIPv6Connectivity != 1 ||
+              (dirauth_options->AuthDirHasIPv6Connectivity != 1 ||
                tor_addr_is_null(&router->ipv6_addr) ||
                now < node->last_reachable6 + REACHABLE_TIMEOUT));
   }
@@ -531,42 +540,49 @@ dirserv_set_router_is_running(routerinfo_t *router, time_t now)
   node->is_running = answer;
 }
 
-/** Extract status information from <b>ri</b> and from other authority
- * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
- * set.
- *
- * We assume that ri-\>is_running has already been set, e.g. by
- *   dirserv_set_router_is_running(ri, now);
+/* Check <b>node</b> and <b>ri</b> on whether or not we should publish a
+ * relay's IPv6 addresses. */
+static int
+should_publish_node_ipv6(const node_t *node, const routerinfo_t *ri,
+                         time_t now)
+{
+  const dirauth_options_t *options = dirauth_get_options();
+
+  return options->AuthDirHasIPv6Connectivity == 1 &&
+    !tor_addr_is_null(&ri->ipv6_addr) &&
+    ((node->last_reachable6 >= now - REACHABLE_TIMEOUT) ||
+     router_is_me(ri));
+}
+
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>, as per
+ * <b>set_routerstatus_from_routerinfo</b>.  Additionally, sets information
+ * in from the authority subsystem.
  */
 void
-set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                 node_t *node,
-                                 routerinfo_t *ri,
-                                 time_t now,
-                                 int listbadexits)
+dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                         node_t *node,
+                                         const routerinfo_t *ri,
+                                         time_t now,
+                                         int listbadexits)
 {
   const or_options_t *options = get_options();
   uint32_t routerbw_kb = dirserv_get_credible_bandwidth_kb(ri);
 
-  memset(rs, 0, sizeof(routerstatus_t));
-
-  rs->is_authority =
-    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
-
-  /* Already set by compute_performance_thresholds. */
-  rs->is_exit = node->is_exit;
-  rs->is_stable = node->is_stable =
-    !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
-  rs->is_fast = node->is_fast =
-    !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
-  rs->is_flagged_running = node->is_running; /* computed above */
+  /* Set these flags so that set_routerstatus_from_routerinfo can copy them.
+   */
+  node->is_stable = !dirserv_thinks_router_is_unreliable(now, ri, 1, 0);
+  node->is_fast = !dirserv_thinks_router_is_unreliable(now, ri, 0, 1);
+  node->is_hs_dir = dirserv_thinks_router_is_hs_dir(ri, node, now);
 
-  rs->is_valid = node->is_valid;
+  set_routerstatus_from_routerinfo(rs, node, ri);
 
+  /* Override rs->is_possible_guard. */
+  const uint64_t bw_opt = dirauth_get_options()->AuthDirGuardBWGuarantee;
   if (node->is_fast && node->is_stable &&
       ri->supports_tunnelled_dir_requests &&
-      ((options->AuthDirGuardBWGuarantee &&
-        routerbw_kb >= options->AuthDirGuardBWGuarantee/1000) ||
+      ((bw_opt && routerbw_kb >= bw_opt / 1000) ||
        routerbw_kb >= MIN(guard_bandwidth_including_exits_kb,
                           guard_bandwidth_excluding_exits_kb))) {
     long tk = rep_hist_get_weighted_time_known(
@@ -578,29 +594,16 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
     rs->is_possible_guard = 0;
   }
 
+  /* Override rs->is_bad_exit */
   rs->is_bad_exit = listbadexits && node->is_bad_exit;
-  rs->is_hs_dir = node->is_hs_dir =
-    dirserv_thinks_router_is_hs_dir(ri, node, now);
-
-  rs->is_named = rs->is_unnamed = 0;
-
-  rs->published_on = ri->cache_info.published_on;
-  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
-  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
-         DIGEST_LEN);
-  rs->addr = ri->addr;
-  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
-  rs->or_port = ri->or_port;
-  rs->dir_port = ri->dir_port;
-  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
-  if (options->AuthDirHasIPv6Connectivity == 1 &&
-      !tor_addr_is_null(&ri->ipv6_addr) &&
-      node->last_reachable6 >= now - REACHABLE_TIMEOUT) {
-    /* We're configured as having IPv6 connectivity. There's an IPv6
-       OR port and it's reachable so copy it to the routerstatus.  */
-    tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
-    rs->ipv6_orport = ri->ipv6_orport;
-  } else {
+
+  /* Set rs->is_staledesc. */
+  rs->is_staledesc =
+    (ri->cache_info.published_on + DESC_IS_STALE_INTERVAL) < now;
+
+  if (! should_publish_node_ipv6(node, ri, now)) {
+    /* We're not configured as having IPv6 connectivity or the node isn't:
+     * zero its IPv6 information. */
     tor_addr_make_null(&rs->ipv6_addr, AF_INET6);
     rs->ipv6_orport = 0;
   }
@@ -617,9 +620,9 @@ set_routerstatus_from_routerinfo(routerstatus_t *rs,
 STATIC void
 dirserv_set_routerstatus_testing(routerstatus_t *rs)
 {
-  const or_options_t *options = get_options();
+  const dirauth_options_t *options = dirauth_get_options();
 
-  tor_assert(options->TestingTorNetwork);
+  tor_assert(get_options()->TestingTorNetwork);
 
   if (routerset_contains_routerstatus(options->TestingDirAuthVoteExit,
                                       rs, 0)) {
@@ -642,3 +645,20 @@ dirserv_set_routerstatus_testing(routerstatus_t *rs)
     rs->is_hs_dir = 0;
   }
 }
+
+/** Use dirserv_set_router_is_running() to set bridges as running if they're
+ * reachable.
+ *
+ * This function is called from set_bridge_running_callback() when running as
+ * a bridge authority.
+ */
+void
+dirserv_set_bridges_running(time_t now)
+{
+  routerlist_t *rl = router_get_routerlist();
+
+  SMARTLIST_FOREACH_BEGIN(rl->routers, routerinfo_t *, ri) {
+    if (ri->purpose == ROUTER_PURPOSE_BRIDGE)
+      dirserv_set_router_is_running(ri, now);
+  } SMARTLIST_FOREACH_END(ri);
+}
diff --git a/src/feature/dirauth/voteflags.h b/src/feature/dirauth/voteflags.h
index aa7b6ed082..91f3854573 100644
--- a/src/feature/dirauth/voteflags.h
+++ b/src/feature/dirauth/voteflags.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,20 +12,28 @@
 #ifndef TOR_VOTEFLAGS_H
 #define TOR_VOTEFLAGS_H
 
+#ifdef HAVE_MODULE_DIRAUTH
 void dirserv_set_router_is_running(routerinfo_t *router, time_t now);
 char *dirserv_get_flag_thresholds_line(void);
 void dirserv_compute_bridge_flag_thresholds(void);
 int running_long_enough_to_decide_unreachable(void);
 
-void set_routerstatus_from_routerinfo(routerstatus_t *rs,
-                                      node_t *node,
-                                      routerinfo_t *ri, time_t now,
-                                      int listbadexits);
+void dirauth_set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                              node_t *node,
+                                              const routerinfo_t *ri,
+                                              time_t now,
+                                              int listbadexits);
 
 void dirserv_compute_performance_thresholds(digestmap_t *omit_as_sybil);
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
+
+void dirserv_set_bridges_running(time_t now);
 
 #ifdef VOTEFLAGS_PRIVATE
+/** Any descriptor older than this age causes the authorities to set the
+ * StaleDesc flag. */
+#define DESC_IS_STALE_INTERVAL (18*60*60)
 STATIC void dirserv_set_routerstatus_testing(routerstatus_t *rs);
-#endif
+#endif /* defined(VOTEFLAGS_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_VOTEFLAGS_H) */
diff --git a/src/feature/dircommon/voting_schedule.c b/src/feature/dirauth/voting_schedule.c
index 0a7476eda7..efc4a0b316 100644
--- a/src/feature/dircommon/voting_schedule.c
+++ b/src/feature/dirauth/voting_schedule.c
@@ -1,15 +1,13 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file voting_schedule.c
- * \brief This file contains functions that are from the directory authority
- *        subsystem related to voting specifically but used by many part of
- *        tor. The full feature is built as part of the dirauth module.
+ * \brief Compute information about our voting schedule as a directory
+ *    authority.
  **/
 
-#define VOTING_SCHEDULE_PRIVATE
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -21,55 +19,11 @@
  * Vote scheduling
  * ===== */
 
-/** Return the start of the next interval of size <b>interval</b> (in
- * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
- * starts a fresh interval, and if the last interval of a day would be
- * truncated to less than half its size, it is rolled into the
- * previous interval. */
-time_t
-voting_schedule_get_start_of_next_interval(time_t now, int interval,
-                                           int offset)
-{
-  struct tm tm;
-  time_t midnight_today=0;
-  time_t midnight_tomorrow;
-  time_t next;
-
-  tor_gmtime_r(&now, &tm);
-  tm.tm_hour = 0;
-  tm.tm_min = 0;
-  tm.tm_sec = 0;
-
-  if (tor_timegm(&tm, &midnight_today) < 0) {
-    // LCOV_EXCL_START
-    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
-    // LCOV_EXCL_STOP
-  }
-  midnight_tomorrow = midnight_today + (24*60*60);
-
-  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
-
-  /* Intervals never cross midnight. */
-  if (next > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  /* If the interval would only last half as long as it's supposed to, then
-   * skip over to the next day. */
-  if (next + interval/2 > midnight_tomorrow)
-    next = midnight_tomorrow;
-
-  next += offset;
-  if (next - interval > now)
-    next -= interval;
-
-  return next;
-}
-
 /* Populate and return a new voting_schedule_t that can be used to schedule
  * voting. The object is allocated on the heap and it's the responsibility of
  * the caller to free it. Can't fail. */
 static voting_schedule_t *
-get_voting_schedule(const or_options_t *options, time_t now, int severity)
+create_voting_schedule(const or_options_t *options, time_t now, int severity)
 {
   int interval, vote_delay, dist_delay;
   time_t start;
@@ -96,14 +50,15 @@ get_voting_schedule(const or_options_t *options, time_t now, int severity)
   }
 
   tor_assert(interval > 0);
+  new_voting_schedule->interval = interval;
 
   if (vote_delay + dist_delay > interval/2)
     vote_delay = dist_delay = interval / 4;
 
   start = new_voting_schedule->interval_starts =
-    voting_schedule_get_start_of_next_interval(now,interval,
+    voting_sched_get_start_of_interval_after(now,interval,
                                       options->TestingV3AuthVotingStartOffset);
-  end = voting_schedule_get_start_of_next_interval(start+1, interval,
+  end = voting_sched_get_start_of_interval_after(start+1, interval,
                                       options->TestingV3AuthVotingStartOffset);
 
   tor_assert(end > start);
@@ -140,9 +95,13 @@ voting_schedule_free_(voting_schedule_t *voting_schedule_to_free)
 
 voting_schedule_t voting_schedule;
 
-/* Using the time <b>now</b>, return the next voting valid-after time. */
-time_t
-voting_schedule_get_next_valid_after_time(void)
+/**
+ * Return the current voting schedule, recreating it if necessary.
+ *
+ * Dirauth only.
+ **/
+static const voting_schedule_t *
+dirauth_get_voting_schedule(void)
 {
   time_t now = approx_time();
   bool need_to_recalculate_voting_schedule = false;
@@ -150,7 +109,7 @@ voting_schedule_get_next_valid_after_time(void)
   /* This is a safe guard in order to make sure that the voting schedule
    * static object is at least initialized. Using this function with a zeroed
    * voting schedule can lead to bugs. */
-  if (tor_mem_is_zero((const char *) &voting_schedule,
+  if (fast_mem_is_zero((const char *) &voting_schedule,
                       sizeof(voting_schedule))) {
     need_to_recalculate_voting_schedule = true;
     goto done; /* no need for next check if we have to recalculate anyway */
@@ -168,27 +127,62 @@ voting_schedule_get_next_valid_after_time(void)
 
  done:
   if (need_to_recalculate_voting_schedule) {
-    voting_schedule_recalculate_timing(get_options(), approx_time());
+    dirauth_sched_recalculate_timing(get_options(), approx_time());
     voting_schedule.created_on_demand = 1;
   }
 
-  return voting_schedule.interval_starts;
+  return &voting_schedule;
+}
+
+/** Return the next voting valid-after time.
+ *
+ * Dirauth only. */
+time_t
+dirauth_sched_get_next_valid_after_time(void)
+{
+  return dirauth_get_voting_schedule()->interval_starts;
+}
+
+/**
+ * Return our best idea of what the valid-after time for the _current_
+ * consensus, whether we have one or not.
+ *
+ * Dirauth only.
+ **/
+time_t
+dirauth_sched_get_cur_valid_after_time(void)
+{
+  const voting_schedule_t *sched = dirauth_get_voting_schedule();
+  time_t next_start = sched->interval_starts;
+  int interval = sched->interval;
+  int offset = get_options()->TestingV3AuthVotingStartOffset;
+  return voting_sched_get_start_of_interval_after(next_start - interval - 1,
+                                                  interval,
+                                                  offset);
+}
+
+/** Return the voting interval that we are configured to use.
+ *
+ * Dirauth only. */
+int
+dirauth_sched_get_configured_interval(void)
+{
+  return get_options()->V3AuthVotingInterval;
 }
 
 /** Set voting_schedule to hold the timing for the next vote we should be
  * doing. All type of tor do that because HS subsystem needs the timing as
  * well to function properly. */
 void
-voting_schedule_recalculate_timing(const or_options_t *options, time_t now)
+dirauth_sched_recalculate_timing(const or_options_t *options, time_t now)
 {
   voting_schedule_t *new_voting_schedule;
 
   /* get the new voting schedule */
-  new_voting_schedule = get_voting_schedule(options, now, LOG_INFO);
+  new_voting_schedule = create_voting_schedule(options, now, LOG_INFO);
   tor_assert(new_voting_schedule);
 
   /* Fill in the global static struct now */
   memcpy(&voting_schedule, new_voting_schedule, sizeof(voting_schedule));
   voting_schedule_free(new_voting_schedule);
 }
-
diff --git a/src/feature/dircommon/voting_schedule.h b/src/feature/dirauth/voting_schedule.h
index bafd81184e..9e2ac29c75 100644
--- a/src/feature/dircommon/voting_schedule.h
+++ b/src/feature/dirauth/voting_schedule.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,8 @@
 
 #include "core/or/or.h"
 
+#ifdef HAVE_MODULE_DIRAUTH
+
 /** Scheduling information for a voting interval. */
 typedef struct {
   /** When do we generate and distribute our vote for this interval? */
@@ -26,15 +28,18 @@ typedef struct {
   /** When do we publish the consensus? */
   time_t interval_starts;
 
-  /* True iff we have generated and distributed our vote. */
+  /** Our computed dirauth interval */
+  int interval;
+
+  /** True iff we have generated and distributed our vote. */
   int have_voted;
-  /* True iff we've requested missing votes. */
+  /** True iff we've requested missing votes. */
   int have_fetched_missing_votes;
-  /* True iff we have built a consensus and sent the signatures around. */
+  /** True iff we have built a consensus and sent the signatures around. */
   int have_built_consensus;
-  /* True iff we've fetched missing signatures. */
+  /** True iff we've fetched missing signatures. */
   int have_fetched_missing_signatures;
-  /* True iff we have published our consensus. */
+  /** True iff we have published our consensus. */
   int have_published_consensus;
 
   /* True iff this voting schedule was set on demand meaning not through the
@@ -53,13 +58,36 @@ typedef struct {
 
 extern voting_schedule_t voting_schedule;
 
-void voting_schedule_recalculate_timing(const or_options_t *options,
+void dirauth_sched_recalculate_timing(const or_options_t *options,
                                         time_t now);
 
-time_t voting_schedule_get_start_of_next_interval(time_t now,
-                                                  int interval,
-                                                  int offset);
-time_t voting_schedule_get_next_valid_after_time(void);
+time_t dirauth_sched_get_next_valid_after_time(void);
+time_t dirauth_sched_get_cur_valid_after_time(void);
+int dirauth_sched_get_configured_interval(void);
+
+#else /* !defined(HAVE_MODULE_DIRAUTH) */
+
+#define dirauth_sched_recalculate_timing(opt,now) \
+  ((void)(opt), (void)(now))
 
-#endif /* TOR_VOTING_SCHEDULE_H */
+static inline time_t
+dirauth_sched_get_next_valid_after_time(void)
+{
+  tor_assert_unreached();
+  return 0;
+}
+static inline time_t
+dirauth_sched_get_cur_valid_after_time(void)
+{
+  tor_assert_unreached();
+  return 0;
+}
+static inline int
+dirauth_sched_get_configured_interval(void)
+{
+  tor_assert_unreached();
+  return 1;
+}
+#endif /* defined(HAVE_MODULE_DIRAUTH) */
 
+#endif /* !defined(TOR_VOTING_SCHEDULE_H) */
diff --git a/src/feature/dircache/.may_include b/src/feature/dircache/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dircache/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dircache/cached_dir_st.h b/src/feature/dircache/cached_dir_st.h
index 71dca8c3a2..ede1d028da 100644
--- a/src/feature/dircache/cached_dir_st.h
+++ b/src/feature/dircache/cached_dir_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file cached_dir_st.h
+ * @brief Cached large directory object structure.
+ **/
+
 #ifndef CACHED_DIR_ST_H
 #define CACHED_DIR_ST_H
 
@@ -21,5 +26,4 @@ struct cached_dir_t {
   int refcnt; /**< Reference count for this cached_dir_t. */
 };
 
-#endif
-
+#endif /* !defined(CACHED_DIR_ST_H) */
diff --git a/src/feature/dircache/conscache.c b/src/feature/dircache/conscache.c
index cf4fe8701d..2a831aa447 100644
--- a/src/feature/dircache/conscache.c
+++ b/src/feature/dircache/conscache.c
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file conscache.c
+ * @brief Consensus and diff on-disk cache.
+ **/
+
 #include "core/or/or.h"
 
 #include "app/config/config.h"
@@ -92,7 +97,7 @@ consensus_cache_open(const char *subdir, int max_entries)
    */
 #define VERY_LARGE_STORAGEDIR_LIMIT (1000*1000)
   storagedir_max_entries = VERY_LARGE_STORAGEDIR_LIMIT;
-#else /* !(defined(MUST_UNMAP_TO_UNLINK)) */
+#else /* !defined(MUST_UNMAP_TO_UNLINK) */
   /* Otherwise, we can just tell the storagedir to use the same limits
    * as this cache. */
   storagedir_max_entries = max_entries;
@@ -127,13 +132,22 @@ consensus_cache_may_overallocate(consensus_cache_t *cache)
 #endif
 }
 
+// HACK: GCC on Appveyor hates that we may assert before returning. Work around
+// the error.
+#ifdef _WIN32
+#ifndef COCCI
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsuggest-attribute=noreturn"
+#endif
+#endif /* defined(_WIN32) */
+
 /**
  * Tell the sandbox (if any) configured by <b>cfg</b> to allow the
  * operations that <b>cache</b> will need.
  */
 int
 consensus_cache_register_with_sandbox(consensus_cache_t *cache,
-                                      struct sandbox_cfg_elem **cfg)
+                                      struct sandbox_cfg_elem_t **cfg)
 {
 #ifdef MUST_UNMAP_TO_UNLINK
   /* Our Linux sandbox doesn't support huge file lists like the one that would
@@ -151,6 +165,12 @@ consensus_cache_register_with_sandbox(consensus_cache_t *cache,
   return storage_dir_register_with_sandbox(cache->dir, cfg);
 }
 
+#ifdef _WIN32
+#ifndef COCCI
+#pragma GCC diagnostic pop
+#endif
+#endif
+
 /**
  * Helper: clear all entries from <b>cache</b> (but do not delete
  * any that aren't marked for removal
@@ -246,7 +266,7 @@ consensus_cache_find_first(consensus_cache_t *cache,
 }
 
 /**
- * Given a <b>cache</b>, add every entry to <b>out<b> for which
+ * Given a <b>cache</b>, add every entry to <b>out</b> for which
  * <b>key</b>=<b>value</b>.  If <b>key</b> is NULL, add every entry.
  *
  * Do not add any entry that has been marked for removal.
diff --git a/src/feature/dircache/conscache.h b/src/feature/dircache/conscache.h
index d848e57617..ace5908e40 100644
--- a/src/feature/dircache/conscache.h
+++ b/src/feature/dircache/conscache.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file conscache.h
+ * @brief Header for conscache.c
+ **/
+
 #ifndef TOR_CONSCACHE_H
 #define TOR_CONSCACHE_H
 
@@ -9,6 +14,8 @@
 typedef struct consensus_cache_entry_t consensus_cache_entry_t;
 typedef struct consensus_cache_t consensus_cache_t;
 
+struct config_line_t;
+
 HANDLE_DECL(consensus_cache_entry, consensus_cache_entry_t, )
 #define consensus_cache_entry_handle_free(h)    \
   FREE_AND_NULL(consensus_cache_entry_handle_t, \
@@ -18,10 +25,10 @@ consensus_cache_t *consensus_cache_open(const char *subdir, int max_entries);
 void consensus_cache_free_(consensus_cache_t *cache);
 #define consensus_cache_free(cache) \
   FREE_AND_NULL(consensus_cache_t, consensus_cache_free_, (cache))
-struct sandbox_cfg_elem;
+struct sandbox_cfg_elem_t;
 int consensus_cache_may_overallocate(consensus_cache_t *cache);
 int consensus_cache_register_with_sandbox(consensus_cache_t *cache,
-                                          struct sandbox_cfg_elem **cfg);
+                                          struct sandbox_cfg_elem_t **cfg);
 void consensus_cache_unmap_lazy(consensus_cache_t *cache, time_t cutoff);
 void consensus_cache_delete_pending(consensus_cache_t *cache,
                                     int force);
diff --git a/src/feature/dircache/consdiffmgr.c b/src/feature/dircache/consdiffmgr.c
index 025361fa60..10590cd6d2 100644
--- a/src/feature/dircache/consdiffmgr.c
+++ b/src/feature/dircache/consdiffmgr.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file consdiffmsr.c
+ * \file consdiffmgr.c
  *
  * \brief consensus diff manager functions
  *
@@ -189,6 +189,7 @@ static consdiff_cfg_t consdiff_cfg = {
 
 static int consdiffmgr_ensure_space_for_files(int n);
 static int consensus_queue_compression_work(const char *consensus,
+                                            size_t consensus_len,
                                             const networkstatus_t *as_parsed);
 static int consensus_diff_queue_diff_work(consensus_cache_entry_t *diff_from,
                                           consensus_cache_entry_t *diff_to);
@@ -217,9 +218,9 @@ cdm_diff_eq(const cdm_diff_t *diff1, const cdm_diff_t *diff2)
     diff1->compress_method == diff2->compress_method;
 }
 
-HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq)
+HT_PROTOTYPE(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq);
 HT_GENERATE2(cdm_diff_ht, cdm_diff_t, node, cdm_diff_hash, cdm_diff_eq,
-             0.6, tor_reallocarray, tor_free_)
+             0.6, tor_reallocarray, tor_free_);
 
 #define cdm_diff_free(diff) \
   FREE_AND_NULL(cdm_diff_t, cdm_diff_free_, (diff))
@@ -509,8 +510,25 @@ get_max_age_to_cache(void)
                                         MAX_MAX_AGE_TO_CACHE);
 }
 
+#ifdef TOR_UNIT_TESTS
+/** As consdiffmgr_add_consensus, but requires a nul-terminated input. For
+ * testing. */
+int
+consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                  const networkstatus_t *as_parsed)
+{
+  size_t len = strlen(consensus);
+  /* make a non-nul-terminated copy so that we can have a better chance
+   * of catching errors. */
+  char *ctmp = tor_memdup(consensus, len);
+  int r = consdiffmgr_add_consensus(ctmp, len, as_parsed);
+  tor_free(ctmp);
+  return r;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /**
- * Given a string containing a networkstatus consensus, and the results of
+ * Given a buffer containing a networkstatus consensus, and the results of
  * having parsed that consensus, add that consensus to the cache if it is not
  * already present and not too old.  Create new consensus diffs from or to
  * that consensus as appropriate.
@@ -519,6 +537,7 @@ get_max_age_to_cache(void)
  */
 int
 consdiffmgr_add_consensus(const char *consensus,
+                          size_t consensus_len,
                           const networkstatus_t *as_parsed)
 {
   if (BUG(consensus == NULL) || BUG(as_parsed == NULL))
@@ -544,7 +563,7 @@ consdiffmgr_add_consensus(const char *consensus,
   }
 
   /* We don't have it. Add it to the cache. */
-  return consensus_queue_compression_work(consensus, as_parsed);
+  return consensus_queue_compression_work(consensus, consensus_len, as_parsed);
 }
 
 /**
@@ -825,7 +844,7 @@ consdiffmgr_configure(const consdiff_cfg_t *cfg)
  * operations that the consensus diff manager will need.
  */
 int
-consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg)
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
 {
   return consensus_cache_register_with_sandbox(cdm_cache_get(), cfg);
 }
@@ -1274,7 +1293,7 @@ typedef struct compressed_result_t {
 
 /**
  * Compress the bytestring <b>input</b> of length <b>len</b> using the
- * <n>n_methods</b> compression methods listed in the array <b>methods</b>.
+ * <b>n_methods</b> compression methods listed in the array <b>methods</b>.
  *
  * For each successful compression, set the fields in the <b>results_out</b>
  * array in the position corresponding to the compression method. Use
@@ -1387,19 +1406,21 @@ typedef struct consensus_diff_worker_job_t {
 } consensus_diff_worker_job_t;
 
 /** Given a consensus_cache_entry_t, check whether it has a label claiming
- * that it was compressed.  If so, uncompress its contents into <b>out</b> and
- * set <b>outlen</b> to hold their size.  If not, just copy the body into
- * <b>out</b> and set <b>outlen</b> to its length.  Return 0 on success,
- * -1 on failure.
- *
- * In all cases, the output is nul-terminated. */
+ * that it was compressed.  If so, uncompress its contents into *<b>out</b> and
+ * set <b>outlen</b> to hold their size, and set *<b>owned_out</b> to a pointer
+ * that the caller will need to free.  If not, just set *<b>out</b> and
+ * <b>outlen</b> to its extent in memory.  Return 0 on success, -1 on failure.
+ **/
 STATIC int
-uncompress_or_copy(char **out, size_t *outlen,
-                   consensus_cache_entry_t *ent)
+uncompress_or_set_ptr(const char **out, size_t *outlen,
+                      char **owned_out,
+                      consensus_cache_entry_t *ent)
 {
   const uint8_t *body;
   size_t bodylen;
 
+  *owned_out = NULL;
+
   if (consensus_cache_entry_get_body(ent, &body, &bodylen) < 0)
     return -1;
 
@@ -1410,8 +1431,17 @@ uncompress_or_copy(char **out, size_t *outlen,
   if (lv_compression)
     method = compression_method_get_by_name(lv_compression);
 
-  return tor_uncompress(out, outlen, (const char *)body, bodylen,
+  int rv;
+  if (method == NO_METHOD) {
+    *out = (const char *)body;
+    *outlen = bodylen;
+    rv = 0;
+  } else {
+    rv = tor_uncompress(owned_out, outlen, (const char *)body, bodylen,
                         method, 1, LOG_WARN);
+    *out = *owned_out;
+  }
+  return rv;
 }
 
 /**
@@ -1478,16 +1508,17 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
 
   char *consensus_diff;
   {
-    char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    const char *diff_from_nt = NULL, *diff_to_nt = NULL;
+    char *owned1 = NULL, *owned2 = NULL;
     size_t diff_from_nt_len, diff_to_nt_len;
 
-    if (uncompress_or_copy(&diff_from_nt, &diff_from_nt_len,
-                           job->diff_from) < 0) {
+    if (uncompress_or_set_ptr(&diff_from_nt, &diff_from_nt_len, &owned1,
+                              job->diff_from) < 0) {
       return WQ_RPL_REPLY;
     }
-    if (uncompress_or_copy(&diff_to_nt, &diff_to_nt_len,
-                           job->diff_to) < 0) {
-      tor_free(diff_from_nt);
+    if (uncompress_or_set_ptr(&diff_to_nt, &diff_to_nt_len, &owned2,
+                              job->diff_to) < 0) {
+      tor_free(owned1);
       return WQ_RPL_REPLY;
     }
     tor_assert(diff_from_nt);
@@ -1496,9 +1527,12 @@ consensus_diff_worker_threadfn(void *state_, void *work_)
     // XXXX ugh; this is going to calculate the SHA3 of both its
     // XXXX inputs again, even though we already have that. Maybe it's time
     // XXXX to change the API here?
-    consensus_diff = consensus_diff_generate(diff_from_nt, diff_to_nt);
-    tor_free(diff_from_nt);
-    tor_free(diff_to_nt);
+    consensus_diff = consensus_diff_generate(diff_from_nt,
+                                             diff_from_nt_len,
+                                             diff_to_nt,
+                                             diff_to_nt_len);
+    tor_free(owned1);
+    tor_free(owned2);
   }
   if (!consensus_diff) {
     /* Couldn't generate consensus; we'll leave the reply blank. */
@@ -1746,8 +1780,8 @@ consensus_compress_worker_threadfn(void *state_, void *work_)
                           (const uint8_t *)consensus, bodylen);
   {
     const char *start, *end;
-    if (router_get_networkstatus_v3_signed_boundaries(consensus,
-                                                        &start, &end) < 0) {
+    if (router_get_networkstatus_v3_signed_boundaries(consensus, bodylen,
+                                                      &start, &end) < 0) {
       start = consensus;
       end = consensus+bodylen;
     }
@@ -1811,14 +1845,15 @@ static int background_compression = 0;
  */
 static int
 consensus_queue_compression_work(const char *consensus,
+                                 size_t consensus_len,
                                  const networkstatus_t *as_parsed)
 {
   tor_assert(consensus);
   tor_assert(as_parsed);
 
   consensus_compress_worker_job_t *job = tor_malloc_zero(sizeof(*job));
-  job->consensus = tor_strdup(consensus);
-  job->consensus_len = strlen(consensus);
+  job->consensus = tor_memdup_nulterm(consensus, consensus_len);
+  job->consensus_len = strlen(job->consensus);
   job->flavor = as_parsed->flavor;
 
   char va_str[ISO_TIME_LEN+1];
diff --git a/src/feature/dircache/consdiffmgr.h b/src/feature/dircache/consdiffmgr.h
index 39e8fa31cb..27b8165e94 100644
--- a/src/feature/dircache/consdiffmgr.h
+++ b/src/feature/dircache/consdiffmgr.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file consdiffmgr.h
+ * @brief Header for consdiffmgr.c
+ **/
+
 #ifndef TOR_CONSDIFFMGR_H
 #define TOR_CONSDIFFMGR_H
 
@@ -22,6 +27,7 @@ typedef struct consdiff_cfg_t {
 struct consensus_cache_entry_t; // from conscache.h
 
 int consdiffmgr_add_consensus(const char *consensus,
+                              size_t consensus_len,
                               const networkstatus_t *as_parsed);
 
 consdiff_status_t consdiffmgr_find_consensus(
@@ -54,22 +60,30 @@ void consdiffmgr_rescan(void);
 int consdiffmgr_cleanup(void);
 void consdiffmgr_enable_background_compression(void);
 void consdiffmgr_configure(const consdiff_cfg_t *cfg);
-struct sandbox_cfg_elem;
-int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem **cfg);
+struct sandbox_cfg_elem_t;
+int consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg);
 void consdiffmgr_free_all(void);
 int consdiffmgr_validate(void);
 
 #ifdef CONSDIFFMGR_PRIVATE
+struct consensus_cache_t;
+struct consensus_cache_entry_t;
 STATIC unsigned n_diff_compression_methods(void);
 STATIC unsigned n_consensus_compression_methods(void);
-STATIC consensus_cache_t *cdm_cache_get(void);
-STATIC consensus_cache_entry_t *cdm_cache_lookup_consensus(
+STATIC struct consensus_cache_t *cdm_cache_get(void);
+STATIC struct consensus_cache_entry_t *cdm_cache_lookup_consensus(
                           consensus_flavor_t flavor, time_t valid_after);
 STATIC int cdm_entry_get_sha3_value(uint8_t *digest_out,
-                                    consensus_cache_entry_t *ent,
+                                    struct consensus_cache_entry_t *ent,
                                     const char *label);
-STATIC int uncompress_or_copy(char **out, size_t *outlen,
-                              consensus_cache_entry_t *ent);
+STATIC int uncompress_or_set_ptr(const char **out, size_t *outlen,
+                                 char **owned_out,
+                                 struct consensus_cache_entry_t *ent);
 #endif /* defined(CONSDIFFMGR_PRIVATE) */
 
+#ifdef TOR_UNIT_TESTS
+int consdiffmgr_add_consensus_nulterm(const char *consensus,
+                                      const networkstatus_t *as_parsed);
+#endif
+
 #endif /* !defined(TOR_CONSDIFFMGR_H) */
diff --git a/src/feature/dircache/dircache.c b/src/feature/dircache/dircache.c
index e8cb284165..ca127720f2 100644
--- a/src/feature/dircache/dircache.c
+++ b/src/feature/dircache/dircache.c
@@ -1,13 +1,19 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dircache.c
+ * @brief Cache directories and serve them to clients.
+ **/
+
 #define DIRCACHE_PRIVATE
 
 #include "core/or/or.h"
 
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/mainloop/connection.h"
 #include "core/or/relay.h"
 #include "feature/dirauth/dirvote.h"
@@ -23,6 +29,7 @@
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/relay/relay_config.h"
 #include "feature/relay/routermode.h"
 #include "feature/rend/rendcache.h"
 #include "feature/stats/geoip_stats.h"
@@ -49,7 +56,8 @@
 #define ROUTERDESC_BY_DIGEST_CACHE_LIFETIME (48*60*60)
 #define ROBOTS_CACHE_LIFETIME (24*60*60)
 #define MICRODESC_CACHE_LIFETIME (48*60*60)
-
+/* Bandwidth files change every hour. */
+#define BANDWIDTH_CACHE_LIFETIME (30*60)
 /** Parse an HTTP request string <b>headers</b> of the form
  * \verbatim
  * "\%s [http[s]://]\%s HTTP/1..."
@@ -123,7 +131,7 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
                            long cache_lifetime)
 {
   char date[RFC1123_TIME_LEN+1];
-  time_t now = time(NULL);
+  time_t now = approx_time();
   buf_t *buf = buf_new_with_capacity(1024);
 
   tor_assert(conn);
@@ -166,22 +174,16 @@ write_http_response_header_impl(dir_connection_t *conn, ssize_t length,
   buf_free(buf);
 }
 
-/** As write_http_response_header_impl, but sets encoding and content-typed
- * based on whether the response will be <b>compressed</b> or not. */
+/** As write_http_response_header_impl, but translates method into
+ * encoding */
 static void
 write_http_response_headers(dir_connection_t *conn, ssize_t length,
                             compress_method_t method,
                             const char *extra_headers, long cache_lifetime)
 {
-  const char *methodname = compression_method_get_name(method);
-  const char *doctype;
-  if (method == NO_METHOD)
-    doctype = "text/plain";
-  else
-    doctype = "application/octet-stream";
   write_http_response_header_impl(conn, length,
-                                  doctype,
-                                  methodname,
+                                  "text/plain",
+                                  compression_method_get_name(method),
                                   extra_headers,
                                   cache_lifetime);
 }
@@ -333,7 +335,7 @@ typedef struct get_handler_args_t {
  * an arguments structure, and must return 0 on success or -1 if we should
  * close the connection.
  **/
-typedef struct url_table_ent_s {
+typedef struct url_table_ent_t {
   const char *string;
   int is_prefix;
   int (*handler)(dir_connection_t *conn, const get_handler_args_t *args);
@@ -357,12 +359,15 @@ static int handle_get_robots(dir_connection_t *conn,
                                 const get_handler_args_t *args);
 static int handle_get_networkstatus_bridges(dir_connection_t *conn,
                                 const get_handler_args_t *args);
+static int handle_get_next_bandwidth(dir_connection_t *conn,
+                                     const get_handler_args_t *args);
 
 /** Table for handling GET requests. */
 static const url_table_ent_t url_table[] = {
   { "/tor/", 0, handle_get_frontpage },
   { "/tor/status-vote/current/consensus", 1, handle_get_current_consensus },
   { "/tor/status-vote/current/", 1, handle_get_status_vote },
+  { "/tor/status-vote/next/bandwidth", 0, handle_get_next_bandwidth },
   { "/tor/status-vote/next/", 1, handle_get_status_vote },
   { "/tor/micro/d/", 1, handle_get_microdesc },
   { "/tor/server/", 1, handle_get_descriptor },
@@ -475,7 +480,7 @@ static int
 handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 {
   (void) args; /* unused */
-  const char *frontpage = get_dirportfrontpage();
+  const char *frontpage = relay_get_dirportfrontpage();
 
   if (frontpage) {
     size_t dlen;
@@ -495,28 +500,47 @@ handle_get_frontpage(dir_connection_t *conn, const get_handler_args_t *args)
 }
 
 /** Warn that the cached consensus <b>consensus</b> of type
- * <b>flavor</b> is too old and will not be served to clients. Rate-limit the
- * warning to avoid logging an entry on every request.
+ * <b>flavor</b> too new or too old, based on <b>is_too_new</b>,
+ * and will not be served to clients. Rate-limit the warning to avoid logging
+ * an entry on every request.
  */
 static void
-warn_consensus_is_too_old(const struct consensus_cache_entry_t *consensus,
-                          const char *flavor, time_t now)
+warn_consensus_is_not_reasonably_live(
+                          const struct consensus_cache_entry_t *consensus,
+                          const char *flavor, time_t now, bool is_too_new)
 {
-#define TOO_OLD_WARNING_INTERVAL (60*60)
-  static ratelim_t warned = RATELIM_INIT(TOO_OLD_WARNING_INTERVAL);
+#define NOT_REASONABLY_LIVE_WARNING_INTERVAL (60*60)
+  static ratelim_t warned[2] = { RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL),
+                                RATELIM_INIT(
+                                      NOT_REASONABLY_LIVE_WARNING_INTERVAL) };
   char timestamp[ISO_TIME_LEN+1];
-  time_t valid_until;
-  char *dupes;
+  /* valid_after if is_too_new, valid_until if !is_too_new */
+  time_t valid_time = 0;
+  char *dupes = NULL;
 
-  if (consensus_cache_entry_get_valid_until(consensus, &valid_until))
-    return;
-
-  if ((dupes = rate_limit_log(&warned, now))) {
-    format_local_iso_time(timestamp, valid_until);
-    log_warn(LD_DIRSERV, "Our %s%sconsensus is too old, so we will not "
-             "serve it to clients. It was valid until %s local time and we "
-             "continued to serve it for up to 24 hours after it expired.%s",
-             flavor ? flavor : "", flavor ? " " : "", timestamp, dupes);
+  if (is_too_new) {
+    if (consensus_cache_entry_get_valid_after(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[1], now);
+  } else {
+    if (consensus_cache_entry_get_valid_until(consensus, &valid_time))
+      return;
+    dupes = rate_limit_log(&warned[0], now);
+  }
+
+  if (dupes) {
+    format_local_iso_time(timestamp, valid_time);
+    log_warn(LD_DIRSERV, "Our %s%sconsensus is too %s, so we will not "
+             "serve it to clients. It was valid %s %s local time and we "
+             "continued to serve it for up to 24 hours %s.%s",
+             flavor ? flavor : "",
+             flavor ? " " : "",
+             is_too_new ? "new" : "old",
+             is_too_new ? "after" : "until",
+             timestamp,
+             is_too_new ? "before it was valid" : "after it expired",
+             dupes);
     tor_free(dupes);
   }
 }
@@ -543,7 +567,7 @@ parse_one_diff_hash(uint8_t *digest, const char *hex, const char *location,
 }
 
 /** If there is an X-Or-Diff-From-Consensus header included in <b>headers</b>,
- * set <b>digest_out<b> to a new smartlist containing every 256-bit
+ * set <b>digest_out</b> to a new smartlist containing every 256-bit
  * hex-encoded digest listed in that header and return 0.  Otherwise return
  * -1.  */
 static int
@@ -859,7 +883,6 @@ handle_get_current_consensus(dir_connection_t *conn,
 
   if (req.diff_only && !cached_consensus) {
     write_short_http_response(conn, 404, "No such diff available");
-    // XXXX warn_consensus_is_too_old(v, req.flavor, now);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -870,19 +893,30 @@ handle_get_current_consensus(dir_connection_t *conn,
                                            &compression_used);
   }
 
-  time_t fresh_until, valid_until;
-  int have_fresh_until = 0, have_valid_until = 0;
+  time_t valid_after, fresh_until, valid_until;
+  int have_valid_after = 0, have_fresh_until = 0, have_valid_until = 0;
   if (cached_consensus) {
+    have_valid_after =
+      !consensus_cache_entry_get_valid_after(cached_consensus, &valid_after);
     have_fresh_until =
       !consensus_cache_entry_get_fresh_until(cached_consensus, &fresh_until);
     have_valid_until =
       !consensus_cache_entry_get_valid_until(cached_consensus, &valid_until);
   }
 
-  if (cached_consensus && have_valid_until &&
+  if (cached_consensus && have_valid_after &&
+      !networkstatus_valid_after_is_reasonably_live(valid_after, now)) {
+    write_short_http_response(conn, 404, "Consensus is too new");
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          1);
+    geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
+    goto done;
+  } else if (
+      cached_consensus && have_valid_until &&
       !networkstatus_valid_until_is_reasonably_live(valid_until, now)) {
     write_short_http_response(conn, 404, "Consensus is too old");
-    warn_consensus_is_too_old(cached_consensus, req.flavor, now);
+    warn_consensus_is_not_reasonably_live(cached_consensus, req.flavor, now,
+                                          0);
     geoip_note_ns_response(GEOIP_REJECT_NOT_FOUND);
     goto done;
   }
@@ -924,7 +958,7 @@ handle_get_current_consensus(dir_connection_t *conn,
     goto done;
   }
 
-  if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+  if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
     log_debug(LD_DIRSERV,
               "Client asked for network status lists, but we've been "
               "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1033,7 +1067,7 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
         }
       });
 
-    if (global_write_bucket_low(TO_CONN(conn), estimated_len, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), estimated_len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto vote_done;
     }
@@ -1045,13 +1079,11 @@ handle_get_status_vote(dir_connection_t *conn, const get_handler_args_t *args)
       if (compress_method != NO_METHOD) {
         conn->compress_state = tor_compress_new(1, compress_method,
                            choose_compression_level(estimated_len));
-        SMARTLIST_FOREACH(items, const char *, c,
-                 connection_buf_add_compress(c, strlen(c), conn, 0));
-        connection_buf_add_compress("", 0, conn, 1);
-      } else {
-        SMARTLIST_FOREACH(items, const char *, c,
-                         connection_buf_add(c, strlen(c), TO_CONN(conn)));
       }
+
+      SMARTLIST_FOREACH(items, const char *, c,
+                        connection_dir_buf_add(c, strlen(c), conn,
+                                               c_sl_idx == c_sl_len - 1));
     } else {
       SMARTLIST_FOREACH(dir_items, cached_dir_t *, d,
           connection_buf_add(compress_method != NO_METHOD ?
@@ -1094,7 +1126,7 @@ handle_get_microdesc(dir_connection_t *conn, const get_handler_args_t *args)
       write_short_http_response(conn, 404, "Not found");
       goto done;
     }
-    if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
       log_info(LD_DIRSERV,
                "Client asked for server descriptors, but we've been "
                "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1192,7 +1224,7 @@ handle_get_descriptor(dir_connection_t *conn, const get_handler_args_t *args)
         msg = "Not found";
       write_short_http_response(conn, 404, msg);
     } else {
-      if (global_write_bucket_low(TO_CONN(conn), size_guess, 2)) {
+      if (connection_dir_is_global_write_low(TO_CONN(conn), size_guess)) {
         log_info(LD_DIRSERV,
                  "Client asked for server descriptors, but we've been "
                  "writing too many bytes lately. Sending 503 Dir busy.");
@@ -1288,9 +1320,8 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     SMARTLIST_FOREACH(certs, authority_cert_t *, c,
                       len += c->cache_info.signed_descriptor_len);
 
-    if (global_write_bucket_low(TO_CONN(conn),
-                                compress_method != NO_METHOD ? len/2 : len,
-                                2)) {
+    if (connection_dir_is_global_write_low(TO_CONN(conn),
+                                compress_method != NO_METHOD ? len/2 : len)) {
       write_short_http_response(conn, 503, "Directory busy, try again later");
       goto keys_done;
     }
@@ -1302,19 +1333,13 @@ handle_get_keys(dir_connection_t *conn, const get_handler_args_t *args)
     if (compress_method != NO_METHOD) {
       conn->compress_state = tor_compress_new(1, compress_method,
                                               choose_compression_level(len));
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add_compress(
-                c->cache_info.signed_descriptor_body,
-                c->cache_info.signed_descriptor_len,
-                conn, 0));
-      connection_buf_add_compress("", 0, conn, 1);
-    } else {
-      SMARTLIST_FOREACH(certs, authority_cert_t *, c,
-            connection_buf_add(c->cache_info.signed_descriptor_body,
-                                    c->cache_info.signed_descriptor_len,
-                                    TO_CONN(conn)));
     }
-  keys_done:
+
+    SMARTLIST_FOREACH(certs, authority_cert_t *, c,
+          connection_dir_buf_add(c->cache_info.signed_descriptor_body,
+                                 c->cache_info.signed_descriptor_len,
+                                 conn, c_sl_idx == c_sl_len - 1));
+ keys_done:
     smartlist_free(certs);
     goto done;
   }
@@ -1360,7 +1385,7 @@ handle_get_hs_descriptor_v2(dir_connection_t *conn,
   return 0;
 }
 
-/** Helper function for GET /tor/hs/3/<z>. Only for version 3.
+/** Helper function for GET `/tor/hs/3/...`. Only for version 3.
  */
 STATIC int
 handle_get_hs_descriptor_v3(dir_connection_t *conn,
@@ -1371,9 +1396,11 @@ handle_get_hs_descriptor_v3(dir_connection_t *conn,
   const char *pubkey_str = NULL;
   const char *url = args->url;
 
-  /* Reject unencrypted dir connections */
-  if (!connection_dir_is_encrypted(conn)) {
-    write_short_http_response(conn, 404, "Not found");
+  /* Reject non anonymous dir connections (which also tests if encrypted). We
+   * do not allow single hop clients to query an HSDir. */
+  if (!connection_dir_is_anonymous(conn)) {
+    write_short_http_response(conn, 503,
+                              "Rejecting single hop HS v3 descriptor request");
     goto done;
   }
 
@@ -1438,6 +1465,39 @@ handle_get_networkstatus_bridges(dir_connection_t *conn,
   return 0;
 }
 
+/** Helper function for GET the bandwidth file used for the next vote */
+static int
+handle_get_next_bandwidth(dir_connection_t *conn,
+                          const get_handler_args_t *args)
+{
+  log_debug(LD_DIR, "Getting next bandwidth.");
+  const or_options_t *options = get_options();
+  const compress_method_t compress_method =
+    find_best_compression_method(args->compression_supported, 1);
+
+  if (options->V3BandwidthsFile) {
+    char *bandwidth = read_file_to_str(options->V3BandwidthsFile,
+                                       RFTS_IGNORE_MISSING, NULL);
+    if (bandwidth != NULL) {
+      ssize_t len = strlen(bandwidth);
+      write_http_response_header(conn, compress_method != NO_METHOD ? -1 : len,
+                                 compress_method, BANDWIDTH_CACHE_LIFETIME);
+      if (compress_method != NO_METHOD) {
+        conn->compress_state = tor_compress_new(1, compress_method,
+                                        choose_compression_level(len/2));
+        log_debug(LD_DIR, "Compressing bandwidth file.");
+      } else {
+        log_debug(LD_DIR, "Not compressing bandwidth file.");
+      }
+      connection_dir_buf_add((const char*)bandwidth, len, conn, 1);
+      tor_free(bandwidth);
+      return 0;
+    }
+  }
+  write_short_http_response(conn, 404, "Not found");
+  return 0;
+}
+
 /** Helper function for GET robots.txt or /tor/robots.txt */
 static int
 handle_get_robots(dir_connection_t *conn, const get_handler_args_t *args)
@@ -1580,10 +1640,15 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     goto done;
   }
 
-  /* Handle HS descriptor publish request. */
-  /* XXX: This should be disabled with a consensus param until we want to
-   * the prop224 be deployed and thus use. */
-  if (connection_dir_is_encrypted(conn) && !strcmpstart(url, "/tor/hs/")) {
+  /* Handle HS descriptor publish request. We force an anonymous connection
+   * (which also tests for encrypted). We do not allow single-hop client to
+   * post a descriptor onto an HSDir. */
+  if (!strcmpstart(url, "/tor/hs/")) {
+    if (!connection_dir_is_anonymous(conn)) {
+      write_short_http_response(conn, 503,
+                                "Rejecting single hop HS descriptor post");
+      goto done;
+    }
     const char *msg = "HS descriptor stored successfully.";
 
     /* We most probably have a publish request for an HS descriptor. */
@@ -1608,8 +1673,8 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
     const char *msg = "[None]";
     uint8_t purpose = authdir_mode_bridge(options) ?
                       ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-    was_router_added_t r = dirserv_add_multiple_descriptors(body, purpose,
-                                             conn->base_.address, &msg);
+    was_router_added_t r = dirserv_add_multiple_descriptors(body, body_len,
+                                           purpose, conn->base_.address, &msg);
     tor_assert(msg);
 
     if (r == ROUTER_ADDED_SUCCESSFULLY) {
@@ -1631,7 +1696,7 @@ directory_handle_command_post,(dir_connection_t *conn, const char *headers,
       !strcmp(url,"/tor/post/vote")) { /* v3 networkstatus vote */
     const char *msg = "OK";
     int status;
-    if (dirvote_add_vote(body, &msg, &status)) {
+    if (dirvote_add_vote(body, approx_time(), &msg, &status)) {
       write_short_http_response(conn, status, "Vote stored");
     } else {
       tor_assert(msg);
diff --git a/src/feature/dircache/dircache.h b/src/feature/dircache/dircache.h
index 236ea649ef..d6392e2d42 100644
--- a/src/feature/dircache/dircache.h
+++ b/src/feature/dircache/dircache.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,6 +38,6 @@ STATIC int parse_hs_version_from_post(const char *url, const char *prefix,
                                       const char **end_pos);
 
 STATIC unsigned parse_accept_encoding_header(const char *h);
-#endif
+#endif /* defined(DIRCACHE_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCACHE_H) */
diff --git a/src/feature/dircache/dircache_stub.c b/src/feature/dircache/dircache_stub.c
new file mode 100644
index 0000000000..725c44bd4d
--- /dev/null
+++ b/src/feature/dircache/dircache_stub.c
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dircache_stub.c
+ * @brief Stub declarations for use when dircache module is disabled.
+ **/
+
+#include "core/or/or.h"
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/dircache/dircache.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/dircommon/dir_connection_st.h"
+
+DISABLE_GCC_WARNING("-Wmissing-noreturn")
+int
+directory_handle_command(dir_connection_t *conn)
+{
+  (void) conn;
+  tor_assert_nonfatal_unreached_once();
+  return -1;
+}
+
+int
+connection_dirserv_flushed_some(dir_connection_t *conn)
+{
+  (void) conn;
+  tor_assert_nonfatal_unreached_once();
+  return -1;
+}
+ENABLE_GCC_WARNING("-Wmissing-noreturn")
+
+void
+dir_conn_clear_spool(dir_connection_t *conn)
+{
+  if (!conn)
+    return;
+  tor_assert_nonfatal_once(conn->spool == NULL);
+}
+
+void
+consdiffmgr_enable_background_compression(void)
+{
+}
+
+int
+consdiffmgr_add_consensus(const char *consensus,
+                          size_t consensus_len,
+                          const networkstatus_t *as_parsed)
+{
+  (void)consensus;
+  (void)consensus_len;
+  (void)as_parsed;
+  return 0;
+}
+
+int
+consdiffmgr_register_with_sandbox(struct sandbox_cfg_elem_t **cfg)
+{
+  (void)cfg;
+  return 0;
+}
+
+int
+consdiffmgr_cleanup(void)
+{
+  return 0;
+}
+
+void
+consdiffmgr_free_all(void)
+{
+}
+
+void
+dirserv_free_all(void)
+{
+}
diff --git a/src/feature/dircache/dirserv.c b/src/feature/dircache/dirserv.c
index 213c490314..fb8db879a4 100644
--- a/src/feature/dircache/dirserv.c
+++ b/src/feature/dircache/dirserv.c
@@ -1,9 +1,8 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define DIRSERV_PRIVATE
 #include "core/or/or.h"
 
 #include "app/config/config.h"
@@ -69,55 +68,7 @@ static cached_dir_t *lookup_cached_dir_by_fp(const uint8_t *fp);
 /********************************************************************/
 
 /* A set of functions to answer questions about how we'd like to behave
- * as a directory mirror/client. */
-
-/** Return 1 if we fetch our directory material directly from the
- * authorities, rather than from a mirror. */
-int
-directory_fetches_from_authorities(const or_options_t *options)
-{
-  const routerinfo_t *me;
-  uint32_t addr;
-  int refuseunknown;
-  if (options->FetchDirInfoEarly)
-    return 1;
-  if (options->BridgeRelay == 1)
-    return 0;
-  if (server_mode(options) &&
-      router_pick_published_address(options, &addr, 1) < 0)
-    return 1; /* we don't know our IP address; ask an authority. */
-  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
-    should_refuse_unknown_exits(options);
-  if (!dir_server_mode(options) && !refuseunknown)
-    return 0;
-  if (!server_mode(options) || !advertised_server_mode())
-    return 0;
-  me = router_get_my_routerinfo();
-  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
-    return 0; /* if we don't service directory requests, return 0 too */
-  return 1;
-}
-
-/** Return 1 if we should fetch new networkstatuses, descriptors, etc
- * on the "mirror" schedule rather than the "client" schedule.
- */
-int
-directory_fetches_dir_info_early(const or_options_t *options)
-{
-  return directory_fetches_from_authorities(options);
-}
-
-/** Return 1 if we should fetch new networkstatuses, descriptors, etc
- * on a very passive schedule -- waiting long enough for ordinary clients
- * to probably have the info we want. These would include bridge users,
- * and maybe others in the future e.g. if a Tor client uses another Tor
- * client as a directory guard.
- */
-int
-directory_fetches_dir_info_later(const or_options_t *options)
-{
-  return options->UseBridges != 0;
-}
+ * as a directory mirror */
 
 /** Return true iff we want to serve certificates for authorities
  * that we don't acknowledge as authorities ourself.
@@ -161,19 +112,6 @@ directory_permits_begindir_requests(const or_options_t *options)
   return options->BridgeRelay != 0 || dir_server_mode(options);
 }
 
-/** Return 1 if we have no need to fetch new descriptors. This generally
- * happens when we're not a dir cache and we haven't built any circuits
- * lately.
- */
-int
-directory_too_idle_to_fetch_descriptors(const or_options_t *options,
-                                        time_t now)
-{
-  return !directory_caches_dir_info(options) &&
-         !options->FetchUselessDescriptors &&
-         rep_hist_circbuilding_dormant(now);
-}
-
 /********************************************************************/
 
 /** Map from flavor name to the cached_dir_t for the v3 consensuses that we're
@@ -234,6 +172,7 @@ free_cached_dir_(void *_d)
  * validation is performed. */
 void
 dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
+                                           size_t networkstatus_len,
                                            const char *flavor_name,
                                            const common_digests_t *digests,
                                            const uint8_t *sha3_as_signed,
@@ -244,7 +183,9 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
   if (!cached_consensuses)
     cached_consensuses = strmap_new();
 
-  new_networkstatus = new_cached_dir(tor_strdup(networkstatus), published);
+  new_networkstatus =
+    new_cached_dir(tor_memdup_nulterm(networkstatus, networkstatus_len),
+                   published);
   memcpy(&new_networkstatus->digests, digests, sizeof(common_digests_t));
   memcpy(&new_networkstatus->digest_sha3_as_signed, sha3_as_signed,
          DIGEST256_LEN);
@@ -256,14 +197,45 @@ dirserv_set_cached_consensus_networkstatus(const char *networkstatus,
 
 /** Return the latest downloaded consensus networkstatus in encoded, signed,
  * optionally compressed format, suitable for sending to clients. */
-cached_dir_t *
-dirserv_get_consensus(const char *flavor_name)
+MOCK_IMPL(cached_dir_t *,
+dirserv_get_consensus,(const char *flavor_name))
 {
   if (!cached_consensuses)
     return NULL;
   return strmap_get(cached_consensuses, flavor_name);
 }
 
+/** As dir_split_resource_into_fingerprints, but instead fills
+ * <b>spool_out</b> with a list of spoolable_resource_t for the resource
+ * identified through <b>source</b>. */
+int
+dir_split_resource_into_spoolable(const char *resource,
+                                  dir_spool_source_t source,
+                                  smartlist_t *spool_out,
+                                  int *compressed_out,
+                                  int flags)
+{
+  smartlist_t *fingerprints = smartlist_new();
+
+  tor_assert(flags & (DSR_HEX|DSR_BASE64));
+  const size_t digest_len =
+    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
+
+  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
+                                               compressed_out, flags);
+  /* This is not a very efficient implementation XXXX */
+  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
+    spooled_resource_t *spooled =
+      spooled_resource_new(source, digest, digest_len);
+    if (spooled)
+      smartlist_add(spool_out, spooled);
+    tor_free(digest);
+  } SMARTLIST_FOREACH_END(digest);
+
+  smartlist_free(fingerprints);
+  return r;
+}
+
 /** As dirserv_get_routerdescs(), but instead of getting signed_descriptor_t
  * pointers, adds copies of digests to fps_out, and doesn't use the
  * /tor/server/ prefix.  For a /d/ request, adds descriptor digests; for other
@@ -330,87 +302,6 @@ dirserv_get_routerdesc_spool(smartlist_t *spool_out,
   return 0;
 }
 
-/** Add a signed_descriptor_t to <b>descs_out</b> for each router matching
- * <b>key</b>.  The key should be either
- *   - "/tor/server/authority" for our own routerinfo;
- *   - "/tor/server/all" for all the routerinfos we have, concatenated;
- *   - "/tor/server/fp/FP" where FP is a plus-separated sequence of
- *     hex identity digests; or
- *   - "/tor/server/d/D" where D is a plus-separated sequence
- *     of server descriptor digests, in hex.
- *
- * Return 0 if we found some matching descriptors, or -1 if we do not
- * have any descriptors, no matching descriptors, or if we did not
- * recognize the key (URL).
- * If -1 is returned *<b>msg</b> will be set to an appropriate error
- * message.
- *
- * XXXX rename this function.  It's only called from the controller.
- * XXXX in fact, refactor this function, merging as much as possible.
- */
-int
-dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
-                        const char **msg)
-{
-  *msg = NULL;
-
-  if (!strcmp(key, "/tor/server/all")) {
-    routerlist_t *rl = router_get_routerlist();
-    SMARTLIST_FOREACH(rl->routers, routerinfo_t *, r,
-                      smartlist_add(descs_out, &(r->cache_info)));
-  } else if (!strcmp(key, "/tor/server/authority")) {
-    const routerinfo_t *ri = router_get_my_routerinfo();
-    if (ri)
-      smartlist_add(descs_out, (void*) &(ri->cache_info));
-  } else if (!strcmpstart(key, "/tor/server/d/")) {
-    smartlist_t *digests = smartlist_new();
-    key += strlen("/tor/server/d/");
-    dir_split_resource_into_fingerprints(key, digests, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-    SMARTLIST_FOREACH(digests, const char *, d,
-       {
-         signed_descriptor_t *sd = router_get_by_descriptor_digest(d);
-         if (sd)
-           smartlist_add(descs_out,sd);
-       });
-    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
-    smartlist_free(digests);
-  } else if (!strcmpstart(key, "/tor/server/fp/")) {
-    smartlist_t *digests = smartlist_new();
-    time_t cutoff = time(NULL) - ROUTER_MAX_AGE_TO_PUBLISH;
-    key += strlen("/tor/server/fp/");
-    dir_split_resource_into_fingerprints(key, digests, NULL,
-                                         DSR_HEX|DSR_SORT_UNIQ);
-    SMARTLIST_FOREACH_BEGIN(digests, const char *, d) {
-         if (router_digest_is_me(d)) {
-           /* calling router_get_my_routerinfo() to make sure it exists */
-           const routerinfo_t *ri = router_get_my_routerinfo();
-           if (ri)
-             smartlist_add(descs_out, (void*) &(ri->cache_info));
-         } else {
-           const routerinfo_t *ri = router_get_by_id_digest(d);
-           /* Don't actually serve a descriptor that everyone will think is
-            * expired.  This is an (ugly) workaround to keep buggy 0.1.1.10
-            * Tors from downloading descriptors that they will throw away.
-            */
-           if (ri && ri->cache_info.published_on > cutoff)
-             smartlist_add(descs_out, (void*) &(ri->cache_info));
-         }
-    } SMARTLIST_FOREACH_END(d);
-    SMARTLIST_FOREACH(digests, char *, d, tor_free(d));
-    smartlist_free(digests);
-  } else {
-    *msg = "Key not recognized";
-    return -1;
-  }
-
-  if (!smartlist_len(descs_out)) {
-    *msg = "Servers unavailable";
-    return -1;
-  }
-  return 0;
-}
-
 /* ==========
  * Spooling code.
  * ========== */
@@ -580,11 +471,9 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
       /* Absent objects count as "done". */
       return SRFS_DONE;
     }
-    if (conn->compress_state) {
-      connection_buf_add_compress((const char*)body, bodylen, conn, 0);
-    } else {
-      connection_buf_add((const char*)body, bodylen, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add((const char*)body, bodylen, conn, 0);
+
     return SRFS_DONE;
   } else {
     cached_dir_t *cached = spooled->cached_dir_ref;
@@ -619,14 +508,10 @@ spooled_resource_flush_some(spooled_resource_t *spooled,
     if (BUG(remaining < 0))
       return SRFS_ERR;
     ssize_t bytes = (ssize_t) MIN(DIRSERV_CACHED_DIR_CHUNK_SIZE, remaining);
-    if (conn->compress_state) {
-      connection_buf_add_compress(
-              ptr + spooled->cached_dir_offset,
-              bytes, conn, 0);
-    } else {
-      connection_buf_add(ptr + spooled->cached_dir_offset,
-                              bytes, TO_CONN(conn));
-    }
+
+    connection_dir_buf_add(ptr + spooled->cached_dir_offset,
+                           bytes, conn, 0);
+
     spooled->cached_dir_offset += bytes;
     if (spooled->cached_dir_offset >= (off_t)total_len) {
       return SRFS_DONE;
diff --git a/src/feature/dircache/dirserv.h b/src/feature/dircache/dirserv.h
index 890b10fd80..73a64b1b7e 100644
--- a/src/feature/dircache/dirserv.h
+++ b/src/feature/dircache/dirserv.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -73,28 +73,52 @@ typedef struct spooled_resource_t {
 
 int connection_dirserv_flushed_some(dir_connection_t *conn);
 
-int directory_fetches_from_authorities(const or_options_t *options);
-int directory_fetches_dir_info_early(const or_options_t *options);
-int directory_fetches_dir_info_later(const or_options_t *options);
+enum dir_spool_source_t;
+int dir_split_resource_into_spoolable(const char *resource,
+                                      enum dir_spool_source_t source,
+                                      smartlist_t *spool_out,
+                                      int *compressed_out,
+                                      int flags);
+
+#ifdef HAVE_MODULE_DIRCACHE
+/** Is the dircache module enabled? */
+#define have_module_dircache() (1)
 int directory_caches_unknown_auth_certs(const or_options_t *options);
 int directory_caches_dir_info(const or_options_t *options);
 int directory_permits_begindir_requests(const or_options_t *options);
-int directory_too_idle_to_fetch_descriptors(const or_options_t *options,
-                                            time_t now);
-
-cached_dir_t *dirserv_get_consensus(const char *flavor_name);
+MOCK_DECL(cached_dir_t *, dirserv_get_consensus, (const char *flavor_name));
 void dirserv_set_cached_consensus_networkstatus(const char *consensus,
+                                              size_t consensus_len,
                                               const char *flavor_name,
                                               const common_digests_t *digests,
                                               const uint8_t *sha3_as_signed,
                                               time_t published);
+#else /* !defined(HAVE_MODULE_DIRCACHE) */
+#define have_module_dircache() (0)
+#define directory_caches_unknown_auth_certs(opt) \
+  ((void)(opt), 0)
+#define directory_caches_dir_info(opt) \
+  ((void)(opt), 0)
+#define directory_permits_begindir_requests(opt) \
+  ((void)(opt), 0)
+#define dirserv_get_consensus(flav) \
+  ((void)(flav), NULL)
+#define dirserv_set_cached_consensus_networkstatus(a,b,c,d,e,f) \
+  STMT_BEGIN {                                                  \
+    (void)(a);                                                  \
+    (void)(b);                                                  \
+    (void)(c);                                                  \
+    (void)(d);                                                  \
+    (void)(e);                                                  \
+    (void)(f);                                                  \
+  } STMT_END
+#endif /* defined(HAVE_MODULE_DIRCACHE) */
+
 void dirserv_clear_old_networkstatuses(time_t cutoff);
 int dirserv_get_routerdesc_spool(smartlist_t *spools_out, const char *key,
                                  dir_spool_source_t source,
                                  int conn_is_encrypted,
                                  const char **msg_out);
-int dirserv_get_routerdescs(smartlist_t *descs_out, const char *key,
-                            const char **msg);
 
 void dirserv_free_all(void);
 void cached_dir_decref(cached_dir_t *d);
diff --git a/src/feature/dircache/feature_dircache.md b/src/feature/dircache/feature_dircache.md
new file mode 100644
index 0000000000..97734f2a34
--- /dev/null
+++ b/src/feature/dircache/feature_dircache.md
@@ -0,0 +1,6 @@
+@dir /feature/dircache
+@brief feature/dircache: Run as a directory cache server
+
+This module handles the directory caching functionality that all relays may
+provide, for serving cached directory objects to objects.
+
diff --git a/src/feature/dircache/include.am b/src/feature/dircache/include.am
new file mode 100644
index 0000000000..ab162565f7
--- /dev/null
+++ b/src/feature/dircache/include.am
@@ -0,0 +1,21 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+MODULE_DIRCACHE_SOURCES = 			\
+	src/feature/dircache/conscache.c	\
+	src/feature/dircache/consdiffmgr.c	\
+	src/feature/dircache/dircache.c		\
+	src/feature/dircache/dirserv.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dircache/cached_dir_st.h		\
+	src/feature/dircache/conscache.h		\
+	src/feature/dircache/consdiffmgr.h		\
+	src/feature/dircache/dircache.h			\
+	src/feature/dircache/dirserv.h
+
+if BUILD_MODULE_DIRCACHE
+LIBTOR_APP_A_SOURCES += $(MODULE_DIRCACHE_SOURCES)
+else
+LIBTOR_APP_A_STUB_SOURCES += src/feature/dircache/dircache_stub.c
+endif
diff --git a/src/feature/dirclient/.may_include b/src/feature/dirclient/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dirclient/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dirclient/dir_server_st.h b/src/feature/dirclient/dir_server_st.h
index 2f5706cdd9..37fa3148a7 100644
--- a/src/feature/dirclient/dir_server_st.h
+++ b/src/feature/dirclient/dir_server_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dir_server_st.h
+ * @brief Trusted/fallback directory server structure.
+ **/
+
 #ifndef DIR_SERVER_ST_H
 #define DIR_SERVER_ST_H
 
@@ -51,4 +56,4 @@ struct dir_server_t {
                                **/
 };
 
-#endif
+#endif /* !defined(DIR_SERVER_ST_H) */
diff --git a/src/feature/dirclient/dirclient.c b/src/feature/dirclient/dirclient.c
index 6725fc3369..ae1e018df2 100644
--- a/src/feature/dirclient/dirclient.c
+++ b/src/feature/dirclient/dirclient.c
@@ -1,8 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dirclient.c
+ * @brief Download directory information
+ **/
+
 #define DIRCLIENT_PRIVATE
 
 #include "core/or/or.h"
@@ -14,12 +19,14 @@
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
+#include "feature/dirclient/dirclient.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/shared_random.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/consdiff.h"
 #include "feature/dircommon/directory.h"
@@ -37,6 +44,7 @@
 #include "feature/nodelist/routerinfo.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
+#include "feature/relay/relay_find_addr.h"
 #include "feature/relay/routermode.h"
 #include "feature/relay/selftest.h"
 #include "feature/rend/rendcache.h"
@@ -45,6 +53,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 
+#include "lib/cc/ctassert.h"
 #include "lib/compress/compress.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -448,7 +457,7 @@ directory_get_from_dirserver,(
 {
   const routerstatus_t *rs = NULL;
   const or_options_t *options = get_options();
-  int prefer_authority = (directory_fetches_from_authorities(options)
+  int prefer_authority = (dirclient_fetches_from_authorities(options)
                           || want_authority == DL_WANT_AUTHORITY);
   int require_authority = 0;
   int get_via_tor = purpose_needs_anonymity(dir_purpose, router_purpose,
@@ -667,7 +676,7 @@ directory_choose_address_routerstatus(const routerstatus_t *status,
   if (indirection == DIRIND_DIRECT_CONN ||
       indirection == DIRIND_ANON_DIRPORT ||
       (indirection == DIRIND_ONEHOP
-       && !directory_must_use_begindir(options))) {
+       && !dirclient_must_use_begindir(options))) {
     fascist_firewall_choose_address_rs(status, FIREWALL_DIR_CONNECTION, 0,
                                        use_dir_ap);
     have_dir = tor_addr_port_is_valid_ap(use_dir_ap, 0);
@@ -866,16 +875,6 @@ connection_dir_download_cert_failed(dir_connection_t *conn, int status)
   update_certificate_downloads(time(NULL));
 }
 
-/* Should this tor instance only use begindir for all its directory requests?
- */
-int
-directory_must_use_begindir(const or_options_t *options)
-{
-  /* Clients, onion services, and bridges must use begindir,
-   * relays and authorities do not have to */
-  return !public_server_mode(options);
-}
-
 /** Evaluate the situation and decide if we should use an encrypted
  * "begindir-style" connection for this directory request.
  * 0) If there is no DirPort, yes.
@@ -927,7 +926,7 @@ directory_command_should_use_begindir(const or_options_t *options,
   }
   /* Reasons why we want to avoid using begindir */
   if (indirection == DIRIND_ONEHOP) {
-    if (!directory_must_use_begindir(options)) {
+    if (!dirclient_must_use_begindir(options)) {
       *reason = "in relay mode";
       return 0;
     }
@@ -1289,7 +1288,7 @@ directory_initiate_request,(directory_request_t *request))
 
   /* use encrypted begindir connections for everything except relays
    * this provides better protection for directory fetches */
-  if (!use_begindir && directory_must_use_begindir(options)) {
+  if (!use_begindir && dirclient_must_use_begindir(options)) {
     log_warn(LD_BUG, "Client could not use begindir connection: %s",
              begindir_reason ? begindir_reason : "(NULL)");
     return;
@@ -1447,9 +1446,7 @@ compare_strs_(const void **a, const void **b)
 }
 
 #define CONDITIONAL_CONSENSUS_FPR_LEN 3
-#if (CONDITIONAL_CONSENSUS_FPR_LEN > DIGEST_LEN)
-#error "conditional consensus fingerprint length is larger than digest length"
-#endif
+CTASSERT(CONDITIONAL_CONSENSUS_FPR_LEN <= DIGEST_LEN);
 
 /** Return the URL we should use for a consensus download.
  *
@@ -1968,6 +1965,44 @@ dir_client_decompress_response_body(char **bodyp, size_t *bodylenp,
   return rv;
 }
 
+/**
+ * Total number of bytes downloaded of each directory purpose, when
+ * bootstrapped, and when not bootstrapped.
+ *
+ * (For example, the number of bytes downloaded of purpose p while
+ * not fully bootstrapped is total_dl[p][false].)
+ **/
+static uint64_t total_dl[DIR_PURPOSE_MAX_][2];
+
+/**
+ * Heartbeat: dump a summary of how many bytes of which purpose we've
+ * downloaded, when bootstrapping and when not bootstrapping.
+ **/
+void
+dirclient_dump_total_dls(void)
+{
+  const or_options_t *options = get_options();
+  for (int bootstrapped = 0; bootstrapped < 2; ++bootstrapped) {
+    bool first_time = true;
+    for (int i=0; i < DIR_PURPOSE_MAX_; ++i) {
+      uint64_t n = total_dl[i][bootstrapped];
+      if (n == 0)
+        continue;
+      if (options->SafeLogging_ != SAFELOG_SCRUB_NONE &&
+          purpose_needs_anonymity(i, ROUTER_PURPOSE_GENERAL, NULL))
+        continue;
+      if (first_time) {
+        log_notice(LD_NET,
+                   "While %sbootstrapping, fetched this many bytes: ",
+                   bootstrapped?"not ":"");
+        first_time = false;
+      }
+      log_notice(LD_NET, "    %"PRIu64" (%s)",
+                 n, dir_conn_purpose_to_string(i));
+    }
+  }
+}
+
 /** We are a client, and we've finished reading the server's
  * response. Parse it and act appropriately.
  *
@@ -2001,6 +2036,16 @@ connection_dir_client_reached_eof(dir_connection_t *conn)
 
   received_bytes = connection_get_inbuf_len(TO_CONN(conn));
 
+  log_debug(LD_DIR, "Downloaded %"TOR_PRIuSZ" bytes on connection of purpose "
+             "%s; bootstrap %d%%",
+             received_bytes,
+             dir_conn_purpose_to_string(conn->base_.purpose),
+             control_get_bootstrap_percent());
+  {
+    bool bootstrapped = control_get_bootstrap_percent() == 100;
+    total_dl[conn->base_.purpose][bootstrapped] += received_bytes;
+  }
+
   switch (connection_fetch_from_buf_http(TO_CONN(conn),
                               &headers, MAX_HEADERS_SIZE,
                               &body, &body_len, MAX_DIR_DL_SIZE,
@@ -2205,24 +2250,31 @@ handle_response_fetch_consensus(dir_connection_t *conn,
   if (looks_like_a_consensus_diff(body, body_len)) {
     /* First find our previous consensus. Maybe it's in ram, maybe not. */
     cached_dir_t *cd = dirserv_get_consensus(flavname);
-    const char *consensus_body;
-    char *owned_consensus = NULL;
+    const char *consensus_body = NULL;
+    size_t consensus_body_len;
+    tor_mmap_t *mapped_consensus = NULL;
     if (cd) {
       consensus_body = cd->dir;
+      consensus_body_len = cd->dir_len;
     } else {
-      owned_consensus = networkstatus_read_cached_consensus(flavname);
-      consensus_body = owned_consensus;
+      mapped_consensus = networkstatus_map_cached_consensus(flavname);
+      if (mapped_consensus) {
+        consensus_body = mapped_consensus->data;
+        consensus_body_len = mapped_consensus->size;
+      }
     }
     if (!consensus_body) {
       log_warn(LD_DIR, "Received a consensus diff, but we can't find "
                "any %s-flavored consensus in our current cache.",flavname);
+      tor_munmap_file(mapped_consensus);
       networkstatus_consensus_download_failed(0, flavname);
       // XXXX if this happens too much, see below
       return -1;
     }
 
-    new_consensus = consensus_diff_apply(consensus_body, body);
-    tor_free(owned_consensus);
+    new_consensus = consensus_diff_apply(consensus_body, consensus_body_len,
+                                         body, body_len);
+    tor_munmap_file(mapped_consensus);
     if (new_consensus == NULL) {
       log_warn(LD_DIR, "Could not apply consensus diff received from server "
                "'%s:%d'", conn->base_.address, conn->base_.port);
@@ -2244,7 +2296,9 @@ handle_response_fetch_consensus(dir_connection_t *conn,
     sourcename = "downloaded";
   }
 
-  if ((r=networkstatus_set_current_consensus(consensus, flavname, 0,
+  if ((r=networkstatus_set_current_consensus(consensus,
+                                             strlen(consensus),
+                                             flavname, 0,
                                              conn->identity_digest))<0) {
     log_fn(r<-1?LOG_WARN:LOG_INFO, LD_DIR,
            "Unable to load %s consensus directory %s from "
@@ -2359,7 +2413,7 @@ handle_response_fetch_status_vote(dir_connection_t *conn,
              conn->base_.port, conn->requested_resource);
     return -1;
   }
-  dirvote_add_vote(body, &msg, &st);
+  dirvote_add_vote(body, 0, &msg, &st);
   if (st > 299) {
     log_warn(LD_DIR, "Error adding retrieved vote: %s", msg);
   } else {
@@ -2522,7 +2576,7 @@ handle_response_fetch_microdesc(dir_connection_t *conn,
            conn->base_.port);
   tor_assert(conn->requested_resource &&
              !strcmpstart(conn->requested_resource, "d/"));
-  tor_assert_nonfatal(!tor_mem_is_zero(conn->identity_digest, DIGEST_LEN));
+  tor_assert_nonfatal(!fast_mem_is_zero(conn->identity_digest, DIGEST_LEN));
   which = smartlist_new();
   dir_split_resource_into_fingerprints(conn->requested_resource+2,
                                        which, NULL,
@@ -2719,62 +2773,7 @@ handle_response_fetch_hsdesc_v3(dir_connection_t *conn,
   log_info(LD_REND,"Received v3 hsdesc (body size %d, status %d (%s))",
            (int)body_len, status_code, escaped(reason));
 
-  switch (status_code) {
-  case 200:
-    /* We got something: Try storing it in the cache. */
-    if (hs_cache_store_as_client(body, &conn->hs_ident->identity_pk) < 0) {
-      log_info(LD_REND, "Failed to store hidden service descriptor");
-      /* Fire control port FAILED event. */
-      hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                   "BAD_DESC");
-      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                    NULL);
-    } else {
-      log_info(LD_REND, "Stored hidden service descriptor successfully.");
-      TO_CONN(conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
-      hs_client_desc_has_arrived(conn->hs_ident);
-      /* Fire control port RECEIVED event. */
-      hs_control_desc_event_received(conn->hs_ident, conn->identity_digest);
-      hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                    body);
-    }
-    break;
-  case 404:
-    /* Not there. We'll retry when connection_about_to_close_connection()
-     * tries to clean this conn up. */
-    log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
-                      "Retrying at another directory.");
-    /* Fire control port FAILED event. */
-    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                 "NOT_FOUND");
-    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                  NULL);
-    break;
-  case 400:
-    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
-                      "http status 400 (%s). Dirserver didn't like our "
-                      "query? Retrying at another directory.",
-             escaped(reason));
-    /* Fire control port FAILED event. */
-    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                 "QUERY_REJECTED");
-    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                  NULL);
-    break;
-  default:
-    log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
-             "http status %d (%s) response unexpected from HSDir server "
-             "'%s:%d'. Retrying at another directory.",
-             status_code, escaped(reason), TO_CONN(conn)->address,
-             TO_CONN(conn)->port);
-    /* Fire control port FAILED event. */
-    hs_control_desc_event_failed(conn->hs_ident, conn->identity_digest,
-                                 "UNEXPECTED");
-    hs_control_desc_event_content(conn->hs_ident, conn->identity_digest,
-                                  NULL);
-    break;
-  }
-
+  hs_client_dir_fetch_done(conn, reason, body, status_code);
   return 0;
 }
 
@@ -3134,7 +3133,7 @@ dir_routerdesc_download_failed(smartlist_t *failed, int status_code,
 {
   char digest[DIGEST_LEN];
   time_t now = time(NULL);
-  int server = directory_fetches_from_authorities(get_options());
+  int server = dirclient_fetches_from_authorities(get_options());
   if (!was_descriptor_digests) {
     if (router_purpose == ROUTER_PURPOSE_BRIDGE) {
       tor_assert(!was_extrainfo);
@@ -3179,7 +3178,7 @@ dir_microdesc_download_failed(smartlist_t *failed,
   routerstatus_t *rs;
   download_status_t *dls;
   time_t now = time(NULL);
-  int server = directory_fetches_from_authorities(get_options());
+  int server = dirclient_fetches_from_authorities(get_options());
 
   if (! consensus)
     return;
diff --git a/src/feature/dirclient/dirclient.h b/src/feature/dirclient/dirclient.h
index 1a93265dc3..096b197526 100644
--- a/src/feature/dirclient/dirclient.h
+++ b/src/feature/dirclient/dirclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,8 @@
 
 #include "feature/hs/hs_ident.h"
 
+void dirclient_dump_total_dls(void);
+
 int directories_have_accepted_server_descriptor(void);
 void directory_post_to_dirservers(uint8_t dir_purpose, uint8_t router_purpose,
                                   dirinfo_type_t type, const char *payload,
@@ -41,8 +43,6 @@ typedef enum {
   DIRIND_ANON_DIRPORT,
 } dir_indirection_t;
 
-int directory_must_use_begindir(const or_options_t *options);
-
 /**
  * A directory_request_t describes the information about a directory request
  * at the client side.  It describes what we're going to ask for, which
@@ -167,6 +167,6 @@ STATIC int handle_response_fetch_consensus(dir_connection_t *conn,
 
 STATIC dirinfo_type_t dir_fetch_type(int dir_purpose, int router_purpose,
                                      const char *resource);
-#endif
+#endif /* defined(DIRCLIENT_PRIVATE) */
 
 #endif /* !defined(TOR_DIRCLIENT_H) */
diff --git a/src/feature/dirclient/dirclient_modes.c b/src/feature/dirclient/dirclient_modes.c
new file mode 100644
index 0000000000..31a3f8af58
--- /dev/null
+++ b/src/feature/dirclient/dirclient_modes.c
@@ -0,0 +1,96 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirclient_modes.c
+ * @brief Functions to answer questions about how we'd like to behave
+ *     as a directory client
+ **/
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+
+#include "feature/dirclient/dirclient_modes.h"
+#include "feature/dircache/dirserv.h"
+#include "feature/relay/relay_find_addr.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/stats/predict_ports.h"
+
+#include "app/config/or_options_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+/* Should this tor instance only use begindir for all its directory requests?
+ */
+int
+dirclient_must_use_begindir(const or_options_t *options)
+{
+  /* Clients, onion services, and bridges must use begindir,
+   * relays and authorities do not have to */
+  return !public_server_mode(options);
+}
+
+/** Return 1 if we fetch our directory material directly from the
+ * authorities, rather than from a mirror. */
+int
+dirclient_fetches_from_authorities(const or_options_t *options)
+{
+  const routerinfo_t *me;
+  uint32_t addr;
+  int refuseunknown;
+  if (options->FetchDirInfoEarly)
+    return 1;
+  if (options->BridgeRelay == 1)
+    return 0;
+  if (server_mode(options) &&
+      router_pick_published_address(options, &addr, 1) < 0)
+    return 1; /* we don't know our IP address; ask an authority. */
+  refuseunknown = ! router_my_exit_policy_is_reject_star() &&
+    should_refuse_unknown_exits(options);
+  if (!dir_server_mode(options) && !refuseunknown)
+    return 0;
+  if (!server_mode(options) || !advertised_server_mode())
+    return 0;
+  me = router_get_my_routerinfo();
+  if (!me || (!me->supports_tunnelled_dir_requests && !refuseunknown))
+    return 0; /* if we don't service directory requests, return 0 too */
+  return 1;
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on the "mirror" schedule rather than the "client" schedule.
+ */
+int
+dirclient_fetches_dir_info_early(const or_options_t *options)
+{
+  return dirclient_fetches_from_authorities(options);
+}
+
+/** Return 1 if we should fetch new networkstatuses, descriptors, etc
+ * on a very passive schedule -- waiting long enough for ordinary clients
+ * to probably have the info we want. These would include bridge users,
+ * and maybe others in the future e.g. if a Tor client uses another Tor
+ * client as a directory guard.
+ */
+int
+dirclient_fetches_dir_info_later(const or_options_t *options)
+{
+  return options->UseBridges != 0;
+}
+
+/** Return 1 if we have no need to fetch new descriptors. This generally
+ * happens when we're not a dir cache and we haven't built any circuits
+ * lately.
+ */
+int
+dirclient_too_idle_to_fetch_descriptors(const or_options_t *options,
+                                        time_t now)
+{
+  return !directory_caches_dir_info(options) &&
+         !options->FetchUselessDescriptors &&
+         rep_hist_circbuilding_dormant(now);
+}
diff --git a/src/feature/dirclient/dirclient_modes.h b/src/feature/dirclient/dirclient_modes.h
new file mode 100644
index 0000000000..c402207724
--- /dev/null
+++ b/src/feature/dirclient/dirclient_modes.h
@@ -0,0 +1,24 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirclient_modes.h
+ * @brief Header for feature/dirclient/dirclient_modes.c
+ **/
+
+#ifndef TOR_FEATURE_DIRCLIENT_DIRCLIENT_MODES_H
+#define TOR_FEATURE_DIRCLIENT_DIRCLIENT_MODES_H
+
+struct or_options_t;
+
+int dirclient_must_use_begindir(const or_options_t *options);
+int dirclient_fetches_from_authorities(const struct or_options_t *options);
+int dirclient_fetches_dir_info_early(const struct or_options_t *options);
+int dirclient_fetches_dir_info_later(const struct or_options_t *options);
+int dirclient_too_idle_to_fetch_descriptors(const struct or_options_t *options,
+                                            time_t now);
+
+#endif /* !defined(TOR_FEATURE_DIRCLIENT_DIRCLIENT_MODES_H) */
diff --git a/src/feature/dirclient/dlstatus.c b/src/feature/dirclient/dlstatus.c
index 0842a2c676..ab3fbb8577 100644
--- a/src/feature/dirclient/dlstatus.c
+++ b/src/feature/dirclient/dlstatus.c
@@ -1,8 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dlstatus.c
+ * @brief Track status and retry schedule of a downloadable object.
+ **/
+
 #define DLSTATUS_PRIVATE
 
 #include "core/or/or.h"
diff --git a/src/feature/dirclient/dlstatus.h b/src/feature/dirclient/dlstatus.h
index 99e0d0225b..e5c8b756c4 100644
--- a/src/feature/dirclient/dlstatus.h
+++ b/src/feature/dirclient/dlstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -53,6 +53,6 @@ STATIC void next_random_exponential_delay_range(int *low_bound_out,
 /* no more than triple the previous delay */
 #define DIR_TEST_NET_RANDOM_MULTIPLIER (2)
 
-#endif
+#endif /* defined(DLSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_DLSTATUS_H) */
diff --git a/src/feature/dirclient/download_status_st.h b/src/feature/dirclient/download_status_st.h
index 11555a1dcc..92efcb44d0 100644
--- a/src/feature/dirclient/download_status_st.h
+++ b/src/feature/dirclient/download_status_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file download_status_st.h
+ * @brief Directory download status/schedule structure.
+ **/
+
 #ifndef DOWNLOAD_STATUS_ST_H
 #define DOWNLOAD_STATUS_ST_H
 
@@ -61,5 +66,4 @@ struct download_status_t {
                         * only updated if backoff == 1 */
 };
 
-#endif
-
+#endif /* !defined(DOWNLOAD_STATUS_ST_H) */
diff --git a/src/feature/dirclient/feature_dirclient.md b/src/feature/dirclient/feature_dirclient.md
new file mode 100644
index 0000000000..5c7ee964d3
--- /dev/null
+++ b/src/feature/dirclient/feature_dirclient.md
@@ -0,0 +1,7 @@
+@dir /feature/dirclient
+@brief feature/dirclient: Directory client implementation.
+
+The code here is used by all Tor instances that need to download directory
+information.  Currently, that is all of them, since even authorities need to
+launch downloads to learn about relays that other authorities have listed.
+
diff --git a/src/feature/dirclient/include.am b/src/feature/dirclient/include.am
new file mode 100644
index 0000000000..24cae9eedd
--- /dev/null
+++ b/src/feature/dirclient/include.am
@@ -0,0 +1,14 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/dirclient/dirclient.c	\
+	src/feature/dirclient/dirclient_modes.c	\
+	src/feature/dirclient/dlstatus.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dirclient/dir_server_st.h		\
+	src/feature/dirclient/dirclient.h		\
+	src/feature/dirclient/dirclient_modes.h		\
+	src/feature/dirclient/dlstatus.h		\
+	src/feature/dirclient/download_status_st.h
diff --git a/src/feature/dircommon/.may_include b/src/feature/dircommon/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dircommon/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dircommon/consdiff.c b/src/feature/dircommon/consdiff.c
index d0f7594ce3..e42378c44c 100644
--- a/src/feature/dircommon/consdiff.c
+++ b/src/feature/dircommon/consdiff.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2014, Daniel Martí
- * Copyright (c) 2014-2019, The Tor Project, Inc. */
+ * Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -101,11 +101,11 @@ smartlist_add_linecpy(smartlist_t *lst, memarea_t *area, const char *s)
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest,(const char *cons,
+consensus_compute_digest,(const char *cons, size_t len,
                           consensus_digest_t *digest_out))
 {
   int r = crypto_digest256((char*)digest_out->sha3_256,
-                           cons, strlen(cons), DIGEST_SHA3_256);
+                           cons, len, DIGEST_SHA3_256);
   return r;
 }
 
@@ -114,11 +114,11 @@ consensus_compute_digest,(const char *cons,
 /* This is a separate, mockable function so that we can override it when
  * fuzzing. */
 MOCK_IMPL(STATIC int,
-consensus_compute_digest_as_signed,(const char *cons,
+consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out))
 {
   return router_get_networkstatus_v3_sha3_as_signed(digest_out->sha3_256,
-                                                    cons);
+                                                    cons, len);
 }
 
 /** Return true iff <b>d1</b> and <b>d2</b> contain the same digest */
@@ -530,10 +530,12 @@ typedef struct router_id_iterator_t {
   cdline_t hash;
 } router_id_iterator_t;
 
+#ifndef COCCI
 /**
  * Initializer for a router_id_iterator_t.
  */
 #define ROUTER_ID_ITERATOR_INIT { { NULL, 0 }, { NULL, 0 } }
+#endif /* !defined(COCCI) */
 
 /** Given an index *<b>idxp</b> into the consensus at <b>cons</b>, advance
  * the index to the next router line ("r ...") in the consensus, or to
@@ -570,7 +572,7 @@ find_next_router_line(const smartlist_t *cons,
 /** Pre-process a consensus in <b>cons</b> (represented as a list of cdline_t)
  * to remove the signatures from it.  If the footer is removed, return a
  * cdline_t containing a delete command to delete the footer, allocated in
- * <b>area</>.  If no footer is removed, return NULL.
+ * <b>area</b>.  If no footer is removed, return NULL.
  *
  * We remove the signatures here because they are not themselves signed, and
  * as such there might be different encodings for them.
@@ -1048,7 +1050,7 @@ consdiff_gen_diff(const smartlist_t *cons1,
   if (smartlist_len(cons2) == smartlist_len(ed_cons2)) {
     SMARTLIST_FOREACH_BEGIN(cons2, const cdline_t *, line1) {
       const cdline_t *line2 = smartlist_get(ed_cons2, line1_sl_idx);
-      if (! lines_eq(line1, line2) ) {
+      if (!lines_eq(line1, line2)) {
         cons2_eq = 0;
         break;
       }
@@ -1229,7 +1231,8 @@ consdiff_apply_diff(const smartlist_t *cons1,
   cons2_str = consensus_join_lines(cons2);
 
   consensus_digest_t cons2_digests;
-  if (consensus_compute_digest(cons2_str, &cons2_digests) < 0) {
+  if (consensus_compute_digest(cons2_str, strlen(cons2_str),
+                               &cons2_digests) < 0) {
     /* LCOV_EXCL_START -- digest can't fail */
     log_warn(LD_CONSDIFF, "Could not compute digests of the consensus "
         "resulting from applying a consensus diff.");
@@ -1283,12 +1286,13 @@ consdiff_apply_diff(const smartlist_t *cons1,
  * generated cdlines will become invalid.
  */
 STATIC int
-consensus_split_lines(smartlist_t *out, const char *s, memarea_t *area)
+consensus_split_lines(smartlist_t *out,
+                      const char *s, size_t len,
+                      memarea_t *area)
 {
-  const char *end_of_str = s + strlen(s);
-  tor_assert(*end_of_str == '\0');
+  const char *end_of_str = s + len;
 
-  while (*s) {
+  while (s < end_of_str) {
     const char *eol = memchr(s, '\n', end_of_str - s);
     if (!eol) {
       /* File doesn't end with newline. */
@@ -1334,25 +1338,25 @@ consensus_join_lines(const smartlist_t *inp)
  * success, retun a newly allocated string containing that diff.  On failure,
  * return NULL. */
 char *
-consensus_diff_generate(const char *cons1,
-                        const char *cons2)
+consensus_diff_generate(const char *cons1, size_t cons1len,
+                        const char *cons2, size_t cons2len)
 {
   consensus_digest_t d1, d2;
   smartlist_t *lines1 = NULL, *lines2 = NULL, *result_lines = NULL;
   int r1, r2;
   char *result = NULL;
 
-  r1 = consensus_compute_digest_as_signed(cons1, &d1);
-  r2 = consensus_compute_digest(cons2, &d2);
+  r1 = consensus_compute_digest_as_signed(cons1, cons1len, &d1);
+  r2 = consensus_compute_digest(cons2, cons2len, &d2);
   if (BUG(r1 < 0 || r2 < 0))
     return NULL; // LCOV_EXCL_LINE
 
   memarea_t *area = memarea_new();
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, cons1, area) < 0)
+  if (consensus_split_lines(lines1, cons1, cons1len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, cons2, area) < 0)
+  if (consensus_split_lines(lines2, cons2, cons2len, area) < 0)
     goto done;
 
   result_lines = consdiff_gen_diff(lines1, lines2, &d1, &d2, area);
@@ -1375,7 +1379,9 @@ consensus_diff_generate(const char *cons1,
  * consensus.  On failure, return NULL. */
 char *
 consensus_diff_apply(const char *consensus,
-                     const char *diff)
+                     size_t consensus_len,
+                     const char *diff,
+                     size_t diff_len)
 {
   consensus_digest_t d1;
   smartlist_t *lines1 = NULL, *lines2 = NULL;
@@ -1383,15 +1389,15 @@ consensus_diff_apply(const char *consensus,
   char *result = NULL;
   memarea_t *area = memarea_new();
 
-  r1 = consensus_compute_digest_as_signed(consensus, &d1);
+  r1 = consensus_compute_digest_as_signed(consensus, consensus_len, &d1);
   if (BUG(r1 < 0))
     goto done;
 
   lines1 = smartlist_new();
   lines2 = smartlist_new();
-  if (consensus_split_lines(lines1, consensus, area) < 0)
+  if (consensus_split_lines(lines1, consensus, consensus_len, area) < 0)
     goto done;
-  if (consensus_split_lines(lines2, diff, area) < 0)
+  if (consensus_split_lines(lines2, diff, diff_len, area) < 0)
     goto done;
 
   result = consdiff_apply_diff(lines1, lines2, &d1);
diff --git a/src/feature/dircommon/consdiff.h b/src/feature/dircommon/consdiff.h
index 98217e6d46..c2dcb6da24 100644
--- a/src/feature/dircommon/consdiff.h
+++ b/src/feature/dircommon/consdiff.h
@@ -1,16 +1,21 @@
 /* Copyright (c) 2014, Daniel Martí
- * Copyright (c) 2014-2019, The Tor Project, Inc. */
+ * Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file consdiff.h
+ * @brief Header for consdiff.c
+ **/
+
 #ifndef TOR_CONSDIFF_H
 #define TOR_CONSDIFF_H
 
 #include "core/or/or.h"
 
-char *consensus_diff_generate(const char *cons1,
-                              const char *cons2);
-char *consensus_diff_apply(const char *consensus,
-                           const char *diff);
+char *consensus_diff_generate(const char *cons1, size_t cons1len,
+                              const char *cons2, size_t cons2len);
+char *consensus_diff_apply(const char *consensus, size_t consensus_len,
+                           const char *diff, size_t diff_len);
 
 int looks_like_a_consensus_diff(const char *document, size_t len);
 
@@ -78,7 +83,8 @@ STATIC int smartlist_slice_string_pos(const smartlist_slice_t *slice,
 STATIC void set_changed(bitarray_t *changed1, bitarray_t *changed2,
                         const smartlist_slice_t *slice1,
                         const smartlist_slice_t *slice2);
-STATIC int consensus_split_lines(smartlist_t *out, const char *s,
+STATIC int consensus_split_lines(smartlist_t *out,
+                                 const char *s, size_t len,
                                  struct memarea_t *area);
 STATIC void smartlist_add_linecpy(smartlist_t *lst, struct memarea_t *area,
                                   const char *s);
@@ -86,10 +92,10 @@ STATIC int lines_eq(const cdline_t *a, const cdline_t *b);
 STATIC int line_str_eq(const cdline_t *a, const char *b);
 
 MOCK_DECL(STATIC int,
-          consensus_compute_digest,(const char *cons,
+          consensus_compute_digest,(const char *cons, size_t len,
                                     consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
-          consensus_compute_digest_as_signed,(const char *cons,
+          consensus_compute_digest_as_signed,(const char *cons, size_t len,
                                               consensus_digest_t *digest_out));
 MOCK_DECL(STATIC int,
           consensus_digest_eq,(const uint8_t *d1,
diff --git a/src/feature/dircommon/dir_connection_st.h b/src/feature/dircommon/dir_connection_st.h
index 8c59cc7a46..12230e6741 100644
--- a/src/feature/dircommon/dir_connection_st.h
+++ b/src/feature/dircommon/dir_connection_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file dir_connection_st.h
+ * @brief Client/server directory connection structure.
+ **/
+
 #ifndef DIR_CONNECTION_ST_H
 #define DIR_CONNECTION_ST_H
 
@@ -23,7 +28,9 @@ struct dir_connection_t {
   * fingerprints.
   **/
   char *requested_resource;
-  unsigned int dirconn_direct:1; /**< Is this dirconn direct, or via Tor? */
+  /** Is this dirconn direct, or via a multi-hop Tor circuit?
+   * Direct connections can use the DirPort, or BEGINDIR over the ORPort. */
+  unsigned int dirconn_direct:1;
 
   /** If we're fetching descriptors, what router purpose shall we assign
    * to them? */
@@ -64,4 +71,4 @@ struct dir_connection_t {
 #endif /* defined(MEASUREMENTS_21206) */
 };
 
-#endif
+#endif /* !defined(DIR_CONNECTION_ST_H) */
diff --git a/src/feature/dircommon/directory.c b/src/feature/dircommon/directory.c
index 9e6f72e9ac..b177fe5201 100644
--- a/src/feature/dircommon/directory.c
+++ b/src/feature/dircommon/directory.c
@@ -1,12 +1,16 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
 
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
+#include "core/or/circuitlist.h"
+#include "core/or/connection_edge.h"
+#include "core/or/connection_or.h"
+#include "core/or/channeltls.h"
 #include "feature/dircache/dircache.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -15,6 +19,10 @@
 #include "feature/stats/geoip_stats.h"
 #include "lib/compress/compress.h"
 
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 
@@ -167,6 +175,82 @@ connection_dir_is_encrypted(const dir_connection_t *conn)
   return TO_CONN(conn)->linked;
 }
 
+/** Return true iff the given directory connection <b>dir_conn</b> is
+ * anonymous, that is, it is on a circuit via a public relay and not directly
+ * from a client or bridge.
+ *
+ * For client circuits via relays: true for 2-hop+ paths.
+ * For client circuits via bridges: true for 3-hop+ paths.
+ *
+ * This first test if the connection is encrypted since it is a strong
+ * requirement for anonymity. */
+bool
+connection_dir_is_anonymous(const dir_connection_t *dir_conn)
+{
+  const connection_t *conn, *linked_conn;
+  const edge_connection_t *edge_conn;
+  const circuit_t *circ;
+
+  tor_assert(dir_conn);
+
+  if (!connection_dir_is_encrypted(dir_conn)) {
+    return false;
+  }
+
+  /*
+   * Buckle up, we'll do a deep dive into the connection in order to get the
+   * final connection channel of that connection in order to figure out if
+   * this is a client or relay link.
+   *
+   * We go: dir_conn -> linked_conn -> edge_conn -> on_circuit -> p_chan.
+   */
+
+  conn = TO_CONN(dir_conn);
+  linked_conn = conn->linked_conn;
+
+  /* The dir connection should be connected to an edge connection. It can not
+   * be closed or marked for close. */
+  if (linked_conn == NULL || linked_conn->magic != EDGE_CONNECTION_MAGIC ||
+      conn->linked_conn_is_closed || conn->linked_conn->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not linked to edge");
+    return false;
+  }
+
+  edge_conn = TO_EDGE_CONN((connection_t *) linked_conn);
+  circ = edge_conn->on_circuit;
+
+  /* Can't be a circuit we initiated and without a circuit, no channel. */
+  if (circ == NULL || CIRCUIT_IS_ORIGIN(circ)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "not on OR circuit");
+    return false;
+  }
+
+  /* It is possible that the circuit was closed because one of the channel was
+   * closed or a DESTROY cell was received. Either way, this connection can
+   * not continue so return that it is not anonymous since we can not know for
+   * sure if it is. */
+  if (circ->marked_for_close) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "circuit marked for close");
+    return false;
+  }
+
+  /* Get the previous channel to learn if it is a client or relay link. We
+   * BUG() because if the circuit is not mark for close, we ought to have a
+   * p_chan else we have a code flow issue. */
+  if (BUG(CONST_TO_OR_CIRCUIT(circ)->p_chan == NULL)) {
+    log_debug(LD_DIR, "Directory connection is not anonymous: "
+                      "no p_chan on circuit");
+    return false;
+  }
+
+  /* Will be true if the channel is an unauthenticated peer which is only true
+   * for clients and bridges. */
+  return !channel_is_client(CONST_TO_OR_CIRCUIT(circ)->p_chan);
+}
+
 /** Parse an HTTP request line at the start of a headers string.  On failure,
  * return -1.  On success, set *<b>command_out</b> to a copy of the HTTP
  * command ("get", "post", etc), set *<b>url_out</b> to a copy of the URL, and
@@ -618,34 +702,3 @@ dir_split_resource_into_fingerprints(const char *resource,
   smartlist_free(fp_tmp);
   return 0;
 }
-
-/** As dir_split_resource_into_fingerprints, but instead fills
- * <b>spool_out</b> with a list of spoolable_resource_t for the resource
- * identified through <b>source</b>. */
-int
-dir_split_resource_into_spoolable(const char *resource,
-                                  dir_spool_source_t source,
-                                  smartlist_t *spool_out,
-                                  int *compressed_out,
-                                  int flags)
-{
-  smartlist_t *fingerprints = smartlist_new();
-
-  tor_assert(flags & (DSR_HEX|DSR_BASE64));
-  const size_t digest_len =
-    (flags & DSR_DIGEST256) ? DIGEST256_LEN : DIGEST_LEN;
-
-  int r = dir_split_resource_into_fingerprints(resource, fingerprints,
-                                               compressed_out, flags);
-  /* This is not a very efficient implementation XXXX */
-  SMARTLIST_FOREACH_BEGIN(fingerprints, uint8_t *, digest) {
-    spooled_resource_t *spooled =
-      spooled_resource_new(source, digest, digest_len);
-    if (spooled)
-      smartlist_add(spool_out, spooled);
-    tor_free(digest);
-  } SMARTLIST_FOREACH_END(digest);
-
-  smartlist_free(fingerprints);
-  return r;
-}
diff --git a/src/feature/dircommon/directory.h b/src/feature/dircommon/directory.h
index ba3f8c1b0e..0f26cdeff9 100644
--- a/src/feature/dircommon/directory.h
+++ b/src/feature/dircommon/directory.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -94,6 +94,7 @@ int parse_http_command(const char *headers,
 char *http_get_header(const char *headers, const char *which);
 
 int connection_dir_is_encrypted(const dir_connection_t *conn);
+bool connection_dir_is_anonymous(const dir_connection_t *conn);
 int connection_dir_reached_eof(dir_connection_t *conn);
 int connection_dir_process_inbuf(dir_connection_t *conn);
 int connection_dir_finished_flushing(dir_connection_t *conn);
@@ -107,12 +108,6 @@ void connection_dir_about_to_close(dir_connection_t *dir_conn);
 int dir_split_resource_into_fingerprints(const char *resource,
                                      smartlist_t *fp_out, int *compressed_out,
                                      int flags);
-enum dir_spool_source_t;
-int dir_split_resource_into_spoolable(const char *resource,
-                                      enum dir_spool_source_t source,
-                                      smartlist_t *spool_out,
-                                      int *compressed_out,
-                                      int flags);
 int dir_split_resource_into_fingerprint_pairs(const char *res,
                                               smartlist_t *pairs_out);
 char *directory_dump_request_log(void);
diff --git a/src/feature/dircommon/feature_dircommon.md b/src/feature/dircommon/feature_dircommon.md
new file mode 100644
index 0000000000..359049ecd8
--- /dev/null
+++ b/src/feature/dircommon/feature_dircommon.md
@@ -0,0 +1,7 @@
+@dir /feature/dircommon
+@brief feature/dircommon: Directory client and server shared code
+
+This module has the code that directory clients (anybody who download
+information about relays) and directory servers (anybody who serves such
+information) share in common.
+
diff --git a/src/feature/dircommon/fp_pair.c b/src/feature/dircommon/fp_pair.c
index 284600df77..87e1c253bd 100644
--- a/src/feature/dircommon/fp_pair.c
+++ b/src/feature/dircommon/fp_pair.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,14 +22,14 @@
 
 /* Define fp_pair_map_t structures */
 
-struct fp_pair_map_entry_s {
-  HT_ENTRY(fp_pair_map_entry_s) node;
+struct fp_pair_map_entry_t {
+  HT_ENTRY(fp_pair_map_entry_t) node;
   void *val;
   fp_pair_t key;
 };
 
-struct fp_pair_map_s {
-  HT_HEAD(fp_pair_map_impl, fp_pair_map_entry_s) head;
+struct fp_pair_map_t {
+  HT_HEAD(fp_pair_map_impl, fp_pair_map_entry_t) head;
 };
 
 /*
@@ -56,11 +56,11 @@ fp_pair_map_entry_hash(const fp_pair_map_entry_t *a)
  * Hash table functions for fp_pair_map_t
  */
 
-HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_s, node,
-             fp_pair_map_entry_hash, fp_pair_map_entries_eq)
-HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_s, node,
+HT_PROTOTYPE(fp_pair_map_impl, fp_pair_map_entry_t, node,
+             fp_pair_map_entry_hash, fp_pair_map_entries_eq);
+HT_GENERATE2(fp_pair_map_impl, fp_pair_map_entry_t, node,
              fp_pair_map_entry_hash, fp_pair_map_entries_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 /** Constructor to create a new empty map from fp_pair_t to void *
  */
@@ -312,4 +312,3 @@ fp_pair_map_assert_ok(const fp_pair_map_t *map)
 {
   tor_assert(!fp_pair_map_impl_HT_REP_IS_BAD_(&(map->head)));
 }
-
diff --git a/src/feature/dircommon/fp_pair.h b/src/feature/dircommon/fp_pair.h
index 5041583e88..ae71ea7b71 100644
--- a/src/feature/dircommon/fp_pair.h
+++ b/src/feature/dircommon/fp_pair.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,8 @@ typedef struct {
  * Declare fp_pair_map_t functions and structs
  */
 
-typedef struct fp_pair_map_entry_s fp_pair_map_entry_t;
-typedef struct fp_pair_map_s fp_pair_map_t;
+typedef struct fp_pair_map_entry_t fp_pair_map_entry_t;
+typedef struct fp_pair_map_t fp_pair_map_t;
 typedef fp_pair_map_entry_t *fp_pair_map_iter_t;
 
 fp_pair_map_t * fp_pair_map_new(void);
diff --git a/src/feature/dircommon/include.am b/src/feature/dircommon/include.am
new file mode 100644
index 0000000000..87850ce183
--- /dev/null
+++ b/src/feature/dircommon/include.am
@@ -0,0 +1,14 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/dircommon/consdiff.c	\
+	src/feature/dircommon/directory.c	\
+	src/feature/dircommon/fp_pair.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dircommon/consdiff.h		\
+	src/feature/dircommon/dir_connection_st.h	\
+	src/feature/dircommon/directory.h		\
+	src/feature/dircommon/fp_pair.h			\
+	src/feature/dircommon/vote_timing_st.h
diff --git a/src/feature/dircommon/vote_timing_st.h b/src/feature/dircommon/vote_timing_st.h
index 47b90ab009..103d950f86 100644
--- a/src/feature/dircommon/vote_timing_st.h
+++ b/src/feature/dircommon/vote_timing_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file vote_timing_st.h
+ * @brief Directory voting schedule structure.
+ **/
+
 #ifndef VOTE_TIMING_ST_H
 #define VOTE_TIMING_ST_H
 
@@ -20,5 +25,4 @@ struct vote_timing_t {
   int dist_delay;
 };
 
-#endif
-
+#endif /* !defined(VOTE_TIMING_ST_H) */
diff --git a/src/feature/dirparse/.may_include b/src/feature/dirparse/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/dirparse/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/dirparse/authcert_members.h b/src/feature/dirparse/authcert_members.h
new file mode 100644
index 0000000000..53eab175d6
--- /dev/null
+++ b/src/feature/dirparse/authcert_members.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file authcert_members.h
+ *
+ * @brief List of tokens common to V3 authority certificates and V3
+ * consensuses.
+ **/
+
+#ifndef TOR_AUTHCERT_MEMBERS_H
+#define TOR_AUTHCERT_MEMBERS_H
+
+// clang-format off
+#define AUTHCERT_MEMBERS                                           \
+  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,      \
+                                                     GE(1),       NO_OBJ ), \
+  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),\
+  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),\
+  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),\
+  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),\
+  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,\
+                                                     NO_ARGS,     NEED_OBJ),\
+  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ)
+// clang-format on
+
+#endif /* !defined(TOR_AUTHCERT_MEMBERS_H) */
diff --git a/src/feature/dirparse/authcert_members.i b/src/feature/dirparse/authcert_members.i
deleted file mode 100644
index 08cffca97a..0000000000
--- a/src/feature/dirparse/authcert_members.i
+++ /dev/null
@@ -1,13 +0,0 @@
-/*
- * List of tokens common to V3 authority certificates and V3 consensuses.
- */
-  T1("dir-key-certificate-version", K_DIR_KEY_CERTIFICATE_VERSION,
-                                                     GE(1),       NO_OBJ ),
-  T1("dir-identity-key", K_DIR_IDENTITY_KEY,         NO_ARGS,     NEED_KEY ),
-  T1("dir-key-published",K_DIR_KEY_PUBLISHED,        CONCAT_ARGS, NO_OBJ),
-  T1("dir-key-expires",  K_DIR_KEY_EXPIRES,          CONCAT_ARGS, NO_OBJ),
-  T1("dir-signing-key",  K_DIR_SIGNING_KEY,          NO_ARGS,     NEED_KEY ),
-  T1("dir-key-crosscert", K_DIR_KEY_CROSSCERT,       NO_ARGS,     NEED_OBJ ),
-  T1("dir-key-certification", K_DIR_KEY_CERTIFICATION,
-                                                     NO_ARGS,     NEED_OBJ),
-  T01("dir-address",     K_DIR_ADDRESS,              GE(1),       NO_OBJ),
diff --git a/src/feature/dirparse/authcert_parse.c b/src/feature/dirparse/authcert_parse.c
index 1680bdbf30..deb45c12de 100644
--- a/src/feature/dirparse/authcert_parse.c
+++ b/src/feature/dirparse/authcert_parse.c
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file authcert_parse.c
+ * @brief Authority certificate parsing.
+ **/
+
 #include "core/or/or.h"
 #include "feature/dirparse/authcert_parse.h"
 #include "feature/dirparse/parsecommon.h"
@@ -13,18 +18,22 @@
 #include "lib/memarea/memarea.h"
 
 #include "feature/nodelist/authority_cert_st.h"
+#include "feature/dirparse/authcert_members.h"
 
 /** List of tokens recognized in V3 authority certificates. */
+// clang-format off
 static token_rule_t dir_key_certificate_table[] = {
-#include "feature/dirparse/authcert_members.i"
+  AUTHCERT_MEMBERS,
   T1("fingerprint",      K_FINGERPRINT,              CONCAT_ARGS, NO_OBJ ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** Parse a key certificate from <b>s</b>; point <b>end-of-string</b> to
  * the first character after the certificate. */
 authority_cert_t *
-authority_cert_parse_from_string(const char *s, const char **end_of_string)
+authority_cert_parse_from_string(const char *s, size_t maxlen,
+                                 const char **end_of_string)
 {
   /** Reject any certificate at least this big; it is probably an overflow, an
    * attack, a bug, or some other nonsense. */
@@ -35,24 +44,25 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
   char digest[DIGEST_LEN];
   directory_token_t *tok;
   char fp_declared[DIGEST_LEN];
-  char *eos;
+  const char *eos;
   size_t len;
   int found;
   memarea_t *area = NULL;
+  const char *end_of_s = s + maxlen;
   const char *s_dup = s;
 
-  s = eat_whitespace(s);
-  eos = strstr(s, "\ndir-key-certification");
+  s = eat_whitespace_eos(s, end_of_s);
+  eos = tor_memstr(s, end_of_s - s, "\ndir-key-certification");
   if (! eos) {
     log_warn(LD_DIR, "No signature found on key certificate");
     return NULL;
   }
-  eos = strstr(eos, "\n-----END SIGNATURE-----\n");
+  eos = tor_memstr(eos, end_of_s - eos, "\n-----END SIGNATURE-----\n");
   if (! eos) {
     log_warn(LD_DIR, "No end-of-signature found on key certificate");
     return NULL;
   }
-  eos = strchr(eos+2, '\n');
+  eos = memchr(eos+2, '\n', end_of_s - (eos+2));
   tor_assert(eos);
   ++eos;
   len = eos - s;
@@ -69,7 +79,7 @@ authority_cert_parse_from_string(const char *s, const char **end_of_string)
     log_warn(LD_DIR, "Error tokenizing key certificate");
     goto err;
   }
-  if (router_get_hash_impl(s, strlen(s), digest, "dir-key-certificate-version",
+  if (router_get_hash_impl(s, eos - s, digest, "dir-key-certificate-version",
                            "\ndir-key-certification", '\n', DIGEST_SHA1) < 0)
     goto err;
   tok = smartlist_get(tokens, 0);
diff --git a/src/feature/dirparse/authcert_parse.h b/src/feature/dirparse/authcert_parse.h
index ca475ad0e3..7f6dd1c02f 100644
--- a/src/feature/dirparse/authcert_parse.h
+++ b/src/feature/dirparse/authcert_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_AUTHCERT_PARSE_H
 
 authority_cert_t *authority_cert_parse_from_string(const char *s,
+                                                   size_t maxlen,
                                                    const char **end_of_string);
 
 #endif /* !defined(TOR_AUTHCERT_PARSE_H) */
diff --git a/src/feature/dirparse/feature_dirparse.md b/src/feature/dirparse/feature_dirparse.md
new file mode 100644
index 0000000000..e4b34668ba
--- /dev/null
+++ b/src/feature/dirparse/feature_dirparse.md
@@ -0,0 +1,8 @@
+@dir /feature/dirparse
+@brief feature/dirparse: Parsing Tor directory objects
+
+We define a number of "directory objects" in
+[dir-spec.txt](https://gitweb.torproject.org/torspec.git/tree/dir-spec.txt),
+all of them using a common line-oriented meta-format.  This module is used by
+other parts of Tor to parse them.
+
diff --git a/src/feature/dirparse/include.am b/src/feature/dirparse/include.am
new file mode 100644
index 0000000000..edca04f6f7
--- /dev/null
+++ b/src/feature/dirparse/include.am
@@ -0,0 +1,25 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/dirparse/authcert_parse.c	\
+	src/feature/dirparse/microdesc_parse.c	\
+	src/feature/dirparse/ns_parse.c		\
+	src/feature/dirparse/parsecommon.c	\
+	src/feature/dirparse/policy_parse.c	\
+	src/feature/dirparse/routerparse.c	\
+	src/feature/dirparse/sigcommon.c	\
+	src/feature/dirparse/signing.c		\
+	src/feature/dirparse/unparseable.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/dirparse/authcert_members.h		\
+	src/feature/dirparse/authcert_parse.h		\
+	src/feature/dirparse/microdesc_parse.h		\
+	src/feature/dirparse/ns_parse.h			\
+	src/feature/dirparse/parsecommon.h		\
+	src/feature/dirparse/policy_parse.h		\
+	src/feature/dirparse/routerparse.h		\
+	src/feature/dirparse/sigcommon.h		\
+	src/feature/dirparse/signing.h			\
+	src/feature/dirparse/unparseable.h
diff --git a/src/feature/dirparse/microdesc_parse.c b/src/feature/dirparse/microdesc_parse.c
index 5a75af3994..9231080aaa 100644
--- a/src/feature/dirparse/microdesc_parse.c
+++ b/src/feature/dirparse/microdesc_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/relay/router.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -27,17 +28,19 @@
 #include "feature/nodelist/microdesc_st.h"
 
 /** List of tokens recognized in microdescriptors */
+// clang-format off
 static token_rule_t microdesc_token_table[] = {
   T1_START("onion-key",        K_ONION_KEY,        NO_ARGS,     NEED_KEY_1024),
   T01("ntor-onion-key",        K_ONION_KEY_NTOR,   GE(1),       NO_OBJ ),
   T0N("id",                    K_ID,               GE(2),       NO_OBJ ),
   T0N("a",                     K_A,                GE(1),       NO_OBJ ),
-  T01("family",                K_FAMILY,           ARGS,        NO_OBJ ),
+  T01("family",                K_FAMILY,           CONCAT_ARGS, NO_OBJ ),
   T01("p",                     K_P,                CONCAT_ARGS, NO_OBJ ),
   T01("p6",                    K_P6,               CONCAT_ARGS, NO_OBJ ),
   A01("@last-listed",          A_LAST_LISTED,      CONCAT_ARGS, NO_OBJ ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** Assuming that s starts with a microdesc, return the start of the
  * *NEXT* one.  Return NULL on "not found." */
@@ -91,6 +94,190 @@ find_start_of_next_microdesc(const char *s, const char *eos)
 #undef NEXT_LINE
 }
 
+static inline int
+policy_is_reject_star_or_null(struct short_policy_t *policy)
+{
+  return !policy || short_policy_is_reject_star(policy);
+}
+
+/**
+ * Return a human-readable description of a given saved_location_t.
+ * Never returns NULL.
+ **/
+static const char *
+saved_location_to_string(saved_location_t where)
+{
+  const char *location;
+  switch (where) {
+    case SAVED_NOWHERE:
+      location = "download or generated string";
+      break;
+    case SAVED_IN_CACHE:
+      location = "cache";
+      break;
+    case SAVED_IN_JOURNAL:
+      location = "journal";
+      break;
+    default:
+      location = "unknown location";
+      break;
+  }
+  return location;
+}
+
+/**
+ * Given a microdescriptor stored in <b>where</b> which starts at <b>s</b>,
+ * which ends at <b>start_of_next_microdescriptor</b>, and which is located
+ * within a larger document beginning at <b>start</b>: Fill in the body,
+ * bodylen, bodylen, saved_location, off, and digest fields of <b>md</b> as
+ * appropriate.
+ *
+ * The body field will be an alias within <b>s</b> if <b>saved_location</b>
+ * is SAVED_IN_CACHE, and will be copied into body and nul-terminated
+ * otherwise.
+ **/
+static int
+microdesc_extract_body(microdesc_t *md,
+                       const char *start,
+                       const char *s, const char *start_of_next_microdesc,
+                       saved_location_t where)
+{
+  const bool copy_body = (where != SAVED_IN_CACHE);
+
+  const char *cp = tor_memstr(s, start_of_next_microdesc-s, "onion-key");
+
+  const bool no_onion_key = (cp == NULL);
+  if (no_onion_key) {
+    cp = s; /* So that we have *some* junk to put in the body */
+  }
+
+  md->bodylen = start_of_next_microdesc - cp;
+  md->saved_location = where;
+  if (copy_body)
+    md->body = tor_memdup_nulterm(cp, md->bodylen);
+  else
+    md->body = (char*)cp;
+  md->off = cp - start;
+
+  crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
+
+  return no_onion_key ? -1 : 0;
+}
+
+/**
+ * Parse a microdescriptor which begins at <b>s</b> and ends at
+ * <b>start_of_next_microdesc</b>.  Store its fields into <b>md</b>.  Use
+ * <b>where</b> for generating log information.  If <b>allow_annotations</b>
+ * is true, then one or more annotations may precede the microdescriptor body
+ * proper.  Use <b>area</b> for memory management, clearing it when done.
+ *
+ * On success, return 0; otherwise return -1.
+ **/
+static int
+microdesc_parse_fields(microdesc_t *md,
+                       memarea_t *area,
+                       const char *s, const char *start_of_next_microdesc,
+                       int allow_annotations,
+                       saved_location_t where)
+{
+  smartlist_t *tokens = smartlist_new();
+  int rv = -1;
+  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
+  directory_token_t *tok;
+
+  if (tokenize_string(area, s, start_of_next_microdesc, tokens,
+                      microdesc_token_table, flags)) {
+    log_warn(LD_DIR, "Unparseable microdescriptor found in %s",
+             saved_location_to_string(where));
+    goto err;
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
+    if (parse_iso_time(tok->args[0], &md->last_listed)) {
+      log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
+      goto err;
+    }
+  }
+
+  tok = find_by_keyword(tokens, K_ONION_KEY);
+  if (!crypto_pk_public_exponent_ok(tok->key)) {
+    log_warn(LD_DIR,
+             "Relay's onion key had invalid exponent.");
+    goto err;
+  }
+  md->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  md->onion_pkey_len = tok->object_size;
+  crypto_pk_free(tok->key);
+
+  if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
+    curve25519_public_key_t k;
+    tor_assert(tok->n_args >= 1);
+    if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
+      log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
+      goto err;
+    }
+    md->onion_curve25519_pkey =
+      tor_memdup(&k, sizeof(curve25519_public_key_t));
+  }
+
+  smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
+  if (id_lines) {
+    SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
+      tor_assert(t->n_args >= 2);
+      if (!strcmp(t->args[0], "ed25519")) {
+        if (md->ed25519_identity_pkey) {
+          log_warn(LD_DIR, "Extra ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        ed25519_public_key_t k;
+        if (ed25519_public_from_base64(&k, t->args[1])<0) {
+          log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
+          smartlist_free(id_lines);
+          goto err;
+        }
+        md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
+      }
+    } SMARTLIST_FOREACH_END(t);
+    smartlist_free(id_lines);
+  }
+
+  {
+    smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
+    if (a_lines) {
+      find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
+      smartlist_free(a_lines);
+    }
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
+    md->family = nodefamily_parse(tok->args[0],
+                                  NULL,
+                                  NF_WARN_MALFORMED);
+  }
+
+  if ((tok = find_opt_by_keyword(tokens, K_P))) {
+    md->exit_policy = parse_short_policy(tok->args[0]);
+  }
+  if ((tok = find_opt_by_keyword(tokens, K_P6))) {
+    md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
+  }
+
+  if (policy_is_reject_star_or_null(md->exit_policy) &&
+      policy_is_reject_star_or_null(md->ipv6_exit_policy)) {
+    md->policy_is_reject_star = 1;
+  }
+
+  rv = 0;
+ err:
+
+  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
+  memarea_clear(area);
+  smartlist_free(tokens);
+
+  return rv;
+}
+
 /** Parse as many microdescriptors as are found from the string starting at
  * <b>s</b> and ending at <b>eos</b>.  If allow_annotations is set, read any
  * annotations we recognize and ignore ones we don't.
@@ -108,16 +295,11 @@ microdescs_parse_from_string(const char *s, const char *eos,
                              saved_location_t where,
                              smartlist_t *invalid_digests_out)
 {
-  smartlist_t *tokens;
   smartlist_t *result;
   microdesc_t *md = NULL;
   memarea_t *area;
   const char *start = s;
   const char *start_of_next_microdesc;
-  int flags = allow_annotations ? TS_ANNOTATIONS_OK : 0;
-  const int copy_body = (where != SAVED_IN_CACHE);
-
-  directory_token_t *tok;
 
   if (!eos)
     eos = s + strlen(s);
@@ -125,143 +307,47 @@ microdescs_parse_from_string(const char *s, const char *eos,
   s = eat_whitespace_eos(s, eos);
   area = memarea_new();
   result = smartlist_new();
-  tokens = smartlist_new();
 
   while (s < eos) {
-    int okay = 0;
+   bool okay = false;
 
     start_of_next_microdesc = find_start_of_next_microdesc(s, eos);
     if (!start_of_next_microdesc)
       start_of_next_microdesc = eos;
 
     md = tor_malloc_zero(sizeof(microdesc_t));
+    uint8_t md_digest[DIGEST256_LEN];
     {
-      const char *cp = tor_memstr(s, start_of_next_microdesc-s,
-                                  "onion-key");
-      const int no_onion_key = (cp == NULL);
-      if (no_onion_key) {
-        cp = s; /* So that we have *some* junk to put in the body */
-      }
+      const bool body_not_found =
+        microdesc_extract_body(md, start, s,
+                               start_of_next_microdesc,
+                               where) < 0;
 
-      md->bodylen = start_of_next_microdesc - cp;
-      md->saved_location = where;
-      if (copy_body)
-        md->body = tor_memdup_nulterm(cp, md->bodylen);
-      else
-        md->body = (char*)cp;
-      md->off = cp - start;
-      crypto_digest256(md->digest, md->body, md->bodylen, DIGEST_SHA256);
-      if (no_onion_key) {
+      memcpy(md_digest, md->digest, DIGEST256_LEN);
+      if (body_not_found) {
         log_fn(LOG_PROTOCOL_WARN, LD_DIR, "Malformed or truncated descriptor");
         goto next;
       }
     }
 
-    if (tokenize_string(area, s, start_of_next_microdesc, tokens,
-                        microdesc_token_table, flags)) {
-      log_warn(LD_DIR, "Unparseable microdescriptor");
-      goto next;
+    if (microdesc_parse_fields(md, area, s, start_of_next_microdesc,
+                               allow_annotations, where) == 0) {
+      smartlist_add(result, md);
+      md = NULL; // prevent free
+      okay = true;
     }
 
-    if ((tok = find_opt_by_keyword(tokens, A_LAST_LISTED))) {
-      if (parse_iso_time(tok->args[0], &md->last_listed)) {
-        log_warn(LD_DIR, "Bad last-listed time in microdescriptor");
-        goto next;
-      }
-    }
-
-    tok = find_by_keyword(tokens, K_ONION_KEY);
-    if (!crypto_pk_public_exponent_ok(tok->key)) {
-      log_warn(LD_DIR,
-               "Relay's onion key had invalid exponent.");
-      goto next;
-    }
-    router_set_rsa_onion_pkey(tok->key, &md->onion_pkey,
-                              &md->onion_pkey_len);
-    crypto_pk_free(tok->key);
-
-    if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
-      curve25519_public_key_t k;
-      tor_assert(tok->n_args >= 1);
-      if (curve25519_public_from_base64(&k, tok->args[0]) < 0) {
-        log_warn(LD_DIR, "Bogus ntor-onion-key in microdesc");
-        goto next;
-      }
-      md->onion_curve25519_pkey =
-        tor_memdup(&k, sizeof(curve25519_public_key_t));
-    }
-
-    smartlist_t *id_lines = find_all_by_keyword(tokens, K_ID);
-    if (id_lines) {
-      SMARTLIST_FOREACH_BEGIN(id_lines, directory_token_t *, t) {
-        tor_assert(t->n_args >= 2);
-        if (!strcmp(t->args[0], "ed25519")) {
-          if (md->ed25519_identity_pkey) {
-            log_warn(LD_DIR, "Extra ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          ed25519_public_key_t k;
-          if (ed25519_public_from_base64(&k, t->args[1])<0) {
-            log_warn(LD_DIR, "Bogus ed25519 key in microdesc");
-            smartlist_free(id_lines);
-            goto next;
-          }
-          md->ed25519_identity_pkey = tor_memdup(&k, sizeof(k));
-        }
-      } SMARTLIST_FOREACH_END(t);
-      smartlist_free(id_lines);
-    }
-
-    {
-      smartlist_t *a_lines = find_all_by_keyword(tokens, K_A);
-      if (a_lines) {
-        find_single_ipv6_orport(a_lines, &md->ipv6_addr, &md->ipv6_orport);
-        smartlist_free(a_lines);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_FAMILY))) {
-      int i;
-      md->family = smartlist_new();
-      for (i=0;i<tok->n_args;++i) {
-        if (!is_legal_nickname_or_hexdigest(tok->args[i])) {
-          log_warn(LD_DIR, "Illegal nickname %s in family line",
-                   escaped(tok->args[i]));
-          goto next;
-        }
-        smartlist_add_strdup(md->family, tok->args[i]);
-      }
-    }
-
-    if ((tok = find_opt_by_keyword(tokens, K_P))) {
-      md->exit_policy = parse_short_policy(tok->args[0]);
-    }
-    if ((tok = find_opt_by_keyword(tokens, K_P6))) {
-      md->ipv6_exit_policy = parse_short_policy(tok->args[0]);
-    }
-
-    smartlist_add(result, md);
-    okay = 1;
-
-    md = NULL;
   next:
     if (! okay && invalid_digests_out) {
       smartlist_add(invalid_digests_out,
-                    tor_memdup(md->digest, DIGEST256_LEN));
+                    tor_memdup(md_digest, DIGEST256_LEN));
     }
     microdesc_free(md);
     md = NULL;
-
-    SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
-    memarea_clear(area);
-    smartlist_clear(tokens);
     s = start_of_next_microdesc;
   }
 
-  SMARTLIST_FOREACH(tokens, directory_token_t *, t, token_clear(t));
   memarea_drop_all(area);
-  smartlist_free(tokens);
 
   return result;
 }
diff --git a/src/feature/dirparse/microdesc_parse.h b/src/feature/dirparse/microdesc_parse.h
index 23a90084b1..e81126b8cd 100644
--- a/src/feature/dirparse/microdesc_parse.h
+++ b/src/feature/dirparse/microdesc_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@ smartlist_t *microdescs_parse_from_string(const char *s, const char *eos,
                                           saved_location_t where,
                                           smartlist_t *invalid_digests_out);
 
-#endif
+#endif /* !defined(TOR_MICRODESC_PARSE_H) */
diff --git a/src/feature/dirparse/ns_parse.c b/src/feature/dirparse/ns_parse.c
index 109eebeb66..ac9325a608 100644
--- a/src/feature/dirparse/ns_parse.c
+++ b/src/feature/dirparse/ns_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,12 +36,14 @@
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/networkstatus_voter_info_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
+#include "feature/dirparse/authcert_members.h"
 
 #undef log
 #include <math.h>
 
 /** List of tokens recognized in the body part of v3 networkstatus
  * documents. */
+// clang-format off
 static token_rule_t rtrstatus_token_table[] = {
   T01("p",                   K_P,               CONCAT_ARGS, NO_OBJ ),
   T1( "r",                   K_R,                   GE(7),   NO_OBJ ),
@@ -55,8 +57,10 @@ static token_rule_t rtrstatus_token_table[] = {
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in V3 networkstatus votes. */
+// clang-format off
 static token_rule_t networkstatus_token_table[] = {
   T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
                                                    GE(1),       NO_OBJ ),
@@ -84,7 +88,7 @@ static token_rule_t networkstatus_token_table[] = {
   T01("required-relay-protocols",    K_REQUIRED_RELAY_PROTOCOLS,
       CONCAT_ARGS, NO_OBJ ),
 
-#include "feature/dirparse/authcert_members.i"
+  AUTHCERT_MEMBERS,
 
   T0N("opt",                 K_OPT,             CONCAT_ARGS, OBJ_OK ),
   T1( "contact",             K_CONTACT,         CONCAT_ARGS, NO_OBJ ),
@@ -97,8 +101,10 @@ static token_rule_t networkstatus_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in V3 networkstatus consensuses. */
+// clang-format off
 static token_rule_t networkstatus_consensus_token_table[] = {
   T1_START("network-status-version", K_NETWORK_STATUS_VERSION,
                                                    GE(1),       NO_OBJ ),
@@ -135,14 +141,17 @@ static token_rule_t networkstatus_consensus_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in the footer of v1 directory footers. */
+// clang-format off
 static token_rule_t networkstatus_vote_footer_token_table[] = {
   T01("directory-footer",    K_DIRECTORY_FOOTER,    NO_ARGS,   NO_OBJ ),
   T01("bandwidth-weights",   K_BW_WEIGHTS,          ARGS,      NO_OBJ ),
   T(  "directory-signature", K_DIRECTORY_SIGNATURE, GE(2),     NEED_OBJ ),
   END_OF_TABLE
 };
+// clang-format on
 
 /** Try to find the start and end of the signed portion of a networkstatus
  * document in <b>s</b>. On success, set <b>start_out</b> to the first
@@ -151,10 +160,11 @@ static token_rule_t networkstatus_vote_footer_token_table[] = {
  * -1. */
 int
 router_get_networkstatus_v3_signed_boundaries(const char *s,
+                                              size_t len,
                                               const char **start_out,
                                               const char **end_out)
 {
-  return router_get_hash_impl_helper(s, strlen(s),
+  return router_get_hash_impl_helper(s, len,
                                      "network-status-version",
                                      "\ndirectory-signature",
                                      ' ', LOG_INFO,
@@ -166,12 +176,13 @@ router_get_networkstatus_v3_signed_boundaries(const char *s,
  * signed portion can be identified.  Return 0 on success, -1 on failure. */
 int
 router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                           const char *s)
+                                           const char *s, size_t len)
 {
   const char *start, *end;
-  if (router_get_networkstatus_v3_signed_boundaries(s, &start, &end) < 0) {
+  if (router_get_networkstatus_v3_signed_boundaries(s, len,
+                                                    &start, &end) < 0) {
     start = s;
-    end = s + strlen(s);
+    end = s + len;
   }
   tor_assert(start);
   tor_assert(end);
@@ -182,9 +193,10 @@ router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
 /** Set <b>digests</b> to all the digests of the consensus document in
  * <b>s</b> */
 int
-router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
+router_get_networkstatus_v3_hashes(const char *s, size_t len,
+                                   common_digests_t *digests)
 {
-  return router_get_hashes_impl(s,strlen(s),digests,
+  return router_get_hashes_impl(s, len, digests,
                                 "network-status-version",
                                 "\ndirectory-signature",
                                 ' ');
@@ -195,13 +207,13 @@ router_get_networkstatus_v3_hashes(const char *s, common_digests_t *digests)
  * return the start of the directory footer, or the next directory signature.
  * If none is found, return the end of the string. */
 static inline const char *
-find_start_of_next_routerstatus(const char *s)
+find_start_of_next_routerstatus(const char *s, const char *s_eos)
 {
   const char *eos, *footer, *sig;
-  if ((eos = strstr(s, "\nr ")))
+  if ((eos = tor_memstr(s, s_eos - s, "\nr ")))
     ++eos;
   else
-    eos = s + strlen(s);
+    eos = s_eos;
 
   footer = tor_memstr(s, eos-s, "\ndirectory-footer");
   sig = tor_memstr(s, eos-s, "\ndirectory-signature");
@@ -289,7 +301,8 @@ routerstatus_parse_guardfraction(const char *guardfraction_str,
  **/
 STATIC routerstatus_t *
 routerstatus_parse_entry_from_string(memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *s_eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
@@ -308,7 +321,7 @@ routerstatus_parse_entry_from_string(memarea_t *area,
     flav = FLAV_NS;
   tor_assert(flav == FLAV_NS || flav == FLAV_MICRODESC);
 
-  eos = find_start_of_next_routerstatus(*s);
+  eos = find_start_of_next_routerstatus(*s, s_eos);
 
   if (tokenize_string(area,*s, eos, tokens, rtrstatus_token_table,0)) {
     log_warn(LD_DIR, "Error tokenizing router status");
@@ -430,6 +443,8 @@ routerstatus_parse_entry_from_string(memarea_t *area,
         rs->is_hs_dir = 1;
       } else if (!strcmp(tok->args[i], "V2Dir")) {
         rs->is_v2_dir = 1;
+      } else if (!strcmp(tok->args[i], "StaleDesc")) {
+        rs->is_staledesc = 1;
       }
     }
     /* These are implied true by having been included in a consensus made
@@ -1051,7 +1066,9 @@ extract_shared_random_srvs(networkstatus_t *ns, smartlist_t *tokens)
 /** Parse a v3 networkstatus vote, opinion, or consensus (depending on
  * ns_type), from <b>s</b>, and return the result.  Return NULL on failure. */
 networkstatus_t *
-networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
+networkstatus_parse_vote_from_string(const char *s,
+                                     size_t s_len,
+                                     const char **eos_out,
                                      networkstatus_type_t ns_type)
 {
   smartlist_t *tokens = smartlist_new();
@@ -1067,20 +1084,22 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   memarea_t *area = NULL, *rs_area = NULL;
   consensus_flavor_t flav = FLAV_NS;
   char *last_kwd=NULL;
+  const char *eos = s + s_len;
 
   tor_assert(s);
 
   if (eos_out)
     *eos_out = NULL;
 
-  if (router_get_networkstatus_v3_hashes(s, &ns_digests) ||
-      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed, s)<0) {
+  if (router_get_networkstatus_v3_hashes(s, s_len, &ns_digests) ||
+      router_get_networkstatus_v3_sha3_as_signed(sha3_as_signed,
+                                                 s, s_len)<0) {
     log_warn(LD_DIR, "Unable to compute digest of network-status");
     goto err;
   }
 
   area = memarea_new();
-  end_of_header = find_start_of_next_routerstatus(s);
+  end_of_header = find_start_of_next_routerstatus(s, eos);
   if (tokenize_string(area, s, end_of_header, tokens,
                       (ns_type == NS_TYPE_CONSENSUS) ?
                       networkstatus_consensus_token_table :
@@ -1111,10 +1130,12 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   if (ns_type != NS_TYPE_CONSENSUS) {
     const char *end_of_cert = NULL;
-    if (!(cert = strstr(s, "\ndir-key-certificate-version")))
+    if (!(cert = tor_memstr(s, end_of_header - s,
+                            "\ndir-key-certificate-version")))
       goto err;
     ++cert;
-    ns->cert = authority_cert_parse_from_string(cert, &end_of_cert);
+    ns->cert = authority_cert_parse_from_string(cert, end_of_header - cert,
+                                                &end_of_cert);
     if (!ns->cert || !end_of_cert || end_of_cert > end_of_header)
       goto err;
   }
@@ -1424,10 +1445,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
   s = end_of_header;
   ns->routerstatus_list = smartlist_new();
 
-  while (!strcmpstart(s, "r ")) {
+  while (eos - s >= 2 && fast_memeq(s, "r ", 2)) {
     if (ns->type != NS_TYPE_CONSENSUS) {
       vote_routerstatus_t *rs = tor_malloc_zero(sizeof(vote_routerstatus_t));
-      if (routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens, ns,
+      if (routerstatus_parse_entry_from_string(rs_area, &s, eos, rs_tokens, ns,
                                                rs, 0, 0)) {
         smartlist_add(ns->routerstatus_list, rs);
       } else {
@@ -1435,7 +1456,8 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
       }
     } else {
       routerstatus_t *rs;
-      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, rs_tokens,
+      if ((rs = routerstatus_parse_entry_from_string(rs_area, &s, eos,
+                                                     rs_tokens,
                                                      NULL, NULL,
                                                      ns->consensus_method,
                                                      flav))) {
@@ -1465,7 +1487,7 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
     SMARTLIST_FOREACH_BEGIN(ns->routerstatus_list, vote_routerstatus_t *,
                             vrs) {
       if (! vrs->has_ed25519_listing ||
-          tor_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
+          fast_mem_is_zero((const char *)vrs->ed25519_id, DIGEST256_LEN))
         continue;
       if (digest256map_get(ed_id_map, vrs->ed25519_id) != NULL) {
         log_warn(LD_DIR, "Vote networkstatus ed25519 identities were not "
@@ -1480,10 +1502,10 @@ networkstatus_parse_vote_from_string(const char *s, const char **eos_out,
 
   /* Parse footer; check signature. */
   footer_tokens = smartlist_new();
-  if ((end_of_footer = strstr(s, "\nnetwork-status-version ")))
+  if ((end_of_footer = tor_memstr(s, eos-s, "\nnetwork-status-version ")))
     ++end_of_footer;
   else
-    end_of_footer = s + strlen(s);
+    end_of_footer = eos;
   if (tokenize_string(area,s, end_of_footer, footer_tokens,
                       networkstatus_vote_footer_token_table, 0)) {
     log_warn(LD_DIR, "Error tokenizing network-status vote footer.");
diff --git a/src/feature/dirparse/ns_parse.h b/src/feature/dirparse/ns_parse.h
index 10a6f9cefc..6a1ea85c92 100644
--- a/src/feature/dirparse/ns_parse.h
+++ b/src/feature/dirparse/ns_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,18 +12,19 @@
 #ifndef TOR_NS_PARSE_H
 #define TOR_NS_PARSE_H
 
-int router_get_networkstatus_v3_hashes(const char *s,
+int router_get_networkstatus_v3_hashes(const char *s, size_t len,
                                        common_digests_t *digests);
-int router_get_networkstatus_v3_signed_boundaries(const char *s,
+int router_get_networkstatus_v3_signed_boundaries(const char *s, size_t len,
                                                   const char **start_out,
                                                   const char **end_out);
 int router_get_networkstatus_v3_sha3_as_signed(uint8_t *digest_out,
-                                               const char *s);
+                                               const char *s, size_t len);
 int compare_vote_routerstatus_entries(const void **_a, const void **_b);
 
 int networkstatus_verify_bw_weights(networkstatus_t *ns, int);
 enum networkstatus_type_t;
 networkstatus_t *networkstatus_parse_vote_from_string(const char *s,
+                                           size_t len,
                                            const char **eos_out,
                                            enum networkstatus_type_t ns_type);
 
@@ -35,11 +36,12 @@ STATIC int routerstatus_parse_guardfraction(const char *guardfraction_str,
 struct memarea_t;
 STATIC routerstatus_t *routerstatus_parse_entry_from_string(
                                      struct memarea_t *area,
-                                     const char **s, smartlist_t *tokens,
+                                     const char **s, const char *eos,
+                                     smartlist_t *tokens,
                                      networkstatus_t *vote,
                                      vote_routerstatus_t *vote_rs,
                                      int consensus_method,
                                      consensus_flavor_t flav);
-#endif
+#endif /* defined(NS_PARSE_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NS_PARSE_H) */
diff --git a/src/feature/dirparse/parsecommon.c b/src/feature/dirparse/parsecommon.c
index 1664a77bbe..ab465c4d7f 100644
--- a/src/feature/dirparse/parsecommon.c
+++ b/src/feature/dirparse/parsecommon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,6 +15,7 @@
 #include "lib/string/printf.h"
 #include "lib/memarea/memarea.h"
 #include "lib/crypt_ops/crypto_rsa.h"
+#include "lib/ctime/di_ops.h"
 
 #include <string.h>
 
@@ -169,7 +170,6 @@ get_token_arguments(memarea_t *area, directory_token_t *tok,
   char *cp = mem;
   int j = 0;
   char *args[MAX_ARGS];
-  memset(args, 0, sizeof(args));
   while (*cp) {
     if (j == MAX_ARGS)
       return -1;
@@ -251,6 +251,16 @@ token_check_object(memarea_t *area, const char *kwd,
   return tok;
 }
 
+/** Return true iff the <b>memlen</b>-byte chunk of memory at
+ * <b>memlen</b> is the same length as <b>token</b>, and their
+ * contents are equal. */
+static bool
+mem_eq_token(const void *mem, size_t memlen, const char *token)
+{
+  size_t len = strlen(token);
+  return memlen == len && fast_memeq(mem, token, len);
+}
+
 /** Helper function: read the next token from *s, advance *s to the end of the
  * token, and return the parsed token.  Parse *<b>s</b> according to the list
  * of tokens in <b>table</b>.
@@ -266,7 +276,7 @@ get_next_token(memarea_t *area,
    * attack, a bug, or some other nonsense. */
 #define MAX_LINE_LENGTH (128*1024)
 
-  const char *next, *eol, *obstart;
+  const char *next, *eol;
   size_t obname_len;
   int i;
   directory_token_t *tok;
@@ -290,7 +300,7 @@ get_next_token(memarea_t *area,
 
   next = find_whitespace_eos(*s, eol);
 
-  if (!strcmp_len(*s, "opt", next-*s)) {
+  if (mem_eq_token(*s, next-*s, "opt")) {
     /* Skip past an "opt" at the start of the line. */
     *s = eat_whitespace_eos_no_nl(next, eol);
     next = find_whitespace_eos(*s, eol);
@@ -301,7 +311,7 @@ get_next_token(memarea_t *area,
   /* Search the table for the appropriate entry.  (I tried a binary search
    * instead, but it wasn't any faster.) */
   for (i = 0; table[i].t ; ++i) {
-    if (!strcmp_len(*s, table[i].t, next-*s)) {
+    if (mem_eq_token(*s, next-*s, table[i].t)) {
       /* We've found the keyword. */
       kwd = table[i].t;
       tok->tp = table[i].v;
@@ -352,9 +362,8 @@ get_next_token(memarea_t *area,
   if (!eol || eol-*s<11 || strcmpstart(*s, "-----BEGIN ")) /* No object. */
     goto check_object;
 
-  obstart = *s; /* Set obstart to start of object spec */
   if (eol - *s <= 16 || memchr(*s+11,'\0',eol-*s-16) || /* no short lines, */
-      strcmp_len(eol-5, "-----", 5) ||           /* nuls or invalid endings */
+      !mem_eq_token(eol-5, 5, "-----") ||   /* nuls or invalid endings */
       (eol-*s) > MAX_UNPARSED_OBJECT_SIZE) {     /* name too long */
     RET_ERR("Malformed object: bad begin line");
   }
@@ -373,8 +382,8 @@ get_next_token(memarea_t *area,
     eol = eos;
   /* Validate the ending tag, which should be 9 + NAME + 5 + eol */
   if ((size_t)(eol-next) != 9+obname_len+5 ||
-      strcmp_len(next+9, tok->object_type, obname_len) ||
-      strcmp_len(eol-5, "-----", 5)) {
+      !mem_eq_token(next+9, obname_len, tok->object_type) ||
+      !mem_eq_token(eol-5, 5, "-----")) {
     tor_snprintf(ebuf, sizeof(ebuf), "Malformed object: mismatched end tag %s",
              tok->object_type);
     ebuf[sizeof(ebuf)-1] = '\0';
@@ -383,28 +392,32 @@ get_next_token(memarea_t *area,
   if (next - *s > MAX_UNPARSED_OBJECT_SIZE)
     RET_ERR("Couldn't parse object: missing footer or object much too big.");
 
+  {
+    int r;
+    size_t maxsize = base64_decode_maxsize(next-*s);
+    tok->object_body = ALLOC(maxsize);
+    r = base64_decode(tok->object_body, maxsize, *s, next-*s);
+    if (r<0)
+      RET_ERR("Malformed object: bad base64-encoded data");
+    tok->object_size = r;
+  }
+
   if (!strcmp(tok->object_type, "RSA PUBLIC KEY")) { /* If it's a public key */
     if (o_syn != NEED_KEY && o_syn != NEED_KEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected public key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_public_key_from_string(tok->key, obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode(tok->object_body, tok->object_size);
+    if (! tok->key)
       RET_ERR("Couldn't parse public key.");
   } else if (!strcmp(tok->object_type, "RSA PRIVATE KEY")) { /* private key */
     if (o_syn != NEED_SKEY_1024 && o_syn != OBJ_OK) {
       RET_ERR("Unexpected private key.");
     }
-    tok->key = crypto_pk_new();
-    if (crypto_pk_read_private_key1024_from_string(tok->key,
-                                                   obstart, eol-obstart))
+    tok->key = crypto_pk_asn1_decode_private(tok->object_body,
+                                             tok->object_size,
+                                             1024);
+    if (! tok->key)
       RET_ERR("Couldn't parse private key.");
-  } else { /* If it's something else, try to base64-decode it */
-    int r;
-    tok->object_body = ALLOC(next-*s); /* really, this is too much RAM. */
-    r = base64_decode(tok->object_body, next-*s, *s, next-*s);
-    if (r<0)
-      RET_ERR("Malformed object: bad base64-encoded data");
-    tok->object_size = r;
   }
   *s = eol;
 
diff --git a/src/feature/dirparse/parsecommon.h b/src/feature/dirparse/parsecommon.h
index ef74925b26..4db9a89f13 100644
--- a/src/feature/dirparse/parsecommon.h
+++ b/src/feature/dirparse/parsecommon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/policy_parse.c b/src/feature/dirparse/policy_parse.c
index 7562ae409b..28cd174686 100644
--- a/src/feature/dirparse/policy_parse.c
+++ b/src/feature/dirparse/policy_parse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,7 +9,7 @@
  * \brief Code to parse address policies.
  **/
 
-#define  EXPOSE_ROUTERDESC_TOKEN_TABLE
+#define  ROUTERDESC_TOKEN_TABLE_PRIVATE
 
 #include "core/or/or.h"
 
diff --git a/src/feature/dirparse/policy_parse.h b/src/feature/dirparse/policy_parse.h
index e09ee5559f..7764069e66 100644
--- a/src/feature/dirparse/policy_parse.h
+++ b/src/feature/dirparse/policy_parse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/routerparse.c b/src/feature/dirparse/routerparse.c
index e44fbf77f9..8828a0f97a 100644
--- a/src/feature/dirparse/routerparse.c
+++ b/src/feature/dirparse/routerparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -50,7 +50,7 @@
  * </ul>
  **/
 
-#define EXPOSE_ROUTERDESC_TOKEN_TABLE
+#define ROUTERDESC_TOKEN_TABLE_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
@@ -81,6 +81,7 @@
 /****************************************************************************/
 
 /** List of tokens recognized in router descriptors */
+// clang-format off
 const token_rule_t routerdesc_token_table[] = {
   T0N("reject",              K_REJECT,              ARGS,    NO_OBJ ),
   T0N("accept",              K_ACCEPT,              ARGS,    NO_OBJ ),
@@ -123,8 +124,10 @@ const token_rule_t routerdesc_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 /** List of tokens recognized in extra-info documents. */
+// clang-format off
 static token_rule_t extrainfo_token_table[] = {
   T1_END( "router-signature",    K_ROUTER_SIGNATURE,    NO_ARGS, NEED_OBJ ),
   T1( "published",           K_PUBLISHED,       CONCAT_ARGS, NO_OBJ ),
@@ -162,6 +165,7 @@ static token_rule_t extrainfo_token_table[] = {
 
   END_OF_TABLE
 };
+// clang-format on
 
 #undef T
 
@@ -591,8 +595,8 @@ router_parse_entry_from_string(const char *s, const char *end,
              "Relay's onion key had invalid exponent.");
     goto err;
   }
-  router_set_rsa_onion_pkey(tok->key, &router->onion_pkey,
-                            &router->onion_pkey_len);
+  router->onion_pkey = tor_memdup(tok->object_body, tok->object_size);
+  router->onion_pkey_len = tok->object_size;
   crypto_pk_free(tok->key);
 
   if ((tok = find_opt_by_keyword(tokens, K_ONION_KEY_NTOR))) {
diff --git a/src/feature/dirparse/routerparse.h b/src/feature/dirparse/routerparse.h
index f9a13f2168..519044e9b0 100644
--- a/src/feature/dirparse/routerparse.h
+++ b/src/feature/dirparse/routerparse.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,7 +40,8 @@ int find_single_ipv6_orport(const smartlist_t *list,
 void routerparse_init(void);
 void routerparse_free_all(void);
 
-#ifdef EXPOSE_ROUTERDESC_TOKEN_TABLE
+#ifdef ROUTERDESC_TOKEN_TABLE_PRIVATE
+#include "feature/dirparse/parsecommon.h"
 extern const struct token_rule_t routerdesc_token_table[];
 #endif
 
diff --git a/src/feature/dirparse/sigcommon.c b/src/feature/dirparse/sigcommon.c
index 2019e09918..8b970d7d1f 100644
--- a/src/feature/dirparse/sigcommon.c
+++ b/src/feature/dirparse/sigcommon.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/sigcommon.h b/src/feature/dirparse/sigcommon.h
index fdd8e839a9..c2ed9df494 100644
--- a/src/feature/dirparse/sigcommon.h
+++ b/src/feature/dirparse/sigcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,6 +43,6 @@ MOCK_DECL(STATIC int, signed_digest_equals,
 MOCK_DECL(STATIC int, router_compute_hash_final,(char *digest,
                            const char *start, size_t len,
                            digest_algorithm_t alg));
-#endif
+#endif /* defined(SIGCOMMON_PRIVATE) */
 
 #endif /* !defined(TOR_SIGCOMMON_H) */
diff --git a/src/feature/dirparse/signing.c b/src/feature/dirparse/signing.c
index 3ab40c3807..e420e5b6b9 100644
--- a/src/feature/dirparse/signing.c
+++ b/src/feature/dirparse/signing.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/dirparse/signing.h b/src/feature/dirparse/signing.h
index 2e3699baf8..7ca34bb14a 100644
--- a/src/feature/dirparse/signing.h
+++ b/src/feature/dirparse/signing.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
                                    const char *digest,
                                    size_t digest_len,
                                    crypto_pk_t *private_key);
-#endif
+#endif /* !defined(TOR_SIGNING_H) */
diff --git a/src/feature/dirparse/unparseable.c b/src/feature/dirparse/unparseable.c
index 1d623fe701..a91148a661 100644
--- a/src/feature/dirparse/unparseable.c
+++ b/src/feature/dirparse/unparseable.c
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file unparseable.c
+ * @brief Dump unparseable objects to disk.
+ **/
+
 #define UNPARSEABLE_PRIVATE
 
 #include "core/or/or.h"
diff --git a/src/feature/dirparse/unparseable.h b/src/feature/dirparse/unparseable.h
index 853fe8cb0f..cff91c82cc 100644
--- a/src/feature/dirparse/unparseable.h
+++ b/src/feature/dirparse/unparseable.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,7 +26,7 @@ void dump_desc_init(void);
   log_debug(LD_MM, "Area for %s has %lu allocated; using %lu.",   \
             name, (unsigned long)alloc, (unsigned long)used);     \
   STMT_END
-#else /* !(defined(DEBUG_AREA_ALLOC)) */
+#else /* !defined(DEBUG_AREA_ALLOC) */
 #define DUMP_AREA(a,name) STMT_NIL
 #endif /* defined(DEBUG_AREA_ALLOC) */
 
@@ -51,6 +51,6 @@ EXTERN(struct smartlist_t *, descs_dumped)
 MOCK_DECL(STATIC dumped_desc_t *, dump_desc_populate_one_file,
     (const char *dirname, const char *f));
 STATIC void dump_desc_populate_fifo_from_directory(const char *dirname);
-#endif
+#endif /* defined(UNPARSEABLE_PRIVATE) */
 
 #endif /* !defined(TOR_UNPARSEABLE_H) */
diff --git a/src/feature/feature.md b/src/feature/feature.md
new file mode 100644
index 0000000000..acc3487e55
--- /dev/null
+++ b/src/feature/feature.md
@@ -0,0 +1,7 @@
+@dir /feature
+@brief feature: domain-specific modules
+
+The "feature" directory has modules that Tor uses only for a particular
+role or service, such as maintaining/using an onion service, operating as a
+relay or a client, or being a directory authority.
+
diff --git a/src/feature/hibernate/.may_include b/src/feature/hibernate/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/hibernate/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/hibernate/feature_hibernate.md b/src/feature/hibernate/feature_hibernate.md
new file mode 100644
index 0000000000..0eb5ffea0d
--- /dev/null
+++ b/src/feature/hibernate/feature_hibernate.md
@@ -0,0 +1,14 @@
+@dir /feature/hibernate
+@brief feature/hibernate: Bandwidth accounting and hibernation (!)
+
+This module implements two features that are only somewhat related, and
+should probably be separated in the future.  One feature is bandwidth
+accounting (making sure we use no more than so many gigabytes in a day) and
+hibernation (avoiding network activity while we have used up all/most of our
+configured gigabytes).  The other feature is clean shutdown, where we stop
+accepting new connections for a while and give the old ones time to close.
+
+The two features are related only in the sense that "soft hibernation" (being
+almost out of ) is very close to the "shutting down" state.  But it would be
+better in the long run to make the two completely separate.
+
diff --git a/src/feature/hibernate/hibernate.c b/src/feature/hibernate/hibernate.c
index 09932c97ac..82c33659aa 100644
--- a/src/feature/hibernate/hibernate.c
+++ b/src/feature/hibernate/hibernate.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,8 +35,9 @@ hibernating, phase 2:
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "feature/hibernate/hibernate.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/relay/router.h"
@@ -56,7 +57,7 @@ hibernating, phase 2:
  * Coverity. Here's a kludge to unconfuse it.
  */
 #   define __INCLUDE_LEVEL__ 2
-# endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
+#endif /* defined(__COVERITY__) && !defined(__INCLUDE_LEVEL__) */
 #include <systemd/sd-daemon.h>
 #endif /* defined(HAVE_SYSTEMD) */
 
@@ -66,8 +67,9 @@ static hibernate_state_t hibernate_state = HIBERNATE_STATE_INITIAL;
 /** If are hibernating, when do we plan to wake up? Set to 0 if we
  * aren't hibernating. */
 static time_t hibernate_end_time = 0;
-/** If we are shutting down, when do we plan finally exit? Set to 0 if
- * we aren't shutting down. */
+/** If we are shutting down, when do we plan to finally exit? Set to 0 if we
+ * aren't shutting down. (This is obsolete; scheduled shutdowns are supposed
+ * to happen from mainloop_schedule_shutdown() now.) */
 static time_t shutdown_time = 0;
 
 /** A timed event that we'll use when it's time to wake up from
@@ -560,7 +562,7 @@ time_to_record_bandwidth_usage(time_t now)
   /* Note every 600 sec */
 #define NOTE_INTERVAL (600)
   /* Or every 20 megabytes */
-#define NOTE_BYTES 20*(1024*1024)
+#define NOTE_BYTES (20*1024*1024)
   static uint64_t last_read_bytes_noted = 0;
   static uint64_t last_written_bytes_noted = 0;
   static time_t last_time_noted = 0;
@@ -813,7 +815,7 @@ hibernate_soft_limit_reached(void)
    * We want to stop accepting connections when ALL of the following are true:
    *   - We expect to use up the remaining bytes in under 3 hours
    *   - We have used up 95% of our bytes.
-   *   - We have less than 500MB of bytes left.
+   *   - We have less than 500MBytes left.
    */
   uint64_t soft_limit = (uint64_t) (acct_max * SOFT_LIM_PCT);
   if (acct_max > SOFT_LIM_BYTES && acct_max - SOFT_LIM_BYTES > soft_limit) {
@@ -831,8 +833,6 @@ hibernate_soft_limit_reached(void)
   return get_accounting_bytes() >= soft_limit;
 }
 
-#define TOR_USEC_PER_SEC (1000000)
-
 /** Called when we get a SIGINT, or when bandwidth soft limit is
  * reached. Puts us into "loose hibernation": we don't accept new
  * connections, but we continue handling old ones. */
@@ -867,7 +867,13 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
     log_notice(LD_GENERAL,"Interrupt: we have stopped accepting new "
                "connections, and will shut down in %d seconds. Interrupt "
                "again to exit now.", options->ShutdownWaitLength);
-    shutdown_time = time(NULL) + options->ShutdownWaitLength;
+    /* We add an arbitrary delay here so that even if something goes wrong
+     * with the mainloop shutdown code, we can still shutdown from
+     * consider_hibernation() if we call it... but so that the
+     * mainloop_schedule_shutdown() mechanism will be the first one called.
+     */
+    shutdown_time = time(NULL) + options->ShutdownWaitLength + 5;
+    mainloop_schedule_shutdown(options->ShutdownWaitLength);
 #ifdef HAVE_SYSTEMD
     /* tell systemd that we may need more than the default 90 seconds to shut
      * down so they don't kill us. add some extra time to actually finish
@@ -887,7 +893,7 @@ hibernate_begin(hibernate_state_t new_state, time_t now)
      */
     sd_notifyf(0, "EXTEND_TIMEOUT_USEC=%" PRIu64,
             ((uint64_t)(options->ShutdownWaitLength) + 30) * TOR_USEC_PER_SEC);
-#endif
+#endif /* defined(HAVE_SYSTEMD) */
   } else { /* soft limit reached */
     hibernate_end_time = interval_end_time;
   }
@@ -1096,11 +1102,12 @@ consider_hibernation(time_t now)
   hibernate_state_t prev_state = hibernate_state;
 
   /* If we're in 'exiting' mode, then we just shut down after the interval
-   * elapses. */
+   * elapses.  The mainloop was supposed to catch this via
+   * mainloop_schedule_shutdown(), but apparently it didn't. */
   if (hibernate_state == HIBERNATE_STATE_EXITING) {
     tor_assert(shutdown_time);
     if (shutdown_time <= now) {
-      log_notice(LD_GENERAL, "Clean shutdown finished. Exiting.");
+      log_notice(LD_BUG, "Mainloop did not catch shutdown event; exiting.");
       tor_shutdown_event_loop_and_exit(0);
     }
     return; /* if exiting soon, don't worry about bandwidth limits */
@@ -1112,7 +1119,7 @@ consider_hibernation(time_t now)
     if (hibernate_end_time > now && accounting_enabled) {
       /* If we're hibernating, don't wake up until it's time, regardless of
        * whether we're in a new interval. */
-      return ;
+      return;
     } else {
       hibernate_end_time_elapsed(now);
     }
@@ -1240,8 +1247,6 @@ on_hibernate_state_change(hibernate_state_t prev_state)
   if (prev_state != HIBERNATE_STATE_INITIAL) {
     rescan_periodic_events(get_options());
   }
-
-  reschedule_per_second_timer();
 }
 
 /** Free all resources held by the accounting module */
diff --git a/src/feature/hibernate/hibernate.h b/src/feature/hibernate/hibernate.h
index 3309ef0ce3..2383658b20 100644
--- a/src/feature/hibernate/hibernate.h
+++ b/src/feature/hibernate/hibernate.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,6 +32,7 @@ int getinfo_helper_accounting(control_connection_t *conn,
                               const char **errmsg);
 uint64_t get_accounting_max_total(void);
 void accounting_free_all(void);
+bool accounting_tor_is_dormant(void);
 
 #ifdef HIBERNATE_PRIVATE
 /** Possible values of hibernate_state */
diff --git a/src/feature/hibernate/include.am b/src/feature/hibernate/include.am
new file mode 100644
index 0000000000..355e591392
--- /dev/null
+++ b/src/feature/hibernate/include.am
@@ -0,0 +1,8 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/hibernate/hibernate.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/hibernate/hibernate.h
diff --git a/src/feature/hs/.may_include b/src/feature/hs/.may_include
new file mode 100644
index 0000000000..11c5ffbb14
--- /dev/null
+++ b/src/feature/hs/.may_include
@@ -0,0 +1,2 @@
+*.h
+*.inc
diff --git a/src/feature/hs/feature_hs.md b/src/feature/hs/feature_hs.md
new file mode 100644
index 0000000000..299d07e014
--- /dev/null
+++ b/src/feature/hs/feature_hs.md
@@ -0,0 +1,8 @@
+@dir /feature/hs
+@brief feature/hs: v3 (current) onion service protocol
+
+This directory implements the v3 onion service protocol,
+as specified in
+[rend-spec-v3.txt](https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt).
+
+
diff --git a/src/feature/hs/hs_cache.c b/src/feature/hs/hs_cache.c
index 042ec55fa4..ef5e88e947 100644
--- a/src/feature/hs/hs_cache.c
+++ b/src/feature/hs/hs_cache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,12 +28,27 @@
 static int cached_client_descriptor_has_expired(time_t now,
            const hs_cache_client_descriptor_t *cached_desc);
 
+/** Helper function: Return true iff the cache entry has a decrypted
+ * descriptor.
+ *
+ * A NULL desc object in the entry means that we were not able to decrypt the
+ * descriptor because we are likely lacking client authorization.  It is still
+ * a valid entry but some operations can't be done without the decrypted
+ * descriptor thus this function MUST be used to safe guard access to the
+ * decrypted desc object. */
+static inline bool
+entry_has_decrypted_descriptor(const hs_cache_client_descriptor_t *entry)
+{
+  tor_assert(entry);
+  return (entry->desc != NULL);
+}
+
 /********************** Directory HS cache ******************/
 
-/* Directory descriptor cache. Map indexed by blinded key. */
+/** Directory descriptor cache. Map indexed by blinded key. */
 static digest256map_t *hs_cache_v3_dir;
 
-/* Remove a given descriptor from our cache. */
+/** Remove a given descriptor from our cache. */
 static void
 remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
 {
@@ -41,7 +56,7 @@ remove_v3_desc_as_dir(const hs_cache_dir_descriptor_t *desc)
   digest256map_remove(hs_cache_v3_dir, desc->key);
 }
 
-/* Store a given descriptor in our cache. */
+/** Store a given descriptor in our cache. */
 static void
 store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
 {
@@ -49,7 +64,7 @@ store_v3_desc_as_dir(hs_cache_dir_descriptor_t *desc)
   digest256map_set(hs_cache_v3_dir, desc->key, desc);
 }
 
-/* Query our cache and return the entry or NULL if not found. */
+/** Query our cache and return the entry or NULL if not found. */
 static hs_cache_dir_descriptor_t *
 lookup_v3_desc_as_dir(const uint8_t *key)
 {
@@ -60,7 +75,7 @@ lookup_v3_desc_as_dir(const uint8_t *key)
 #define cache_dir_desc_free(val) \
   FREE_AND_NULL(hs_cache_dir_descriptor_t, cache_dir_desc_free_, (val))
 
-/* Free a directory descriptor object. */
+/** Free a directory descriptor object. */
 static void
 cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
 {
@@ -72,7 +87,7 @@ cache_dir_desc_free_(hs_cache_dir_descriptor_t *desc)
   tor_free(desc);
 }
 
-/* Helper function: Use by the free all function using the digest256map
+/** Helper function: Use by the free all function using the digest256map
  * interface to cache entries. */
 static void
 cache_dir_desc_free_void(void *ptr)
@@ -80,7 +95,7 @@ cache_dir_desc_free_void(void *ptr)
   cache_dir_desc_free_(ptr);
 }
 
-/* Create a new directory cache descriptor object from a encoded descriptor.
+/** Create a new directory cache descriptor object from a encoded descriptor.
  * On success, return the heap-allocated cache object, otherwise return NULL if
  * we can't decode the descriptor. */
 static hs_cache_dir_descriptor_t *
@@ -110,7 +125,7 @@ cache_dir_desc_new(const char *desc)
   return NULL;
 }
 
-/* Return the size of a cache entry in bytes. */
+/** Return the size of a cache entry in bytes. */
 static size_t
 cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
 {
@@ -118,7 +133,7 @@ cache_get_dir_entry_size(const hs_cache_dir_descriptor_t *entry)
           + strlen(entry->encoded_desc));
 }
 
-/* Try to store a valid version 3 descriptor in the directory cache. Return 0
+/** Try to store a valid version 3 descriptor in the directory cache. Return 0
  * on success else a negative value is returned indicating that we have a
  * newer version in our cache. On error, caller is responsible to free the
  * given descriptor desc. */
@@ -168,7 +183,7 @@ cache_store_v3_as_dir(hs_cache_dir_descriptor_t *desc)
   return -1;
 }
 
-/* Using the query which is the base64 encoded blinded key of a version 3
+/** Using the query which is the base64 encoded blinded key of a version 3
  * descriptor, lookup in our directory cache the entry. If found, 1 is
  * returned and desc_out is populated with a newly allocated string being the
  * encoded descriptor. If not found, 0 is returned and desc_out is untouched.
@@ -203,7 +218,7 @@ cache_lookup_v3_as_dir(const char *query, const char **desc_out)
   return -1;
 }
 
-/* Clean the v3 cache by removing any entry that has expired using the
+/** Clean the v3 cache by removing any entry that has expired using the
  * <b>global_cutoff</b> value. If <b>global_cutoff</b> is 0, the cleaning
  * process will use the lifetime found in the plaintext data section. Return
  * the number of bytes cleaned. */
@@ -253,7 +268,7 @@ cache_clean_v3_as_dir(time_t now, time_t global_cutoff)
   return bytes_removed;
 }
 
-/* Given an encoded descriptor, store it in the directory cache depending on
+/** Given an encoded descriptor, store it in the directory cache depending on
  * which version it is. Return a negative value on error. On success, 0 is
  * returned. */
 int
@@ -288,7 +303,7 @@ hs_cache_store_as_dir(const char *desc)
   return -1;
 }
 
-/* Using the query, lookup in our directory cache the entry. If found, 1 is
+/** Using the query, lookup in our directory cache the entry. If found, 1 is
  * returned and desc_out is populated with a newly allocated string being
  * the encoded descriptor. If not found, 0 is returned and desc_out is
  * untouched. On error, a negative value is returned and desc_out is
@@ -313,7 +328,7 @@ hs_cache_lookup_as_dir(uint32_t version, const char *query,
   return found;
 }
 
-/* Clean all directory caches using the current time now. */
+/** Clean all directory caches using the current time now. */
 void
 hs_cache_clean_as_dir(time_t now)
 {
@@ -330,23 +345,38 @@ hs_cache_clean_as_dir(time_t now)
 
 /********************** Client-side HS cache ******************/
 
-/* Client-side HS descriptor cache. Map indexed by service identity key. */
+/** Client-side HS descriptor cache. Map indexed by service identity key. */
 static digest256map_t *hs_cache_v3_client;
 
-/* Client-side introduction point state cache. Map indexed by service public
+/** Client-side introduction point state cache. Map indexed by service public
  * identity key (onion address). It contains hs_cache_client_intro_state_t
  * objects all related to a specific service. */
 static digest256map_t *hs_cache_client_intro_state;
 
-/* Return the size of a client cache entry in bytes. */
+/** Return the size of a client cache entry in bytes. */
 static size_t
 cache_get_client_entry_size(const hs_cache_client_descriptor_t *entry)
 {
-  return sizeof(*entry) +
-         strlen(entry->encoded_desc) + hs_desc_obj_size(entry->desc);
+  size_t size = 0;
+
+  if (entry == NULL) {
+    goto end;
+  }
+  size += sizeof(*entry);
+
+  if (entry->encoded_desc) {
+    size += strlen(entry->encoded_desc);
+  }
+
+  if (entry_has_decrypted_descriptor(entry)) {
+    size += hs_desc_obj_size(entry->desc);
+  }
+
+ end:
+  return size;
 }
 
-/* Remove a given descriptor from our cache. */
+/** Remove a given descriptor from our cache. */
 static void
 remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
 {
@@ -356,7 +386,7 @@ remove_v3_desc_as_client(const hs_cache_client_descriptor_t *desc)
   rend_cache_decrement_allocation(cache_get_client_entry_size(desc));
 }
 
-/* Store a given descriptor in our cache. */
+/** Store a given descriptor in our cache. */
 static void
 store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
 {
@@ -366,7 +396,7 @@ store_v3_desc_as_client(hs_cache_client_descriptor_t *desc)
   rend_cache_increment_allocation(cache_get_client_entry_size(desc));
 }
 
-/* Query our cache and return the entry or NULL if not found or if expired. */
+/** Query our cache and return the entry or NULL if not found or if expired. */
 STATIC hs_cache_client_descriptor_t *
 lookup_v3_desc_as_client(const uint8_t *key)
 {
@@ -389,15 +419,17 @@ lookup_v3_desc_as_client(const uint8_t *key)
   return cached_desc;
 }
 
-/* Parse the encoded descriptor in <b>desc_str</b> using
- * <b>service_identity_pk<b> to decrypt it first.
+/** Parse the encoded descriptor in <b>desc_str</b> using
+ * <b>service_identity_pk</b> to decrypt it first.
  *
  * If everything goes well, allocate and return a new
  * hs_cache_client_descriptor_t object. In case of error, return NULL. */
 static hs_cache_client_descriptor_t *
 cache_client_desc_new(const char *desc_str,
-                      const ed25519_public_key_t *service_identity_pk)
+                      const ed25519_public_key_t *service_identity_pk,
+                      hs_desc_decode_status_t *decode_status_out)
 {
+  hs_desc_decode_status_t ret;
   hs_descriptor_t *desc = NULL;
   hs_cache_client_descriptor_t *client_desc = NULL;
 
@@ -405,10 +437,24 @@ cache_client_desc_new(const char *desc_str,
   tor_assert(service_identity_pk);
 
   /* Decode the descriptor we just fetched. */
-  if (hs_client_decode_descriptor(desc_str, service_identity_pk, &desc) < 0) {
+  ret = hs_client_decode_descriptor(desc_str, service_identity_pk, &desc);
+  if (ret != HS_DESC_DECODE_OK &&
+      ret != HS_DESC_DECODE_NEED_CLIENT_AUTH &&
+      ret != HS_DESC_DECODE_BAD_CLIENT_AUTH) {
+    /* In the case of a missing or bad client authorization, we'll keep the
+     * descriptor in the cache because those credentials can arrive later. */
     goto end;
   }
-  tor_assert(desc);
+  /* Make sure we do have a descriptor if decoding was successful. */
+  if (ret == HS_DESC_DECODE_OK) {
+    tor_assert(desc);
+  } else {
+    if (BUG(desc != NULL)) {
+      /* We are not suppose to have a descriptor if the decoding code is not
+       * indicating success. Just in case, bail early to recover. */
+      goto end;
+    }
+  }
 
   /* All is good: make a cache object for this descriptor */
   client_desc = tor_malloc_zero(sizeof(hs_cache_client_descriptor_t));
@@ -421,6 +467,9 @@ cache_client_desc_new(const char *desc_str,
   client_desc->encoded_desc = tor_strdup(desc_str);
 
  end:
+  if (decode_status_out) {
+    *decode_status_out = ret;
+  }
   return client_desc;
 }
 
@@ -449,7 +498,7 @@ cache_client_desc_free_void(void *ptr)
   cache_client_desc_free(desc);
 }
 
-/* Return a newly allocated and initialized hs_cache_intro_state_t object. */
+/** Return a newly allocated and initialized hs_cache_intro_state_t object. */
 static hs_cache_intro_state_t *
 cache_intro_state_new(void)
 {
@@ -461,21 +510,21 @@ cache_intro_state_new(void)
 #define cache_intro_state_free(val) \
   FREE_AND_NULL(hs_cache_intro_state_t, cache_intro_state_free_, (val))
 
-/* Free an hs_cache_intro_state_t object. */
+/** Free an hs_cache_intro_state_t object. */
 static void
 cache_intro_state_free_(hs_cache_intro_state_t *state)
 {
   tor_free(state);
 }
 
-/* Helper function: used by the free all function. */
+/** Helper function: used by the free all function. */
 static void
 cache_intro_state_free_void(void *state)
 {
   cache_intro_state_free_(state);
 }
 
-/* Return a newly allocated and initialized hs_cache_client_intro_state_t
+/** Return a newly allocated and initialized hs_cache_client_intro_state_t
  * object. */
 static hs_cache_client_intro_state_t *
 cache_client_intro_state_new(void)
@@ -489,7 +538,7 @@ cache_client_intro_state_new(void)
   FREE_AND_NULL(hs_cache_client_intro_state_t,          \
                 cache_client_intro_state_free_, (val))
 
-/* Free a cache_client_intro_state object. */
+/** Free a cache_client_intro_state object. */
 static void
 cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
 {
@@ -500,14 +549,14 @@ cache_client_intro_state_free_(hs_cache_client_intro_state_t *cache)
   tor_free(cache);
 }
 
-/* Helper function: used by the free all function. */
+/** Helper function: used by the free all function. */
 static void
 cache_client_intro_state_free_void(void *entry)
 {
   cache_client_intro_state_free_(entry);
 }
 
-/* For the given service identity key service_pk and an introduction
+/** For the given service identity key service_pk and an introduction
  * authentication key auth_key, lookup the intro state object. Return 1 if
  * found and put it in entry if not NULL. Return 0 if not found and entry is
  * untouched. */
@@ -542,7 +591,7 @@ cache_client_intro_state_lookup(const ed25519_public_key_t *service_pk,
   return 0;
 }
 
-/* Note the given failure in state. */
+/** Note the given failure in state. */
 static void
 cache_client_intro_state_note(hs_cache_intro_state_t *state,
                               rend_intro_point_failure_t failure)
@@ -564,7 +613,7 @@ cache_client_intro_state_note(hs_cache_intro_state_t *state,
   }
 }
 
-/* For the given service identity key service_pk and an introduction
+/** For the given service identity key service_pk and an introduction
  * authentication key auth_key, add an entry in the client intro state cache
  * If no entry exists for the service, it will create one. If state is non
  * NULL, it will point to the new intro state entry. */
@@ -598,7 +647,7 @@ cache_client_intro_state_add(const ed25519_public_key_t *service_pk,
   }
 }
 
-/* Remove every intro point state entry from cache that has been created
+/** Remove every intro point state entry from cache that has been created
  * before or at the cutoff. */
 static void
 cache_client_intro_state_clean(time_t cutoff,
@@ -615,7 +664,7 @@ cache_client_intro_state_clean(time_t cutoff,
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Return true iff no intro points are in this cache. */
+/** Return true iff no intro points are in this cache. */
 static int
 cache_client_intro_state_is_empty(const hs_cache_client_intro_state_t *cache)
 {
@@ -636,9 +685,25 @@ cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
   tor_assert(client_desc);
 
   /* Check if we already have a descriptor from this HS in cache. If we do,
-   * check if this descriptor is newer than the cached one */
+   * check if this descriptor is newer than the cached one only if we have a
+   * decoded descriptor. We do keep non-decoded descriptor that requires
+   * client authorization. */
   cache_entry = lookup_v3_desc_as_client(client_desc->key.pubkey);
   if (cache_entry != NULL) {
+    /* If the current or the new cache entry don't have a decrypted descriptor
+     * (missing client authorization), we always replace the current one with
+     * the new one. Reason is that we can't inspect the revision counter
+     * within the plaintext data so we blindly replace. */
+    if (!entry_has_decrypted_descriptor(cache_entry) ||
+        !entry_has_decrypted_descriptor(client_desc)) {
+      remove_v3_desc_as_client(cache_entry);
+      cache_client_desc_free(cache_entry);
+      goto store;
+    }
+
+    /* From this point on, we know that the decrypted descriptor is in the
+     * current entry and new object thus safe to access. */
+
     /* If we have an entry in our cache that has a revision counter greater
      * than the one we just fetched, discard the one we fetched. */
     if (cache_entry->desc->plaintext_data.revision_counter >
@@ -658,6 +723,7 @@ cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
     cache_client_desc_free(cache_entry);
   }
 
+ store:
   /* Store descriptor in cache */
   store_v3_desc_as_client(client_desc);
 
@@ -665,7 +731,7 @@ cache_store_as_client(hs_cache_client_descriptor_t *client_desc)
   return 0;
 }
 
-/* Return true iff the cached client descriptor at <b>cached_desc</b has
+/** Return true iff the cached client descriptor at <b>cached_desc</b> has
  * expired. */
 static int
 cached_client_descriptor_has_expired(time_t now,
@@ -690,7 +756,7 @@ cached_client_descriptor_has_expired(time_t now,
   return 0;
 }
 
-/* clean the client cache using now as the current time. Return the total size
+/** clean the client cache using now as the current time. Return the total size
  * of removed bytes from the cache. */
 static size_t
 cache_clean_v3_as_client(time_t now)
@@ -713,6 +779,15 @@ cache_clean_v3_as_client(time_t now)
     MAP_DEL_CURRENT(key);
     entry_size = cache_get_client_entry_size(entry);
     bytes_removed += entry_size;
+
+    /* We just removed an old descriptor. We need to close all intro circuits
+     * if the descriptor is decrypted so we don't have leftovers that can be
+     * selected while lacking a descriptor. Circuits are selected by intro
+     * authentication key thus we need the descriptor. We leave the rendezvous
+     * circuits opened because they could be in use. */
+    if (entry_has_decrypted_descriptor(entry)) {
+      hs_client_close_intro_circuits_from_desc(entry->desc);
+    }
     /* Entry is not in the cache anymore, destroy it. */
     cache_client_desc_free(entry);
     /* Update our OOM. We didn't use the remove() function because we are in
@@ -750,7 +825,9 @@ hs_cache_lookup_encoded_as_client(const ed25519_public_key_t *key)
 }
 
 /** Public API: Given the HS ed25519 identity public key in <b>key</b>, return
- *  its HS descriptor if it's stored in our cache, or NULL if not. */
+ *  its HS descriptor if it's stored in our cache, or NULL if not or if the
+ *  descriptor was never decrypted. The later can happen if we are waiting for
+ *  client authorization to be added. */
 const hs_descriptor_t *
 hs_cache_lookup_as_client(const ed25519_public_key_t *key)
 {
@@ -759,27 +836,41 @@ hs_cache_lookup_as_client(const ed25519_public_key_t *key)
   tor_assert(key);
 
   cached_desc = lookup_v3_desc_as_client(key->pubkey);
-  if (cached_desc) {
-    tor_assert(cached_desc->desc);
+  if (cached_desc && entry_has_decrypted_descriptor(cached_desc)) {
     return cached_desc->desc;
   }
 
   return NULL;
 }
 
-/** Public API: Given an encoded descriptor, store it in the client HS
- *  cache. Return -1 on error, 0 on success .*/
-int
+/** Public API: Given an encoded descriptor, store it in the client HS cache.
+ *  Return a decode status which changes how we handle the SOCKS connection
+ *  depending on its value:
+ *
+ *  HS_DESC_DECODE_OK: Returned on success. Descriptor was properly decoded
+ *                     and is now stored.
+ *
+ *  HS_DESC_DECODE_NEED_CLIENT_AUTH: Client authorization is needed but the
+ *                                   descriptor was still stored.
+ *
+ *  HS_DESC_DECODE_BAD_CLIENT_AUTH: Client authorization for this descriptor
+ *                                  was not usable but the descriptor was
+ *                                  still stored.
+ *
+ *  Any other codes means indicate where the error occured and the descriptor
+ *  was not stored. */
+hs_desc_decode_status_t
 hs_cache_store_as_client(const char *desc_str,
                          const ed25519_public_key_t *identity_pk)
 {
+  hs_desc_decode_status_t ret;
   hs_cache_client_descriptor_t *client_desc = NULL;
 
   tor_assert(desc_str);
   tor_assert(identity_pk);
 
   /* Create client cache descriptor object */
-  client_desc = cache_client_desc_new(desc_str, identity_pk);
+  client_desc = cache_client_desc_new(desc_str, identity_pk, &ret);
   if (!client_desc) {
     log_warn(LD_GENERAL, "HSDesc parsing failed!");
     log_debug(LD_GENERAL, "Failed to parse HSDesc: %s.", escaped(desc_str));
@@ -788,17 +879,54 @@ hs_cache_store_as_client(const char *desc_str,
 
   /* Push it to the cache */
   if (cache_store_as_client(client_desc) < 0) {
+    ret = HS_DESC_DECODE_GENERIC_ERROR;
     goto err;
   }
 
-  return 0;
+  return ret;
 
  err:
   cache_client_desc_free(client_desc);
-  return -1;
+  return ret;
 }
 
-/* Clean all client caches using the current time now. */
+/** Remove and free a client cache descriptor entry for the given onion
+ * service ed25519 public key. If the descriptor is decoded, the intro
+ * circuits are closed if any.
+ *
+ * This does nothing if no descriptor exists for the given key. */
+void
+hs_cache_remove_as_client(const ed25519_public_key_t *key)
+{
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(key);
+
+  cached_desc = lookup_v3_desc_as_client(key->pubkey);
+  if (!cached_desc) {
+    return;
+  }
+  /* If we have a decrypted/decoded descriptor, attempt to close its
+   * introduction circuit(s). We shouldn't have circuit(s) without a
+   * descriptor else it will lead to a failure. */
+  if (entry_has_decrypted_descriptor(cached_desc)) {
+    hs_client_close_intro_circuits_from_desc(cached_desc->desc);
+  }
+  /* Remove and free. */
+  remove_v3_desc_as_client(cached_desc);
+  cache_client_desc_free(cached_desc);
+
+  /* Logging. */
+  {
+    char key_b64[BASE64_DIGEST256_LEN + 1];
+    digest256_to_base64(key_b64, (const char *) key);
+    log_info(LD_REND, "Onion service v3 descriptor '%s' removed "
+                      "from client cache",
+             safe_str_client(key_b64));
+  }
+}
+
+/** Clean all client caches using the current time now. */
 void
 hs_cache_clean_as_client(time_t now)
 {
@@ -809,7 +937,7 @@ hs_cache_clean_as_client(time_t now)
   cache_clean_v3_as_client(now);
 }
 
-/* Purge the client descriptor cache. */
+/** Purge the client descriptor cache. */
 void
 hs_cache_purge_as_client(void)
 {
@@ -826,7 +954,7 @@ hs_cache_purge_as_client(void)
   log_info(LD_REND, "Hidden service client descriptor cache purged.");
 }
 
-/* For a given service identity public key and an introduction authentication
+/** For a given service identity public key and an introduction authentication
  * key, note the given failure in the client intro state cache. */
 void
 hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
@@ -848,7 +976,7 @@ hs_cache_client_intro_state_note(const ed25519_public_key_t *service_pk,
   cache_client_intro_state_note(entry, failure);
 }
 
-/* For a given service identity public key and an introduction authentication
+/** For a given service identity public key and an introduction authentication
  * key, return true iff it is present in the failure cache. */
 const hs_cache_intro_state_t *
 hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
@@ -859,7 +987,7 @@ hs_cache_client_intro_state_find(const ed25519_public_key_t *service_pk,
   return state;
 }
 
-/* Cleanup the client introduction state cache. */
+/** Cleanup the client introduction state cache. */
 void
 hs_cache_client_intro_state_clean(time_t now)
 {
@@ -879,7 +1007,7 @@ hs_cache_client_intro_state_clean(time_t now)
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Purge the client introduction state cache. */
+/** Purge the client introduction state cache. */
 void
 hs_cache_client_intro_state_purge(void)
 {
@@ -893,9 +1021,41 @@ hs_cache_client_intro_state_purge(void)
                     "cache purged.");
 }
 
+/* This is called when new client authorization was added to the global state.
+ * It attemps to decode the descriptor of the given service identity key.
+ *
+ * Return true if decoding was successful else false. */
+bool
+hs_cache_client_new_auth_parse(const ed25519_public_key_t *service_pk)
+{
+  bool ret = false;
+  hs_cache_client_descriptor_t *cached_desc = NULL;
+
+  tor_assert(service_pk);
+
+  if (!hs_cache_v3_client) {
+    return false;
+  }
+
+  cached_desc = lookup_v3_desc_as_client(service_pk->pubkey);
+  if (cached_desc == NULL || entry_has_decrypted_descriptor(cached_desc)) {
+    /* No entry for that service or the descriptor is already decoded. */
+    goto end;
+  }
+
+  /* Attempt a decode. If we are successful, inform the caller. */
+  if (hs_client_decode_descriptor(cached_desc->encoded_desc, service_pk,
+                                  &cached_desc->desc) == HS_DESC_DECODE_OK) {
+    ret = true;
+  }
+
+ end:
+  return ret;
+}
+
 /**************** Generics *********************************/
 
-/* Do a round of OOM cleanup on all directory caches. Return the amount of
+/** Do a round of OOM cleanup on all directory caches. Return the amount of
  * removed bytes. It is possible that the returned value is lower than
  * min_remove_bytes if the caches get emptied out so the caller should be
  * aware of this. */
@@ -949,7 +1109,7 @@ hs_cache_handle_oom(time_t now, size_t min_remove_bytes)
   return bytes_removed;
 }
 
-/* Return the maximum size of a v3 HS descriptor. */
+/** Return the maximum size of a v3 HS descriptor. */
 unsigned int
 hs_cache_get_max_descriptor_size(void)
 {
@@ -958,7 +1118,7 @@ hs_cache_get_max_descriptor_size(void)
                                             HS_DESC_MAX_LEN, 1, INT32_MAX);
 }
 
-/* Initialize the hidden service cache subsystem. */
+/** Initialize the hidden service cache subsystem. */
 void
 hs_cache_init(void)
 {
@@ -973,7 +1133,7 @@ hs_cache_init(void)
   hs_cache_client_intro_state = digest256map_new();
 }
 
-/* Cleanup the hidden service cache subsystem. */
+/** Cleanup the hidden service cache subsystem. */
 void
 hs_cache_free_all(void)
 {
diff --git a/src/feature/hs/hs_cache.h b/src/feature/hs/hs_cache.h
index 079d31d437..bb3c77f224 100644
--- a/src/feature/hs/hs_cache.h
+++ b/src/feature/hs/hs_cache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,47 +18,47 @@
 
 struct ed25519_public_key_t;
 
-/* This is the maximum time an introduction point state object can stay in the
+/** This is the maximum time an introduction point state object can stay in the
  * client cache in seconds (2 mins or 120 seconds). */
 #define HS_CACHE_CLIENT_INTRO_STATE_MAX_AGE (2 * 60)
 
-/* Introduction point state. */
+/** Introduction point state. */
 typedef struct hs_cache_intro_state_t {
-  /* When this entry was created and put in the cache. */
+  /** When this entry was created and put in the cache. */
   time_t created_ts;
 
-  /* Did it suffered a generic error? */
+  /** Did it suffered a generic error? */
   unsigned int error : 1;
 
-  /* Did it timed out? */
+  /** Did it timed out? */
   unsigned int timed_out : 1;
 
-  /* How many times we tried to reached it and it was unreachable. */
+  /** How many times we tried to reached it and it was unreachable. */
   uint32_t unreachable_count;
 } hs_cache_intro_state_t;
 
 typedef struct hs_cache_client_intro_state_t {
-  /* Contains hs_cache_intro_state_t object indexed by introduction point
+  /** Contains hs_cache_intro_state_t object indexed by introduction point
    * authentication key. */
   digest256map_t *intro_points;
 } hs_cache_client_intro_state_t;
 
-/* Descriptor representation on the directory side which is a subset of
+/** Descriptor representation on the directory side which is a subset of
  * information that the HSDir can decode and serve it. */
 typedef struct hs_cache_dir_descriptor_t {
-  /* This object is indexed using the blinded pubkey located in the plaintext
+  /** This object is indexed using the blinded pubkey located in the plaintext
    * data which is populated only once the descriptor has been successfully
    * decoded and validated. This simply points to that pubkey. */
   const uint8_t *key;
 
-  /* When does this entry has been created. Used to expire entries. */
+  /** When does this entry has been created. Used to expire entries. */
   time_t created_ts;
 
-  /* Descriptor plaintext information. Obviously, we can't decrypt the
+  /** Descriptor plaintext information. Obviously, we can't decrypt the
    * encrypted part of the descriptor. */
   hs_desc_plaintext_data_t *plaintext_data;
 
-  /* Encoded descriptor which is basically in text form. It's a NUL terminated
+  /** Encoded descriptor which is basically in text form. It's a NUL terminated
    * string thus safe to strlen(). */
   char *encoded_desc;
 } hs_cache_dir_descriptor_t;
@@ -83,8 +83,9 @@ const hs_descriptor_t *
 hs_cache_lookup_as_client(const struct ed25519_public_key_t *key);
 const char *
 hs_cache_lookup_encoded_as_client(const struct ed25519_public_key_t *key);
-int hs_cache_store_as_client(const char *desc_str,
-                             const struct ed25519_public_key_t *identity_pk);
+hs_desc_decode_status_t hs_cache_store_as_client(const char *desc_str,
+                           const struct ed25519_public_key_t *identity_pk);
+void hs_cache_remove_as_client(const struct ed25519_public_key_t *key);
 void hs_cache_clean_as_client(time_t now);
 void hs_cache_purge_as_client(void);
 
@@ -99,24 +100,28 @@ const hs_cache_intro_state_t *hs_cache_client_intro_state_find(
 void hs_cache_client_intro_state_clean(time_t now);
 void hs_cache_client_intro_state_purge(void);
 
+bool hs_cache_client_new_auth_parse(const ed25519_public_key_t *service_pk);
+
 #ifdef HS_CACHE_PRIVATE
 #include "lib/crypt_ops/crypto_ed25519.h"
 
 /** Represents a locally cached HS descriptor on a hidden service client. */
 typedef struct hs_cache_client_descriptor_t {
-  /* This object is indexed using the service identity public key */
+  /** This object is indexed using the service identity public key */
   struct ed25519_public_key_t key;
 
-  /* When will this entry expire? We expire cached client descriptors in the
+  /** When will this entry expire? We expire cached client descriptors in the
    * start of the next time period, since that's when clients need to start
    * using the next blinded key of the service. */
   time_t expiration_ts;
 
-  /* The cached descriptor, this object is the owner. It can't be NULL. A
-   * cache object without a valid descriptor is not possible. */
+  /** The cached decoded descriptor, this object is the owner. This can be
+   * NULL if the descriptor couldn't be decoded due to missing or bad client
+   * authorization. It can be decoded later from the encoded_desc object if
+   * the proper client authorization is given tor. */
   hs_descriptor_t *desc;
 
-  /* Encoded descriptor in string form. Can't be NULL. */
+  /** Encoded descriptor in string form. Can't be NULL. */
   char *encoded_desc;
 } hs_cache_client_descriptor_t;
 
diff --git a/src/feature/hs/hs_cell.c b/src/feature/hs/hs_cell.c
index 613ffe7260..fc9f4a2654 100644
--- a/src/feature/hs/hs_cell.c
+++ b/src/feature/hs/hs_cell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #include "feature/hs_common/replaycache.h"
 
 #include "feature/hs/hs_cell.h"
+#include "feature/hs/hs_ob.h"
 #include "core/crypto/hs_ntor.h"
 
 #include "core/or/origin_circuit_st.h"
@@ -24,7 +25,7 @@
 #include "trunnel/hs/cell_introduce1.h"
 #include "trunnel/hs/cell_rendezvous.h"
 
-/* Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
+/** Compute the MAC of an INTRODUCE cell in mac_out. The encoded_cell param is
  * the cell content up to the ENCRYPTED section of length encoded_cell_len.
  * The encrypted param is the start of the ENCRYPTED section of length
  * encrypted_len. The mac_key is the key needed for the computation of the MAC
@@ -67,14 +68,17 @@ compute_introduce_mac(const uint8_t *encoded_cell, size_t encoded_cell_len,
   memwipe(mac_msg, 0, sizeof(mac_msg));
 }
 
-/* From a set of keys, subcredential and the ENCRYPTED section of an
- * INTRODUCE2 cell, return a newly allocated intro cell keys structure.
- * Finally, the client public key is copied in client_pk. On error, return
- * NULL. */
+/**
+ * From a set of keys, a list of subcredentials, and the ENCRYPTED section of
+ * an INTRODUCE2 cell, return an array of newly allocated intro cell keys
+ * structures.  Finally, the client public key is copied in client_pk. On
+ * error, return NULL.
+ **/
 static hs_ntor_intro_cell_keys_t *
 get_introduce2_key_material(const ed25519_public_key_t *auth_key,
                             const curve25519_keypair_t *enc_key,
-                            const uint8_t *subcredential,
+                            size_t n_subcredentials,
+                            const hs_subcredential_t *subcredentials,
                             const uint8_t *encrypted_section,
                             curve25519_public_key_t *client_pk)
 {
@@ -82,17 +86,19 @@ get_introduce2_key_material(const ed25519_public_key_t *auth_key,
 
   tor_assert(auth_key);
   tor_assert(enc_key);
-  tor_assert(subcredential);
+  tor_assert(n_subcredentials > 0);
+  tor_assert(subcredentials);
   tor_assert(encrypted_section);
   tor_assert(client_pk);
 
-  keys = tor_malloc_zero(sizeof(*keys));
+  keys = tor_calloc(n_subcredentials, sizeof(hs_ntor_intro_cell_keys_t));
 
   /* First bytes of the ENCRYPTED section are the client public key. */
   memcpy(client_pk->public_key, encrypted_section, CURVE25519_PUBKEY_LEN);
 
-  if (hs_ntor_service_get_introduce1_keys(auth_key, enc_key, client_pk,
-                                          subcredential, keys) < 0) {
+  if (hs_ntor_service_get_introduce1_keys_multi(auth_key, enc_key, client_pk,
+                                                n_subcredentials,
+                                                subcredentials, keys) < 0) {
     /* Don't rely on the caller to wipe this on error. */
     memwipe(client_pk, 0, sizeof(curve25519_public_key_t));
     tor_free(keys);
@@ -101,7 +107,7 @@ get_introduce2_key_material(const ed25519_public_key_t *auth_key,
   return keys;
 }
 
-/* Using the given encryption key, decrypt the encrypted_section of length
+/** Using the given encryption key, decrypt the encrypted_section of length
  * encrypted_section_len of an INTRODUCE2 cell and return a newly allocated
  * buffer containing the decrypted data. On decryption failure, NULL is
  * returned. */
@@ -136,7 +142,7 @@ decrypt_introduce2(const uint8_t *enc_key, const uint8_t *encrypted_section,
   return decrypted;
 }
 
-/* Given a pointer to the decrypted data of the ENCRYPTED section of an
+/** Given a pointer to the decrypted data of the ENCRYPTED section of an
  * INTRODUCE2 cell of length decrypted_len, parse and validate the cell
  * content. Return a newly allocated cell structure or NULL on error. The
  * circuit and service object are only used for logging purposes. */
@@ -188,7 +194,7 @@ parse_introduce2_encrypted(const uint8_t *decrypted_data,
   return NULL;
 }
 
-/* Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
+/** Build a legacy ESTABLISH_INTRO cell with the given circuit nonce and RSA
  * encryption key. The encoded cell is put in cell_out that MUST at least be
  * of the size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on
  * success else a negative value and cell_out is untouched. */
@@ -210,7 +216,7 @@ build_legacy_establish_intro(const char *circ_nonce, crypto_pk_t *enc_key,
   return cell_len;
 }
 
-/* Parse an INTRODUCE2 cell from payload of size payload_len for the given
+/** Parse an INTRODUCE2 cell from payload of size payload_len for the given
  * service and circuit which are used only for logging purposes. The resulting
  * parsed cell is put in cell_ptr_out.
  *
@@ -249,7 +255,7 @@ parse_introduce2_cell(const hs_service_t *service,
   return -1;
 }
 
-/* Set the onion public key onion_pk in cell, the encrypted section of an
+/** Set the onion public key onion_pk in cell, the encrypted section of an
  * INTRODUCE1 cell. */
 static void
 introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
@@ -266,7 +272,7 @@ introduce1_set_encrypted_onion_key(trn_cell_introduce_encrypted_t *cell,
          trn_cell_introduce_encrypted_getlen_onion_key(cell));
 }
 
-/* Set the link specifiers in lspecs in cell, the encrypted section of an
+/** Set the link specifiers in lspecs in cell, the encrypted section of an
  * INTRODUCE1 cell. */
 static void
 introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
@@ -286,7 +292,7 @@ introduce1_set_encrypted_link_spec(trn_cell_introduce_encrypted_t *cell,
                     trn_cell_introduce_encrypted_add_nspecs(cell, ls));
 }
 
-/* Set padding in the enc_cell only if needed that is the total length of both
+/** Set padding in the enc_cell only if needed that is the total length of both
  * sections are below the mininum required for an INTRODUCE1 cell. */
 static void
 introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
@@ -306,7 +312,7 @@ introduce1_set_encrypted_padding(const trn_cell_introduce1_t *cell,
   }
 }
 
-/* Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
+/** Encrypt the ENCRYPTED payload and encode it in the cell using the enc_cell
  * and the INTRODUCE1 data.
  *
  * This can't fail but it is very important that the caller sets every field
@@ -394,7 +400,7 @@ introduce1_encrypt_and_encode(trn_cell_introduce1_t *cell,
   tor_free(encrypted);
 }
 
-/* Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
+/** Using the INTRODUCE1 data, setup the ENCRYPTED section in cell. This means
  * set it, encrypt it and encode it. */
 static void
 introduce1_set_encrypted(trn_cell_introduce1_t *cell,
@@ -435,7 +441,7 @@ introduce1_set_encrypted(trn_cell_introduce1_t *cell,
   trn_cell_introduce_encrypted_free(enc_cell);
 }
 
-/* Set the authentication key in the INTRODUCE1 cell from the given data. */
+/** Set the authentication key in the INTRODUCE1 cell from the given data. */
 static void
 introduce1_set_auth_key(trn_cell_introduce1_t *cell,
                         const hs_cell_introduce1_data_t *data)
@@ -451,7 +457,7 @@ introduce1_set_auth_key(trn_cell_introduce1_t *cell,
          data->auth_pk->pubkey, trn_cell_introduce1_getlen_auth_key(cell));
 }
 
-/* Set the legacy ID field in the INTRODUCE1 cell from the given data. */
+/** Set the legacy ID field in the INTRODUCE1 cell from the given data. */
 static void
 introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
                          const hs_cell_introduce1_data_t *data)
@@ -473,26 +479,150 @@ introduce1_set_legacy_id(trn_cell_introduce1_t *cell,
   }
 }
 
+/** Build and add to the given DoS cell extension the given parameter type and
+ * value. */
+static void
+build_establish_intro_dos_param(trn_cell_extension_dos_t *dos_ext,
+                                uint8_t param_type, uint64_t param_value)
+{
+  trn_cell_extension_dos_param_t *dos_param =
+    trn_cell_extension_dos_param_new();
+
+  /* Extra safety. We should never send an unknown parameter type. */
+  tor_assert(param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC ||
+             param_type == TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+
+  trn_cell_extension_dos_param_set_type(dos_param, param_type);
+  trn_cell_extension_dos_param_set_value(dos_param, param_value);
+  trn_cell_extension_dos_add_params(dos_ext, dos_param);
+
+  /* Not freeing the trunnel object because it is now owned by dos_ext. */
+}
+
+/** Build the DoS defense cell extension and put it in the given extensions
+ * object. Return 0 on success, -1 on failure.  (Right now, failure is only
+ * possible if there is a bug.) */
+static int
+build_establish_intro_dos_extension(const hs_service_config_t *service_config,
+                                    trn_cell_extension_t *extensions)
+{
+  ssize_t ret;
+  size_t dos_ext_encoded_len;
+  uint8_t *field_array;
+  trn_cell_extension_field_t *field = NULL;
+  trn_cell_extension_dos_t *dos_ext = NULL;
+
+  tor_assert(service_config);
+  tor_assert(extensions);
+
+  /* We are creating a cell extension field of the type DoS. */
+  field = trn_cell_extension_field_new();
+  trn_cell_extension_field_set_field_type(field,
+                                          TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+
+  /* Build DoS extension field. We will put in two parameters. */
+  dos_ext = trn_cell_extension_dos_new();
+  trn_cell_extension_dos_set_n_params(dos_ext, 2);
+
+  /* Build DoS parameter INTRO2 rate per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC,
+                                  service_config->intro_dos_rate_per_sec);
+  /* Build DoS parameter INTRO2 burst per second. */
+  build_establish_intro_dos_param(dos_ext,
+                                  TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC,
+                                  service_config->intro_dos_burst_per_sec);
+
+  /* Set the field with the encoded DoS extension. */
+  ret = trn_cell_extension_dos_encoded_len(dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  dos_ext_encoded_len = ret;
+  /* Set length field and the field array size length. */
+  trn_cell_extension_field_set_field_len(field, dos_ext_encoded_len);
+  trn_cell_extension_field_setlen_field(field, dos_ext_encoded_len);
+  /* Encode the DoS extension into the cell extension field. */
+  field_array = trn_cell_extension_field_getarray_field(field);
+  ret = trn_cell_extension_dos_encode(field_array,
+                 trn_cell_extension_field_getlen_field(field), dos_ext);
+  if (BUG(ret <= 0)) {
+    goto err;
+  }
+  tor_assert(ret == (ssize_t) dos_ext_encoded_len);
+
+  /* Finally, encode field into the cell extension. */
+  trn_cell_extension_add_fields(extensions, field);
+
+  /* We've just add an extension field to the cell extensions so increment the
+   * total number. */
+  trn_cell_extension_set_num(extensions,
+                             trn_cell_extension_get_num(extensions) + 1);
+
+  /* Cleanup. DoS extension has been encoded at this point. */
+  trn_cell_extension_dos_free(dos_ext);
+
+  return 0;
+
+ err:
+  trn_cell_extension_field_free(field);
+  trn_cell_extension_dos_free(dos_ext);
+  return -1;
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
-/* Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
+/** Allocate and build all the ESTABLISH_INTRO cell extension. The given
+ * extensions pointer is always set to a valid cell extension object. */
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip)
+{
+  int ret;
+  trn_cell_extension_t *extensions;
+
+  tor_assert(service_config);
+  tor_assert(ip);
+
+  extensions = trn_cell_extension_new();
+  trn_cell_extension_set_num(extensions, 0);
+
+  /* If the defense has been enabled service side (by the operator with a
+   * torrc option) and the intro point does support it. */
+  if (service_config->has_dos_defense_enabled &&
+      ip->support_intro2_dos_defense) {
+    /* This function takes care to increment the number of extensions. */
+    ret = build_establish_intro_dos_extension(service_config, extensions);
+    if (ret < 0) {
+      /* Return no extensions on error. */
+      goto end;
+    }
+  }
+
+ end:
+  return extensions;
+}
+
+/** Build an ESTABLISH_INTRO cell with the given circuit nonce and intro point
  * object. The encoded cell is put in cell_out that MUST at least be of the
  * size of RELAY_PAYLOAD_SIZE. Return the encoded cell length on success else
  * a negative value and cell_out is untouched. This function also supports
  * legacy cell creation. */
 ssize_t
 hs_cell_build_establish_intro(const char *circ_nonce,
+                              const hs_service_config_t *service_config,
                               const hs_service_intro_point_t *ip,
                               uint8_t *cell_out)
 {
   ssize_t cell_len = -1;
   uint16_t sig_len = ED25519_SIG_LEN;
-  trn_cell_extension_t *ext;
   trn_cell_establish_intro_t *cell = NULL;
+  trn_cell_extension_t *extensions;
 
   tor_assert(circ_nonce);
+  tor_assert(service_config);
   tor_assert(ip);
 
   /* Quickly handle the legacy IP. */
@@ -505,11 +635,12 @@ hs_cell_build_establish_intro(const char *circ_nonce,
     goto done;
   }
 
+  /* Build the extensions, if any. */
+  extensions = build_establish_intro_extensions(service_config, ip);
+
   /* Set extension data. None used here. */
-  ext = trn_cell_extension_new();
-  trn_cell_extension_set_num(ext, 0);
   cell = trn_cell_establish_intro_new();
-  trn_cell_establish_intro_set_extensions(cell, ext);
+  trn_cell_establish_intro_set_extensions(cell, extensions);
   /* Set signature size. Array is then allocated in the cell. We need to do
    * this early so we can use trunnel API to get the signature length. */
   trn_cell_establish_intro_set_sig_len(cell, sig_len);
@@ -600,7 +731,7 @@ hs_cell_build_establish_intro(const char *circ_nonce,
   return cell_len;
 }
 
-/* Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
+/** Parse the INTRO_ESTABLISHED cell in the payload of size payload_len. If we
  * are successful at parsing it, return the length of the parsed cell else a
  * negative value on error. */
 ssize_t
@@ -622,7 +753,75 @@ hs_cell_parse_intro_established(const uint8_t *payload, size_t payload_len)
   return ret;
 }
 
-/* Parse the INTRODUCE2 cell using data which contains everything we need to
+/** For the encrypted INTRO2 cell in <b>encrypted_section</b>, use the crypto
+ * material in <b>data</b> to compute the right ntor keys. Also validate the
+ * INTRO2 MAC to ensure that the keys are the right ones.
+ *
+ * Return NULL on failure to either produce the key material or on MAC
+ * validation. Else return a newly allocated intro keys object. */
+static hs_ntor_intro_cell_keys_t *
+get_introduce2_keys_and_verify_mac(hs_cell_introduce2_data_t *data,
+                                   const uint8_t *encrypted_section,
+                                   size_t encrypted_section_len)
+{
+  hs_ntor_intro_cell_keys_t *intro_keys = NULL;
+  hs_ntor_intro_cell_keys_t *intro_keys_result = NULL;
+
+  /* Build the key material out of the key material found in the cell. */
+  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
+                                           data->n_subcredentials,
+                                           data->subcredentials,
+                                           encrypted_section,
+                                           &data->client_pk);
+  if (intro_keys == NULL) {
+    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
+             "compute key material");
+    return NULL;
+  }
+
+  /* Make sure we are not about to underflow. */
+  if (BUG(encrypted_section_len < DIGEST256_LEN)) {
+    return NULL;
+  }
+
+  /* Validate MAC from the cell and our computed key material. The MAC field
+   * in the cell is at the end of the encrypted section. */
+  intro_keys_result = tor_malloc_zero(sizeof(*intro_keys_result));
+  for (unsigned i = 0; i < data->n_subcredentials; ++i) {
+    uint8_t mac[DIGEST256_LEN];
+
+    /* The MAC field is at the very end of the ENCRYPTED section. */
+    size_t mac_offset = encrypted_section_len - sizeof(mac);
+    /* Compute the MAC. Use the entire encoded payload with a length up to the
+     * ENCRYPTED section. */
+    compute_introduce_mac(data->payload,
+                          data->payload_len - encrypted_section_len,
+                          encrypted_section, encrypted_section_len,
+                          intro_keys[i].mac_key,
+                          sizeof(intro_keys[i].mac_key),
+                          mac, sizeof(mac));
+    /* Time-invariant conditional copy: if the MAC is what we expected, then
+     * set intro_keys_result to intro_keys[i]. Otherwise, don't: but don't
+     * leak which one it was! */
+    bool equal = tor_memeq(mac, encrypted_section + mac_offset, sizeof(mac));
+    memcpy_if_true_timei(equal, intro_keys_result, &intro_keys[i],
+                         sizeof(*intro_keys_result));
+  }
+
+  /* We no longer need intro_keys. */
+  memwipe(intro_keys, 0,
+          sizeof(hs_ntor_intro_cell_keys_t) * data->n_subcredentials);
+  tor_free(intro_keys);
+
+  if (safe_mem_is_zero(intro_keys_result, sizeof(*intro_keys_result))) {
+    log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell");
+    tor_free(intro_keys_result); /* sets intro_keys_result to NULL */
+  }
+
+  return intro_keys_result;
+}
+
+/** Parse the INTRODUCE2 cell using data which contains everything we need to
  * do so and contains the destination buffers of information we extract and
  * compute from the cell. Return 0 on success else a negative value. The
  * service and circ are only used for logging purposes. */
@@ -670,47 +869,29 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
   /* Check our replay cache for this introduction point. */
   if (replaycache_add_test_and_elapsed(data->replay_cache, encrypted_section,
                                        encrypted_section_len, &elapsed)) {
-    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the"
+    log_warn(LD_REND, "Possible replay detected! An INTRODUCE2 cell with the "
                       "same ENCRYPTED section was seen %ld seconds ago. "
                       "Dropping cell.", (long int) elapsed);
     goto done;
   }
 
-  /* Build the key material out of the key material found in the cell. */
-  intro_keys = get_introduce2_key_material(data->auth_pk, data->enc_kp,
-                                           data->subcredential,
-                                           encrypted_section,
-                                           &data->client_pk);
-  if (intro_keys == NULL) {
-    log_info(LD_REND, "Invalid INTRODUCE2 encrypted data. Unable to "
-                      "compute key material on circuit %u for service %s",
-             TO_CIRCUIT(circ)->n_circ_id,
+  /* First bytes of the ENCRYPTED section are the client public key (they are
+   * guaranteed to exist because of the length check above). We are gonna use
+   * the client public key to compute the ntor keys and decrypt the payload:
+   */
+  memcpy(&data->client_pk.public_key, encrypted_section,
+         CURVE25519_PUBKEY_LEN);
+
+  /* Get the right INTRODUCE2 ntor keys and verify the cell MAC */
+  intro_keys = get_introduce2_keys_and_verify_mac(data, encrypted_section,
+                                                  encrypted_section_len);
+  if (!intro_keys) {
+    log_warn(LD_REND, "Could not get valid INTRO2 keys on circuit %u "
+             "for service %s", TO_CIRCUIT(circ)->n_circ_id,
              safe_str_client(service->onion_address));
     goto done;
   }
 
-  /* Validate MAC from the cell and our computed key material. The MAC field
-   * in the cell is at the end of the encrypted section. */
-  {
-    uint8_t mac[DIGEST256_LEN];
-    /* The MAC field is at the very end of the ENCRYPTED section. */
-    size_t mac_offset = encrypted_section_len - sizeof(mac);
-    /* Compute the MAC. Use the entire encoded payload with a length up to the
-     * ENCRYPTED section. */
-    compute_introduce_mac(data->payload,
-                          data->payload_len - encrypted_section_len,
-                          encrypted_section, encrypted_section_len,
-                          intro_keys->mac_key, sizeof(intro_keys->mac_key),
-                          mac, sizeof(mac));
-    if (tor_memcmp(mac, encrypted_section + mac_offset, sizeof(mac))) {
-      log_info(LD_REND, "Invalid MAC validation for INTRODUCE2 cell on "
-                        "circuit %u for service %s",
-               TO_CIRCUIT(circ)->n_circ_id,
-               safe_str_client(service->onion_address));
-      goto done;
-    }
-  }
-
   {
     /* The ENCRYPTED_DATA section starts just after the CLIENT_PK. */
     const uint8_t *encrypted_data =
@@ -758,7 +939,14 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
        idx < trn_cell_introduce_encrypted_get_nspec(enc_cell); idx++) {
     link_specifier_t *lspec =
       trn_cell_introduce_encrypted_get_nspecs(enc_cell, idx);
-    smartlist_add(data->link_specifiers, hs_link_specifier_dup(lspec));
+    if (BUG(!lspec)) {
+      goto done;
+    }
+    link_specifier_t *lspec_dup = link_specifier_dup(lspec);
+    if (BUG(!lspec_dup)) {
+      goto done;
+    }
+    smartlist_add(data->link_specifiers, lspec_dup);
   }
 
   /* Success. */
@@ -776,7 +964,7 @@ hs_cell_parse_introduce2(hs_cell_introduce2_data_t *data,
   return ret;
 }
 
-/* Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
+/** Build a RENDEZVOUS1 cell with the given rendezvous cookie and handshake
  * info. The encoded cell is put in cell_out and the length of the data is
  * returned. This can't fail. */
 ssize_t
@@ -810,7 +998,7 @@ hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
   return cell_len;
 }
 
-/* Build an INTRODUCE1 cell from the given data. The encoded cell is put in
+/** Build an INTRODUCE1 cell from the given data. The encoded cell is put in
  * cell_out which must be of at least size RELAY_PAYLOAD_SIZE. On success, the
  * encoded length is returned else a negative value and the content of
  * cell_out should be ignored. */
@@ -851,7 +1039,7 @@ hs_cell_build_introduce1(const hs_cell_introduce1_data_t *data,
   return cell_len;
 }
 
-/* Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
+/** Build an ESTABLISH_RENDEZVOUS cell from the given rendezvous_cookie. The
  * encoded cell is put in cell_out which must be of at least
  * RELAY_PAYLOAD_SIZE. On success, the encoded length is returned and the
  * caller should clear up the content of the cell.
@@ -868,7 +1056,7 @@ hs_cell_build_establish_rendezvous(const uint8_t *rendezvous_cookie,
   return HS_REND_COOKIE_LEN;
 }
 
-/* Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
+/** Handle an INTRODUCE_ACK cell encoded in payload of length payload_len.
  * Return the status code on success else a negative value if the cell as not
  * decodable. */
 int
@@ -903,7 +1091,7 @@ hs_cell_parse_introduce_ack(const uint8_t *payload, size_t payload_len)
   return ret;
 }
 
-/* Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
+/** Handle a RENDEZVOUS2 cell encoded in payload of length payload_len. On
  * success, handshake_info contains the data in the HANDSHAKE_INFO field, and
  * 0 is returned. On error, a negative value is returned. */
 int
@@ -935,7 +1123,7 @@ hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
   return ret;
 }
 
-/* Clear the given INTRODUCE1 data structure data. */
+/** Clear the given INTRODUCE1 data structure data. */
 void
 hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
 {
@@ -949,4 +1137,3 @@ hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data)
   /* The data object has no ownership of any members. */
   memwipe(data, 0, sizeof(hs_cell_introduce1_data_t));
 }
-
diff --git a/src/feature/hs/hs_cell.h b/src/feature/hs/hs_cell.h
index 9569de535e..2b28c44c50 100644
--- a/src/feature/hs/hs_cell.h
+++ b/src/feature/hs/hs_cell.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,35 +12,37 @@
 #include "core/or/or.h"
 #include "feature/hs/hs_service.h"
 
-/* An INTRODUCE1 cell requires at least this amount of bytes (see section
+/** An INTRODUCE1 cell requires at least this amount of bytes (see section
  * 3.2.2 of the specification). Below this value, the cell must be padded. */
 #define HS_CELL_INTRODUCE1_MIN_SIZE 246
 
-/* This data structure contains data that we need to build an INTRODUCE1 cell
+struct hs_subcredential_t;
+
+/** This data structure contains data that we need to build an INTRODUCE1 cell
  * used by the INTRODUCE1 build function. */
 typedef struct hs_cell_introduce1_data_t {
-  /* Is this a legacy introduction point? */
+  /** Is this a legacy introduction point? */
   unsigned int is_legacy : 1;
-  /* (Legacy only) The encryption key for a legacy intro point. Only set if
+  /** (Legacy only) The encryption key for a legacy intro point. Only set if
    * is_legacy is true. */
   const crypto_pk_t *legacy_key;
-  /* Introduction point authentication public key. */
+  /** Introduction point authentication public key. */
   const ed25519_public_key_t *auth_pk;
-  /* Introduction point encryption public key. */
+  /** Introduction point encryption public key. */
   const curve25519_public_key_t *enc_pk;
-  /* Subcredentials of the service. */
-  const uint8_t *subcredential;
-  /* Onion public key for the ntor handshake. */
+  /** Subcredentials of the service. */
+  const struct hs_subcredential_t *subcredential;
+  /** Onion public key for the ntor handshake. */
   const curve25519_public_key_t *onion_pk;
-  /* Rendezvous cookie. */
+  /** Rendezvous cookie. */
   const uint8_t *rendezvous_cookie;
-  /* Public key put before the encrypted data (CLIENT_PK). */
+  /** Public key put before the encrypted data (CLIENT_PK). */
   const curve25519_keypair_t *client_kp;
-  /* Rendezvous point link specifiers. */
+  /** Rendezvous point link specifiers. */
   smartlist_t *link_specifiers;
 } hs_cell_introduce1_data_t;
 
-/* This data structure contains data that we need to parse an INTRODUCE2 cell
+/** This data structure contains data that we need to parse an INTRODUCE2 cell
  * which is used by the INTRODUCE2 cell parsing function. On a successful
  * parsing, the onion_pk and rendezvous_cookie will be populated with the
  * computed key material from the cell data. This structure is only used during
@@ -48,37 +50,43 @@ typedef struct hs_cell_introduce1_data_t {
 typedef struct hs_cell_introduce2_data_t {
   /*** Immutable Section: Set on structure init. ***/
 
-  /* Introduction point authentication public key. Pointer owned by the
+  /** Introduction point authentication public key. Pointer owned by the
      introduction point object through which we received the INTRO2 cell. */
   const ed25519_public_key_t *auth_pk;
-  /* Introduction point encryption keypair for the ntor handshake. Pointer
+  /** Introduction point encryption keypair for the ntor handshake. Pointer
      owned by the introduction point object through which we received the
      INTRO2 cell*/
   const curve25519_keypair_t *enc_kp;
-  /* Subcredentials of the service. Pointer owned by the descriptor that owns
-     the introduction point through which we received the INTRO2 cell. */
-  const uint8_t *subcredential;
-  /* Payload of the received encoded cell. */
+  /**
+   * Length of the subcredentials array below.
+   **/
+  size_t n_subcredentials;
+  /** Array of <b>n_subcredentials</b> subcredentials for the service. Pointer
+   * owned by the descriptor that owns the introduction point through which we
+   * received the INTRO2 cell. */
+  const struct hs_subcredential_t *subcredentials;
+  /** Payload of the received encoded cell. */
   const uint8_t *payload;
-  /* Size of the payload of the received encoded cell. */
+  /** Size of the payload of the received encoded cell. */
   size_t payload_len;
 
   /*** Mutable Section: Set upon parsing INTRODUCE2 cell. ***/
 
-  /* Onion public key computed using the INTRODUCE2 encrypted section. */
+  /** Onion public key computed using the INTRODUCE2 encrypted section. */
   curve25519_public_key_t onion_pk;
-  /* Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
+  /** Rendezvous cookie taken from the INTRODUCE2 encrypted section. */
   uint8_t rendezvous_cookie[REND_COOKIE_LEN];
-  /* Client public key from the INTRODUCE2 encrypted section. */
+  /** Client public key from the INTRODUCE2 encrypted section. */
   curve25519_public_key_t client_pk;
-  /* Link specifiers of the rendezvous point. Contains link_specifier_t. */
+  /** Link specifiers of the rendezvous point. Contains link_specifier_t. */
   smartlist_t *link_specifiers;
-  /* Replay cache of the introduction point. */
+  /** Replay cache of the introduction point. */
   replaycache_t *replay_cache;
 } hs_cell_introduce2_data_t;
 
 /* Build cell API. */
 ssize_t hs_cell_build_establish_intro(const char *circ_nonce,
+                                      const hs_service_config_t *config,
                                       const hs_service_intro_point_t *ip,
                                       uint8_t *cell_out);
 ssize_t hs_cell_build_rendezvous1(const uint8_t *rendezvous_cookie,
@@ -105,5 +113,14 @@ int hs_cell_parse_rendezvous2(const uint8_t *payload, size_t payload_len,
 /* Util API. */
 void hs_cell_introduce1_data_clear(hs_cell_introduce1_data_t *data);
 
-#endif /* !defined(TOR_HS_CELL_H) */
+#ifdef TOR_UNIT_TESTS
+
+#include "trunnel/hs/cell_common.h"
 
+STATIC trn_cell_extension_t *
+build_establish_intro_extensions(const hs_service_config_t *service_config,
+                                 const hs_service_intro_point_t *ip);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_HS_CELL_H) */
diff --git a/src/feature/hs/hs_circuit.c b/src/feature/hs/hs_circuit.c
index 8acfcbd65b..447f664f81 100644
--- a/src/feature/hs/hs_circuit.c
+++ b/src/feature/hs/hs_circuit.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,15 +15,19 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuit.h"
+#include "feature/hs/hs_ob.h"
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_service.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/rend/rendservice.h"
+#include "feature/rend/rendclient.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
@@ -39,7 +43,7 @@
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
 
-/* A circuit is about to become an e2e rendezvous circuit. Check
+/** A circuit is about to become an e2e rendezvous circuit. Check
  * <b>circ_purpose</b> and ensure that it's properly set. Return true iff
  * circuit purpose is properly set, otherwise return false. */
 static int
@@ -66,7 +70,7 @@ circuit_purpose_is_correct_for_rend(unsigned int circ_purpose,
   return 1;
 }
 
-/* Create and return a crypt path for the final hop of a v3 prop224 rendezvous
+/** Create and return a crypt path for the final hop of a v3 prop224 rendezvous
  * circuit. Initialize the crypt path crypto using the output material from the
  * ntor key exchange at <b>ntor_key_seed</b>.
  *
@@ -89,7 +93,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
   cpath = tor_malloc_zero(sizeof(crypt_path_t));
   cpath->magic = CRYPT_PATH_MAGIC;
 
-  if (circuit_init_cpath_crypto(cpath, (char*)keys, sizeof(keys),
+  if (cpath_init_circuit_crypto(cpath, (char*)keys, sizeof(keys),
                                 is_service_side, 1) < 0) {
     tor_free(cpath);
     goto err;
@@ -100,7 +104,7 @@ create_rend_cpath(const uint8_t *ntor_key_seed, size_t seed_len,
   return cpath;
 }
 
-/* We are a v2 legacy HS client: Create and return a crypt path for the hidden
+/** We are a v2 legacy HS client: Create and return a crypt path for the hidden
  * service on the other side of the rendezvous circuit <b>circ</b>. Initialize
  * the crypt path crypto using the body of the RENDEZVOUS1 cell at
  * <b>rend_cell_body</b> (which must be at least DH1024_KEY_LEN+DIGEST_LEN
@@ -126,7 +130,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
     goto err;
   }
   /* ... and set up cpath. */
-  if (circuit_init_cpath_crypto(hop,
+  if (cpath_init_circuit_crypto(hop,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 0, 0) < 0)
     goto err;
@@ -151,7 +155,7 @@ create_rend_cpath_legacy(origin_circuit_t *circ, const uint8_t *rend_cell_body)
   return hop;
 }
 
-/* Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
+/** Append the final <b>hop</b> to the cpath of the rend <b>circ</b>, and mark
  * <b>circ</b> ready for use to transfer HS relay cells. */
 static void
 finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
@@ -177,7 +181,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
   circ->hs_circ_has_timed_out = 0;
 
   /* Append the hop to the cpath of this circuit */
-  onion_append_to_cpath(&circ->cpath, hop);
+  cpath_extend_linked_list(&circ->cpath, hop);
 
   /* In legacy code, 'pending_final_cpath' points to the final hop we just
    * appended to the cpath. We set the original pointer to NULL so that we
@@ -192,7 +196,7 @@ finalize_rend_circuit(origin_circuit_t *circ, crypt_path_t *hop,
   }
 }
 
-/* For a given circuit and a service introduction point object, register the
+/** For a given circuit and a service introduction point object, register the
  * intro circuit to the circuitmap. This supports legacy intro point. */
 static void
 register_intro_circ(const hs_service_intro_point_t *ip,
@@ -210,7 +214,7 @@ register_intro_circ(const hs_service_intro_point_t *ip,
   }
 }
 
-/* Return the number of opened introduction circuit for the given circuit that
+/** Return the number of opened introduction circuit for the given circuit that
  * is matching its identity key. */
 static unsigned int
 count_opened_desc_intro_point_circuits(const hs_service_t *service,
@@ -242,7 +246,7 @@ count_opened_desc_intro_point_circuits(const hs_service_t *service,
   return count;
 }
 
-/* From a given service, rendezvous cookie and handshake info, create a
+/** From a given service, rendezvous cookie and handshake info, create a
  * rendezvous point circuit identifier. This can't fail. */
 STATIC hs_ident_circuit_t *
 create_rp_circuit_identifier(const hs_service_t *service,
@@ -258,8 +262,7 @@ create_rp_circuit_identifier(const hs_service_t *service,
   tor_assert(server_pk);
   tor_assert(keys);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_RENDEZVOUS);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Copy the RENDEZVOUS_COOKIE which is the unique identifier. */
   memcpy(ident->rendezvous_cookie, rendezvous_cookie,
          sizeof(ident->rendezvous_cookie));
@@ -282,7 +285,7 @@ create_rp_circuit_identifier(const hs_service_t *service,
   return ident;
 }
 
-/* From a given service and service intro point, create an introduction point
+/** From a given service and service intro point, create an introduction point
  * circuit identifier. This can't fail. */
 static hs_ident_circuit_t *
 create_intro_circuit_identifier(const hs_service_t *service,
@@ -293,14 +296,13 @@ create_intro_circuit_identifier(const hs_service_t *service,
   tor_assert(service);
   tor_assert(ip);
 
-  ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                               HS_IDENT_CIRCUIT_INTRO);
+  ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&ident->intro_auth_pk, &ip->auth_key_kp.pubkey);
 
   return ident;
 }
 
-/* For a given introduction point and an introduction circuit, send the
+/** For a given introduction point and an introduction circuit, send the
  * ESTABLISH_INTRO cell. The service object is used for logging. This can fail
  * and if so, the circuit is closed and the intro point object is flagged
  * that the circuit is not established anymore which is important for the
@@ -318,7 +320,7 @@ send_establish_intro(const hs_service_t *service,
 
   /* Encode establish intro cell. */
   cell_len = hs_cell_build_establish_intro(circ->cpath->prev->rend_circ_nonce,
-                                           ip, payload);
+                                           &service->config, ip, payload);
   if (cell_len < 0) {
     log_warn(LD_REND, "Unable to encode ESTABLISH_INTRO cell for service %s "
                       "on circuit %u. Closing circuit.",
@@ -350,7 +352,7 @@ send_establish_intro(const hs_service_t *service,
   memwipe(payload, 0, sizeof(payload));
 }
 
-/* Return a string constant describing the anonymity of service. */
+/** Return a string constant describing the anonymity of service. */
 static const char *
 get_service_anonymity_string(const hs_service_t *service)
 {
@@ -361,15 +363,15 @@ get_service_anonymity_string(const hs_service_t *service)
   }
 }
 
-/* For a given service, the ntor onion key and a rendezvous cookie, launch a
+/** For a given service, the ntor onion key and a rendezvous cookie, launch a
  * circuit to the rendezvous point specified by the link specifiers. On
  * success, a circuit identifier is attached to the circuit with the needed
  * data. This function will try to open a circuit for a maximum value of
  * MAX_REND_FAILURES then it will give up. */
-static void
-launch_rendezvous_point_circuit(const hs_service_t *service,
-                                const hs_service_intro_point_t *ip,
-                                const hs_cell_introduce2_data_t *data)
+MOCK_IMPL(STATIC void,
+launch_rendezvous_point_circuit,(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                 const hs_cell_introduce2_data_t *data))
 {
   int circ_needs_uptime;
   time_t now = time(NULL);
@@ -388,10 +390,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service,
                                         &data->onion_pk,
                                         service->config.is_single_onion);
   if (info == NULL) {
-    /* We are done here, we can't extend to the rendezvous point.
-     * If you're running an IPv6-only v3 single onion service on 0.3.2 or with
-     * 0.3.2 clients, and somehow disable the option check, it will fail here.
-     */
+    /* We are done here, we can't extend to the rendezvous point. */
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Not enough info to open a circuit to a rendezvous point for "
            "%s service %s.",
@@ -473,7 +472,7 @@ launch_rendezvous_point_circuit(const hs_service_t *service,
   extend_info_free(info);
 }
 
-/* Return true iff the given service rendezvous circuit circ is allowed for a
+/** Return true iff the given service rendezvous circuit circ is allowed for a
  * relaunch to the rendezvous point. */
 static int
 can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
@@ -520,7 +519,7 @@ can_relaunch_service_rendezvous_point(const origin_circuit_t *circ)
   return 0;
 }
 
-/* Retry the rendezvous point of circ by launching a new circuit to it. */
+/** Retry the rendezvous point of circ by launching a new circuit to it. */
 static void
 retry_service_rendezvous_point(const origin_circuit_t *circ)
 {
@@ -569,82 +568,7 @@ retry_service_rendezvous_point(const origin_circuit_t *circ)
   return;
 }
 
-/* Add all possible link specifiers in node to lspecs:
- *  - legacy ID is mandatory thus MUST be present in node;
- *  - include ed25519 link specifier if present in the node, and the node
- *    supports ed25519 link authentication, even if its link versions are not
- *    compatible with us;
- *  - include IPv4 link specifier, if the primary address is not IPv4, log a
- *    BUG() warning, and return an empty smartlist;
- *  - include IPv6 link specifier if present in the node. */
-static void
-get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
-{
-  link_specifier_t *ls;
-  tor_addr_port_t ap;
-
-  tor_assert(node);
-  tor_assert(lspecs);
-
-  /* Get the relay's IPv4 address. */
-  node_get_prim_orport(node, &ap);
-
-  /* We expect the node's primary address to be a valid IPv4 address.
-   * This conforms to the protocol, which requires either an IPv4 or IPv6
-   * address (or both). */
-  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
-      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
-    return;
-  }
-
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_IPV4);
-  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
-  link_specifier_set_un_ipv4_port(ls, ap.port);
-  /* Four bytes IPv4 and two bytes port. */
-  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
-                            sizeof(ap.port));
-  smartlist_add(lspecs, ls);
-
-  /* Legacy ID is mandatory and will always be present in node. */
-  ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
-  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
-         link_specifier_getlen_un_legacy_id(ls));
-  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
-  smartlist_add(lspecs, ls);
-
-  /* ed25519 ID is only included if the node has it, and the node declares a
-     protocol version that supports ed25519 link authentication, even if that
-     link version is not compatible with us. (We are sending the ed25519 key
-     to another tor, which may support different link versions.) */
-  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
-      node_supports_ed25519_link_authentication(node, 0)) {
-    ls = link_specifier_new();
-    link_specifier_set_ls_type(ls, LS_ED25519_ID);
-    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
-           link_specifier_getlen_un_ed25519_id(ls));
-    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
-    smartlist_add(lspecs, ls);
-  }
-
-  /* Check for IPv6. If so, include it as well. */
-  if (node_has_ipv6_orport(node)) {
-    ls = link_specifier_new();
-    node_get_pref_ipv6_orport(node, &ap);
-    link_specifier_set_ls_type(ls, LS_IPV6);
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
-    smartlist_add(lspecs, ls);
-  }
-}
-
-/* Using the given descriptor intro point ip, the node of the
+/** Using the given descriptor intro point ip, the node of the
  * rendezvous point rp_node and the service's subcredential, populate the
  * already allocated intro1_data object with the needed key material and link
  * specifiers.
@@ -655,7 +579,7 @@ get_lspecs_from_node(const node_t *node, smartlist_t *lspecs)
 static int
 setup_introduce1_data(const hs_desc_intro_point_t *ip,
                       const node_t *rp_node,
-                      const uint8_t *subcredential,
+                      const hs_subcredential_t *subcredential,
                       hs_cell_introduce1_data_t *intro1_data)
 {
   int ret = -1;
@@ -666,10 +590,9 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip,
   tor_assert(subcredential);
   tor_assert(intro1_data);
 
-  /* Build the link specifiers from the extend information of the rendezvous
-   * circuit that we've picked previously. */
-  rp_lspecs = smartlist_new();
-  get_lspecs_from_node(rp_node, rp_lspecs);
+  /* Build the link specifiers from the node at the end of the rendezvous
+   * circuit that we opened for this introduction. */
+  rp_lspecs = node_get_link_specifier_smartlist(rp_node, 0);
   if (smartlist_len(rp_lspecs) == 0) {
     /* We can't rendezvous without link specifiers. */
     smartlist_free(rp_lspecs);
@@ -698,11 +621,41 @@ setup_introduce1_data(const hs_desc_intro_point_t *ip,
   return ret;
 }
 
+/** Helper: cleanup function for client circuit. This is for every HS version.
+ * It is called from hs_circ_cleanup_on_close() entry point. */
+static void
+cleanup_on_close_client_circ(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circuit_is_hs_v3(circ)) {
+    hs_client_circuit_cleanup_on_close(circ);
+  }
+  /* It is possible the circuit has an HS purpose but no identifier (rend_data
+   * or hs_ident). Thus possible that this passes through. */
+}
+
+/** Helper: cleanup function for client circuit. This is for every HS version.
+ * It is called from hs_circ_cleanup_on_free() entry point. */
+static void
+cleanup_on_free_client_circ(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  if (circuit_is_hs_v2(circ)) {
+    rend_client_circuit_cleanup_on_free(circ);
+  } else if (circuit_is_hs_v3(circ)) {
+    hs_client_circuit_cleanup_on_free(circ);
+  }
+  /* It is possible the circuit has an HS purpose but no identifier (rend_data
+   * or hs_ident). Thus possible that this passes through. */
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
-/* Return an introduction point circuit matching the given intro point object.
+/** Return an introduction point circuit matching the given intro point object.
  * NULL is returned is no such circuit can be found. */
 origin_circuit_t *
 hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
@@ -717,7 +670,29 @@ hs_circ_service_get_intro_circ(const hs_service_intro_point_t *ip)
   }
 }
 
-/* Called when we fail building a rendezvous circuit at some point other than
+/** Return an introduction point established circuit matching the given intro
+ * point object. The circuit purpose has to be CIRCUIT_PURPOSE_S_INTRO. NULL
+ * is returned is no such circuit can be found. */
+origin_circuit_t *
+hs_circ_service_get_established_intro_circ(const hs_service_intro_point_t *ip)
+{
+  origin_circuit_t *circ;
+
+  tor_assert(ip);
+
+  if (ip->base.is_only_legacy) {
+    circ = hs_circuitmap_get_intro_circ_v2_service_side(ip->legacy_key_digest);
+  } else {
+    circ = hs_circuitmap_get_intro_circ_v3_service_side(
+                                        &ip->auth_key_kp.pubkey);
+  }
+
+  /* Only return circuit if it is established. */
+  return (circ && TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_S_INTRO) ?
+          circ : NULL;
+}
+
+/** Called when we fail building a rendezvous circuit at some point other than
  * the last hop: launches a new circuit to the same rendezvous point. This
  * supports legacy service.
  *
@@ -757,7 +732,7 @@ hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ)
   return;
 }
 
-/* For a given service and a service intro point, launch a circuit to the
+/** For a given service and a service intro point, launch a circuit to the
  * extend info ei. If the service is a single onion, and direct_conn is true,
  * a one-hop circuit will be requested.
  *
@@ -818,7 +793,7 @@ hs_circ_launch_intro_point(hs_service_t *service,
   return ret;
 }
 
-/* Called when a service introduction point circuit is done building. Given
+/** Called when a service introduction point circuit is done building. Given
  * the service and intro point object, this function will send the
  * ESTABLISH_INTRO cell on the circuit. Return 0 on success. Return 1 if the
  * circuit has been repurposed to General because we already have too many
@@ -887,7 +862,7 @@ hs_circ_service_intro_has_opened(hs_service_t *service,
   return ret;
 }
 
-/* Called when a service rendezvous point circuit is done building. Given the
+/** Called when a service rendezvous point circuit is done building. Given the
  * service and the circuit, this function will send a RENDEZVOUS1 cell on the
  * circuit using the information in the circuit identifier. If the cell can't
  * be sent, the circuit is closed. */
@@ -953,7 +928,7 @@ hs_circ_service_rp_has_opened(const hs_service_t *service,
   memwipe(payload, 0, sizeof(payload));
 }
 
-/* Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
+/** Circ has been expecting an INTRO_ESTABLISHED cell that just arrived. Handle
  * the INTRO_ESTABLISHED cell payload of length payload_len arriving on the
  * given introduction circuit circ. The service is only used for logging
  * purposes. Return 0 on success else a negative value. */
@@ -998,7 +973,43 @@ hs_circ_handle_intro_established(const hs_service_t *service,
   return ret;
 }
 
-/* We just received an INTRODUCE2 cell on the established introduction circuit
+/**
+ *  Go into <b>data</b> and add the right subcredential to be able to handle
+ *  this incoming cell.
+ *
+ *  <b>desc_subcred</b> is the subcredential of the descriptor that corresponds
+ *  to the intro point that received this intro request. This subcredential
+ *  should be used if we are not an onionbalance instance.
+ *
+ *  Return 0 if everything went well, or -1 in case of internal error.
+ */
+static int
+get_subcredential_for_handling_intro2_cell(const hs_service_t *service,
+                                        hs_cell_introduce2_data_t *data,
+                                        const hs_subcredential_t *desc_subcred)
+{
+  /* Handle the simple case first: We are not an onionbalance instance and we
+   * should just use the regular descriptor subcredential */
+  if (!hs_ob_service_is_instance(service)) {
+    data->n_subcredentials = 1;
+    data->subcredentials = desc_subcred;
+    return 0;
+  }
+
+  /* This should not happen since we should have made onionbalance
+   * subcredentials when we created our descriptors. */
+  if (BUG(!service->state.ob_subcreds)) {
+    return -1;
+  }
+
+  /* We are an onionbalance instance: */
+  data->n_subcredentials = service->state.n_ob_subcreds;
+  data->subcredentials = service->state.ob_subcreds;
+
+  return 0;
+}
+
+/** We just received an INTRODUCE2 cell on the established introduction circuit
  * circ.  Handle the INTRODUCE2 payload of size payload_len for the given
  * circuit and service. This cell is associated with the intro point object ip
  * and the subcredential. Return 0 on success else a negative value. */
@@ -1006,7 +1017,7 @@ int
 hs_circ_handle_introduce2(const hs_service_t *service,
                           const origin_circuit_t *circ,
                           hs_service_intro_point_t *ip,
-                          const uint8_t *subcredential,
+                          const hs_subcredential_t *subcredential,
                           const uint8_t *payload, size_t payload_len)
 {
   int ret = -1;
@@ -1023,12 +1034,16 @@ hs_circ_handle_introduce2(const hs_service_t *service,
    * parsed, decrypted and key material computed correctly. */
   data.auth_pk = &ip->auth_key_kp.pubkey;
   data.enc_kp = &ip->enc_key_kp;
-  data.subcredential = subcredential;
   data.payload = payload;
   data.payload_len = payload_len;
   data.link_specifiers = smartlist_new();
   data.replay_cache = ip->replay_cache;
 
+  if (get_subcredential_for_handling_intro2_cell(service,
+                                                 &data, subcredential)) {
+    goto done;
+  }
+
   if (hs_cell_parse_introduce2(&data, circ, service) < 0) {
     goto done;
   }
@@ -1060,14 +1075,12 @@ hs_circ_handle_introduce2(const hs_service_t *service,
   ret = 0;
 
  done:
-  SMARTLIST_FOREACH(data.link_specifiers, link_specifier_t *, lspec,
-                    link_specifier_free(lspec));
-  smartlist_free(data.link_specifiers);
+  link_specifier_smartlist_free(data.link_specifiers);
   memwipe(&data, 0, sizeof(data));
   return ret;
 }
 
-/* Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
+/** Circuit <b>circ</b> just finished the rend ntor key exchange. Use the key
  * exchange output material at <b>ntor_key_seed</b> and setup <b>circ</b> to
  * serve as a rendezvous end-to-end circuit between the client and the
  * service. If <b>is_service_side</b> is set, then we are the hidden service
@@ -1097,7 +1110,7 @@ hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
   return 0;
 }
 
-/* We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
+/** We are a v2 legacy HS client and we just received a RENDEZVOUS1 cell
  * <b>rend_cell_body</b> on <b>circ</b>. Finish up the DH key exchange and then
  * extend the crypt path of <b>circ</b> so that the hidden service is on the
  * other side. */
@@ -1122,7 +1135,7 @@ hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
   return 0;
 }
 
-/* Given the introduction circuit intro_circ, the rendezvous circuit
+/** Given the introduction circuit intro_circ, the rendezvous circuit
  * rend_circ, a descriptor intro point object ip and the service's
  * subcredential, send an INTRODUCE1 cell on intro_circ.
  *
@@ -1134,7 +1147,7 @@ int
 hs_circ_send_introduce1(origin_circuit_t *intro_circ,
                         origin_circuit_t *rend_circ,
                         const hs_desc_intro_point_t *ip,
-                        const uint8_t *subcredential)
+                        const hs_subcredential_t *subcredential)
 {
   int ret = -1;
   ssize_t payload_len;
@@ -1207,7 +1220,7 @@ hs_circ_send_introduce1(origin_circuit_t *intro_circ,
   return ret;
 }
 
-/* Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
+/** Send an ESTABLISH_RENDEZVOUS cell along the rendezvous circuit circ. On
  * success, 0 is returned else -1 and the circuit is marked for close. */
 int
 hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
@@ -1258,30 +1271,136 @@ hs_circ_send_establish_rendezvous(origin_circuit_t *circ)
   return -1;
 }
 
-/* We are about to close or free this <b>circ</b>. Clean it up from any
- * related HS data structures. This function can be called multiple times
- * safely for the same circuit. */
+/** Circuit cleanup strategy:
+ *
+ *  What follows is a series of functions that notifies the HS subsystem of 3
+ *  different circuit cleanup phase: close, free and repurpose.
+ *
+ *  Tor can call any of those in any orders so they have to be safe between
+ *  each other. In other words, the free should never depend on close to be
+ *  called before.
+ *
+ *  The "on_close()" is called from circuit_mark_for_close() which is
+ *  considered the tor fast path and thus as little work as possible should
+ *  done in that function. Currently, we only remove the circuit from the HS
+ *  circuit map and move on.
+ *
+ *  The "on_free()" is called from circuit circuit_free_() and it is very
+ *  important that at the end of the function, no state or objects related to
+ *  this circuit remains alive.
+ *
+ *  The "on_repurpose()" is called from circuit_change_purpose() for which we
+ *  simply remove it from the HS circuit map. We do not have other cleanup
+ *  requirements after that.
+ *
+ *  NOTE: The onion service code, specifically the service code, cleans up
+ *  lingering objects or state if any of its circuit disappear which is why
+ *  our cleanup strategy doesn't involve any service specific actions. As long
+ *  as the circuit is removed from the HS circuit map, it won't be used.
+ */
+
+/** We are about to close this <b>circ</b>. Clean it up from any related HS
+ * data structures. This function can be called multiple times safely for the
+ * same circuit. */
 void
-hs_circ_cleanup(circuit_t *circ)
+hs_circ_cleanup_on_close(circuit_t *circ)
 {
   tor_assert(circ);
 
-  /* If it's a service-side intro circ, notify the HS subsystem for the intro
-   * point circuit closing so it can be dealt with cleanly. */
-  if (circ->purpose == CIRCUIT_PURPOSE_S_ESTABLISH_INTRO ||
-      circ->purpose == CIRCUIT_PURPOSE_S_INTRO) {
-    hs_service_intro_circ_has_closed(TO_ORIGIN_CIRCUIT(circ));
+  if (circuit_purpose_is_hs_client(circ->purpose)) {
+    cleanup_on_close_client_circ(circ);
   }
 
-  /* Clear HS circuitmap token for this circ (if any). Very important to be
-   * done after the HS subsystem has been notified of the close else the
-   * circuit will not be found.
-   *
-   * We do this at the close if possible because from that point on, the
-   * circuit is good as dead. We can't rely on removing it in the circuit
-   * free() function because we open a race window between the close and free
-   * where we can't register a new circuit for the same intro point. */
+  /* On close, we simply remove it from the circuit map. It can not be used
+   * anymore. We keep this code path fast and lean. */
+
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
+
+/** We are about to free this <b>circ</b>. Clean it up from any related HS
+ * data structures. This function can be called multiple times safely for the
+ * same circuit. */
+void
+hs_circ_cleanup_on_free(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* NOTE: Bulk of the work of cleaning up a circuit is done here. */
+
+  if (circuit_purpose_is_hs_client(circ->purpose)) {
+    cleanup_on_free_client_circ(circ);
+  }
+
+  /* We have no assurance that the given HS circuit has been closed before and
+   * thus removed from the HS map. This actually happens in unit tests. */
   if (circ->hs_token) {
     hs_circuitmap_remove_circuit(circ);
   }
 }
+
+/** We are about to repurpose this <b>circ</b>. Clean it up from any related
+ * HS data structures. This function can be called multiple times safely for
+ * the same circuit. */
+void
+hs_circ_cleanup_on_repurpose(circuit_t *circ)
+{
+  tor_assert(circ);
+
+  /* On repurpose, we simply remove it from the circuit map but we do not do
+   * the on_free actions since we don't treat a repurpose as something we need
+   * to report in the client cache failure. */
+
+  if (circ->hs_token) {
+    hs_circuitmap_remove_circuit(circ);
+  }
+}
+
+/** Return true iff the given established client rendezvous circuit was sent
+ * into the INTRODUCE1 cell. This is called so we can take a decision on
+ * expiring or not the circuit.
+ *
+ * The caller MUST make sure the circuit is an established client rendezvous
+ * circuit (purpose: CIRCUIT_PURPOSE_C_REND_READY).
+ *
+ * This function supports all onion service versions. */
+bool
+hs_circ_is_rend_sent_in_intro1(const origin_circuit_t *circ)
+{
+  tor_assert(circ);
+  /* This can only be called for a rendezvous circuit that is an established
+   * confirmed rendezsvous circuit but without an introduction ACK. */
+  tor_assert(TO_CIRCUIT(circ)->purpose == CIRCUIT_PURPOSE_C_REND_READY);
+
+  /* The v2 and v3 circuit are handled differently:
+   *
+   * v2: A circ's pending_final_cpath field is non-NULL iff it is a rend circ
+   * and we have tried to send an INTRODUCE1 cell specifying it. Thus, if the
+   * pending_final_cpath field *is* NULL, then we want to not spare it.
+   *
+   * v3: When the INTRODUCE1 cell is sent, the introduction encryption public
+   * key is copied in the rendezvous circuit hs identifier. If it is a valid
+   * key, we know that this circuit is waiting the ACK on the introduction
+   * circuit. We want to _not_ spare the circuit if the key was never set. */
+
+  if (circ->rend_data) {
+    /* v2. */
+    if (circ->build_state && circ->build_state->pending_final_cpath != NULL) {
+      return true;
+    }
+  } else if (circ->hs_ident) {
+    /* v3. */
+    if (curve25519_public_key_is_ok(&circ->hs_ident->intro_enc_pk)) {
+      return true;
+    }
+  } else {
+    /* A circuit with an HS purpose without an hs_ident or rend_data in theory
+     * can not happen. In case, scream loudly and return false to the caller
+     * that the rendezvous was not sent in the INTRO1 cell. */
+    tor_assert_nonfatal_unreached();
+  }
+
+  /* The rendezvous has not been specified in the INTRODUCE1 cell. */
+  return false;
+}
diff --git a/src/feature/hs/hs_circuit.h b/src/feature/hs/hs_circuit.h
index e168b301f1..22e936e685 100644
--- a/src/feature/hs/hs_circuit.h
+++ b/src/feature/hs/hs_circuit.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,8 +14,10 @@
 
 #include "feature/hs/hs_service.h"
 
-/* Cleanup function when the circuit is closed or/and freed. */
-void hs_circ_cleanup(circuit_t *circ);
+/* Cleanup function when the circuit is closed or freed. */
+void hs_circ_cleanup_on_close(circuit_t *circ);
+void hs_circ_cleanup_on_free(circuit_t *circ);
+void hs_circ_cleanup_on_repurpose(circuit_t *circ);
 
 /* Circuit API. */
 int hs_circ_service_intro_has_opened(hs_service_t *service,
@@ -35,6 +37,8 @@ void hs_circ_retry_service_rendezvous_point(origin_circuit_t *circ);
 
 origin_circuit_t *hs_circ_service_get_intro_circ(
                                       const hs_service_intro_point_t *ip);
+origin_circuit_t *hs_circ_service_get_established_intro_circ(
+                                      const hs_service_intro_point_t *ip);
 
 /* Cell API. */
 int hs_circ_handle_intro_established(const hs_service_t *service,
@@ -42,15 +46,16 @@ int hs_circ_handle_intro_established(const hs_service_t *service,
                                      origin_circuit_t *circ,
                                      const uint8_t *payload,
                                      size_t payload_len);
+struct hs_subcredential_t;
 int hs_circ_handle_introduce2(const hs_service_t *service,
                               const origin_circuit_t *circ,
                               hs_service_intro_point_t *ip,
-                              const uint8_t *subcredential,
+                              const struct hs_subcredential_t *subcredential,
                               const uint8_t *payload, size_t payload_len);
 int hs_circ_send_introduce1(origin_circuit_t *intro_circ,
                             origin_circuit_t *rend_circ,
                             const hs_desc_intro_point_t *ip,
-                            const uint8_t *subcredential);
+                            const struct hs_subcredential_t *subcredential);
 int hs_circ_send_establish_rendezvous(origin_circuit_t *circ);
 
 /* e2e circuit API. */
@@ -62,15 +67,24 @@ int hs_circuit_setup_e2e_rend_circ(origin_circuit_t *circ,
 int hs_circuit_setup_e2e_rend_circ_legacy_client(origin_circuit_t *circ,
                                           const uint8_t *rend_cell_body);
 
+bool hs_circ_is_rend_sent_in_intro1(const origin_circuit_t *circ);
+
 #ifdef HS_CIRCUIT_PRIVATE
 
+struct hs_ntor_rend_cell_keys_t;
+
 STATIC hs_ident_circuit_t *
 create_rp_circuit_identifier(const hs_service_t *service,
                              const uint8_t *rendezvous_cookie,
                              const curve25519_public_key_t *server_pk,
-                             const hs_ntor_rend_cell_keys_t *keys);
+                             const struct hs_ntor_rend_cell_keys_t *keys);
+
+struct hs_cell_introduce2_data_t;
+MOCK_DECL(STATIC void,
+launch_rendezvous_point_circuit,(const hs_service_t *service,
+                                 const hs_service_intro_point_t *ip,
+                                const struct hs_cell_introduce2_data_t *data));
 
 #endif /* defined(HS_CIRCUIT_PRIVATE) */
 
 #endif /* !defined(TOR_HS_CIRCUIT_H) */
-
diff --git a/src/feature/hs/hs_circuitmap.c b/src/feature/hs/hs_circuitmap.c
index 5480d5eb84..466a02de39 100644
--- a/src/feature/hs/hs_circuitmap.c
+++ b/src/feature/hs/hs_circuitmap.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,13 +23,13 @@
 
 /************************** HS circuitmap code *******************************/
 
-/* This is the hidden service circuitmap. It's a hash table that maps
+/** This is the hidden service circuitmap. It's a hash table that maps
    introduction and rendezvous tokens to specific circuits such that given a
    token it's easy to find the corresponding circuit. */
 static struct hs_circuitmap_ht *the_hs_circuitmap = NULL;
 
-/* This is a helper function used by the hash table code (HT_). It returns 1 if
- * two circuits have the same HS token. */
+/** This is a helper function used by the hash table code (HT_). It returns 1
+ * if two circuits have the same HS token. */
 static int
 hs_circuits_have_same_token(const circuit_t *first_circuit,
                             const circuit_t *second_circuit)
@@ -60,8 +60,9 @@ hs_circuits_have_same_token(const circuit_t *first_circuit,
                    first_token->token_len);
 }
 
-/* This is a helper function for the hash table code (HT_). It hashes a circuit
- * HS token into an unsigned int for use as a key by the hash table routines.*/
+/** This is a helper function for the hash table code (HT_). It hashes a
+ * circuit HS token into an unsigned int for use as a key by the hash table
+ * routines.*/
 static inline unsigned int
 hs_circuit_hash_token(const circuit_t *circuit)
 {
@@ -71,19 +72,19 @@ hs_circuit_hash_token(const circuit_t *circuit)
                                circuit->hs_token->token_len);
 }
 
-/* Register the circuitmap hash table */
+/** Register the circuitmap hash table */
 HT_PROTOTYPE(hs_circuitmap_ht, // The name of the hashtable struct
              circuit_t,    // The name of the element struct,
              hs_circuitmap_node,        // The name of HT_ENTRY member
-             hs_circuit_hash_token, hs_circuits_have_same_token)
+             hs_circuit_hash_token, hs_circuits_have_same_token);
 
 HT_GENERATE2(hs_circuitmap_ht, circuit_t, hs_circuitmap_node,
              hs_circuit_hash_token, hs_circuits_have_same_token,
-             0.6, tor_reallocarray, tor_free_)
+             0.6, tor_reallocarray, tor_free_);
 
 #ifdef TOR_UNIT_TESTS
 
-/* Return the global HS circuitmap. Used by unittests. */
+/** Return the global HS circuitmap. Used by unittests. */
 hs_circuitmap_ht *
 get_hs_circuitmap(void)
 {
@@ -136,7 +137,7 @@ get_circuit_with_token(hs_token_t *search_token)
   return HT_FIND(hs_circuitmap_ht, the_hs_circuitmap, &search_circ);
 }
 
-/* Helper function that registers <b>circ</b> with <b>token</b> on the HS
+/** Helper function that registers <b>circ</b> with <b>token</b> on the HS
    circuitmap. This function steals reference of <b>token</b>. */
 static void
 hs_circuitmap_register_impl(circuit_t *circ, hs_token_t *token)
@@ -186,7 +187,7 @@ hs_circuitmap_register_circuit(circuit_t *circ,
   hs_circuitmap_register_impl(circ, hs_token);
 }
 
-/* Helper function for hs_circuitmap_get_origin_circuit() and
+/** Helper function for hs_circuitmap_get_origin_circuit() and
  * hs_circuitmap_get_or_circuit(). Because only circuit_t are indexed in the
  * circuitmap, this function returns object type so the specialized functions
  * using this helper can upcast it to the right type.
@@ -220,7 +221,7 @@ hs_circuitmap_get_circuit_impl(hs_token_type_t type,
   return found_circ;
 }
 
-/* Helper function: Query circuitmap for origin circuit with <b>token</b> of
+/** Helper function: Query circuitmap for origin circuit with <b>token</b> of
  * size <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose
  * equal to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked
  * for close. Return NULL if no such circuit is found. */
@@ -244,7 +245,7 @@ hs_circuitmap_get_origin_circuit(hs_token_type_t type,
   return TO_ORIGIN_CIRCUIT(circ);
 }
 
-/* Helper function: Query circuitmap for OR circuit with <b>token</b> of size
+/** Helper function: Query circuitmap for OR circuit with <b>token</b> of size
  * <b>token_len</b> and <b>type</b>.  Only returns a circuit with purpose equal
  * to the <b>wanted_circ_purpose</b> parameter and if it is NOT marked for
  * close. Return NULL if no such circuit is found. */
@@ -272,7 +273,34 @@ hs_circuitmap_get_or_circuit(hs_token_type_t type,
 
 /**** Public relay-side getters: */
 
-/* Public function: Return a v3 introduction circuit to this relay with
+/** Public function: Return v2 and v3 introduction circuit to this relay.
+ * Always return a newly allocated list for which it is the caller's
+ * responsability to free it. */
+smartlist_t *
+hs_circuitmap_get_all_intro_circ_relay_side(void)
+{
+  circuit_t **iter;
+  smartlist_t *circuit_list = smartlist_new();
+
+  HT_FOREACH(iter, hs_circuitmap_ht, the_hs_circuitmap) {
+    circuit_t *circ = *iter;
+
+    /* An origin circuit or purpose is wrong or the hs token is not set to be
+     * a v2 or v3 intro relay side type, we ignore the circuit. Else, we have
+     * a match so add it to our list. */
+    if (CIRCUIT_IS_ORIGIN(circ) ||
+        circ->purpose != CIRCUIT_PURPOSE_INTRO_POINT ||
+        (circ->hs_token->type != HS_TOKEN_INTRO_V3_RELAY_SIDE &&
+         circ->hs_token->type != HS_TOKEN_INTRO_V2_RELAY_SIDE)) {
+      continue;
+    }
+    smartlist_add(circuit_list, circ);
+  }
+
+  return circuit_list;
+}
+
+/** Public function: Return a v3 introduction circuit to this relay with
  * <b>auth_key</b>. Return NULL if no such circuit is found in the
  * circuitmap. */
 or_circuit_t *
@@ -284,7 +312,7 @@ hs_circuitmap_get_intro_circ_v3_relay_side(
                                       CIRCUIT_PURPOSE_INTRO_POINT);
 }
 
-/* Public function: Return v2 introduction circuit to this relay with
+/** Public function: Return v2 introduction circuit to this relay with
  * <b>digest</b>. Return NULL if no such circuit is found in the circuitmap. */
 or_circuit_t *
 hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
@@ -294,7 +322,7 @@ hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest)
                                       CIRCUIT_PURPOSE_INTRO_POINT);
 }
 
-/* Public function: Return rendezvous circuit to this relay with rendezvous
+/** Public function: Return rendezvous circuit to this relay with rendezvous
  * <b>cookie</b>. Return NULL if no such circuit is found in the circuitmap. */
 or_circuit_t *
 hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
@@ -306,7 +334,7 @@ hs_circuitmap_get_rend_circ_relay_side(const uint8_t *cookie)
 
 /** Public relay-side setters: */
 
-/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+/** Public function: Register rendezvous circuit with key <b>cookie</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
@@ -316,7 +344,7 @@ hs_circuitmap_register_rend_circ_relay_side(or_circuit_t *circ,
                                  HS_TOKEN_REND_RELAY_SIDE,
                                  REND_TOKEN_LEN, cookie);
 }
-/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+/** Public function: Register v2 intro circuit with key <b>digest</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
@@ -327,7 +355,7 @@ hs_circuitmap_register_intro_circ_v2_relay_side(or_circuit_t *circ,
                                  REND_TOKEN_LEN, digest);
 }
 
-/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+/** Public function: Register v3 intro circuit with key <b>auth_key</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
@@ -340,7 +368,7 @@ hs_circuitmap_register_intro_circ_v3_relay_side(or_circuit_t *circ,
 
 /**** Public servide-side getters: */
 
-/* Public function: Return v3 introduction circuit with <b>auth_key</b>
+/** Public function: Return v3 introduction circuit with <b>auth_key</b>
  * originating from this hidden service. Return NULL if no such circuit is
  * found in the circuitmap. */
 origin_circuit_t *
@@ -365,9 +393,9 @@ hs_circuitmap_get_intro_circ_v3_service_side(const
   return circ;
 }
 
-/* Public function: Return v2 introduction circuit originating from this hidden
- * service with <b>digest</b>. Return NULL if no such circuit is found in the
- * circuitmap. */
+/** Public function: Return v2 introduction circuit originating from this
+ * hidden service with <b>digest</b>. Return NULL if no such circuit is found
+ * in the circuitmap. */
 origin_circuit_t *
 hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
 {
@@ -389,7 +417,7 @@ hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest)
   return circ;
 }
 
-/* Public function: Return rendezvous circuit originating from this hidden
+/** Public function: Return rendezvous circuit originating from this hidden
  * service with rendezvous <b>cookie</b>. Return NULL if no such circuit is
  * found in the circuitmap. */
 origin_circuit_t *
@@ -412,7 +440,7 @@ hs_circuitmap_get_rend_circ_service_side(const uint8_t *cookie)
   return circ;
 }
 
-/* Public function: Return client-side rendezvous circuit with rendezvous
+/** Public function: Return client-side rendezvous circuit with rendezvous
  * <b>cookie</b>. It will look for circuits with the following purposes:
 
  * a) CIRCUIT_PURPOSE_C_REND_READY: Established rend circuit (received
@@ -445,7 +473,7 @@ hs_circuitmap_get_rend_circ_client_side(const uint8_t *cookie)
   return circ;
 }
 
-/*  Public function: Return client-side established rendezvous circuit with
+/**  Public function: Return client-side established rendezvous circuit with
  *  rendezvous <b>cookie</b>. It will look for circuits with the following
  *  purposes:
  *
@@ -487,7 +515,7 @@ hs_circuitmap_get_established_rend_circ_client_side(const uint8_t *cookie)
 
 /**** Public servide-side setters: */
 
-/* Public function: Register v2 intro circuit with key <b>digest</b> to the
+/** Public function: Register v2 intro circuit with key <b>digest</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
@@ -498,7 +526,7 @@ hs_circuitmap_register_intro_circ_v2_service_side(origin_circuit_t *circ,
                                  REND_TOKEN_LEN, digest);
 }
 
-/* Public function: Register v3 intro circuit with key <b>auth_key</b> to the
+/** Public function: Register v3 intro circuit with key <b>auth_key</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
@@ -509,7 +537,7 @@ hs_circuitmap_register_intro_circ_v3_service_side(origin_circuit_t *circ,
                                  ED25519_PUBKEY_LEN, auth_key->pubkey);
 }
 
-/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+/** Public function: Register rendezvous circuit with key <b>cookie</b> to the
  * circuitmap. */
 void
 hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
@@ -520,7 +548,7 @@ hs_circuitmap_register_rend_circ_service_side(origin_circuit_t *circ,
                                  REND_TOKEN_LEN, cookie);
 }
 
-/* Public function: Register rendezvous circuit with key <b>cookie</b> to the
+/** Public function: Register rendezvous circuit with key <b>cookie</b> to the
  * client-side circuitmap. */
 void
 hs_circuitmap_register_rend_circ_client_side(origin_circuit_t *or_circ,
@@ -564,7 +592,7 @@ hs_circuitmap_remove_circuit(circuit_t *circ)
   circ->hs_token = NULL;
 }
 
-/* Public function: Initialize the global HS circuitmap. */
+/** Public function: Initialize the global HS circuitmap. */
 void
 hs_circuitmap_init(void)
 {
@@ -574,7 +602,7 @@ hs_circuitmap_init(void)
   HT_INIT(hs_circuitmap_ht, the_hs_circuitmap);
 }
 
-/* Public function: Free all memory allocated by the global HS circuitmap. */
+/** Public function: Free all memory allocated by the global HS circuitmap. */
 void
 hs_circuitmap_free_all(void)
 {
diff --git a/src/feature/hs/hs_circuitmap.h b/src/feature/hs/hs_circuitmap.h
index c1bbb1ff1c..df3e7a6e7e 100644
--- a/src/feature/hs/hs_circuitmap.h
+++ b/src/feature/hs/hs_circuitmap.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@ typedef HT_HEAD(hs_circuitmap_ht, circuit_t) hs_circuitmap_ht;
 typedef struct hs_token_t hs_token_t;
 struct or_circuit_t;
 struct origin_circuit_t;
+struct ed25519_public_key_t;
 
 /** Public HS circuitmap API: */
 
@@ -21,7 +22,7 @@ struct origin_circuit_t;
 
 struct or_circuit_t *
 hs_circuitmap_get_intro_circ_v3_relay_side(const
-                                           ed25519_public_key_t *auth_key);
+                                   struct ed25519_public_key_t *auth_key);
 struct or_circuit_t *
 hs_circuitmap_get_intro_circ_v2_relay_side(const uint8_t *digest);
 struct or_circuit_t *
@@ -32,13 +33,15 @@ void hs_circuitmap_register_rend_circ_relay_side(struct or_circuit_t *circ,
 void hs_circuitmap_register_intro_circ_v2_relay_side(struct or_circuit_t *circ,
                                                      const uint8_t *digest);
 void hs_circuitmap_register_intro_circ_v3_relay_side(struct or_circuit_t *circ,
-                                         const ed25519_public_key_t *auth_key);
+                                 const struct ed25519_public_key_t *auth_key);
+
+smartlist_t *hs_circuitmap_get_all_intro_circ_relay_side(void);
 
 /** Public service-side API: */
 
 struct origin_circuit_t *
 hs_circuitmap_get_intro_circ_v3_service_side(const
-                                             ed25519_public_key_t *auth_key);
+                                     struct ed25519_public_key_t *auth_key);
 struct origin_circuit_t *
 hs_circuitmap_get_intro_circ_v2_service_side(const uint8_t *digest);
 struct origin_circuit_t *
@@ -52,8 +55,8 @@ void hs_circuitmap_register_intro_circ_v2_service_side(
                                         struct origin_circuit_t *circ,
                                         const uint8_t *digest);
 void hs_circuitmap_register_intro_circ_v3_service_side(
-                                        struct origin_circuit_t *circ,
-                                        const ed25519_public_key_t *auth_key);
+                                 struct origin_circuit_t *circ,
+                                 const struct ed25519_public_key_t *auth_key);
 void hs_circuitmap_register_rend_circ_service_side(
                                         struct origin_circuit_t *circ,
                                         const uint8_t *cookie);
diff --git a/src/feature/hs/hs_client.c b/src/feature/hs/hs_client.c
index e25919ecb7..0f6109195b 100644
--- a/src/feature/hs/hs_client.c
+++ b/src/feature/hs/hs_client.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,14 +43,15 @@
 #include "core/or/entry_connection_st.h"
 #include "core/or/extend_info_st.h"
 #include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
 
-/* Client-side authorizations for hidden services; map of service identity
+/** Client-side authorizations for hidden services; map of service identity
  * public key to hs_client_service_authorization_t *. */
 static digest256map_t *client_auths = NULL;
 
 #include "trunnel/hs/cell_introduce1.h"
 
-/* Return a human-readable string for the client fetch status code. */
+/** Return a human-readable string for the client fetch status code. */
 static const char *
 fetch_status_to_string(hs_client_fetch_status_t status)
 {
@@ -74,7 +75,7 @@ fetch_status_to_string(hs_client_fetch_status_t status)
   }
 }
 
-/* Return true iff tor should close the SOCKS request(s) for the descriptor
+/** Return true iff tor should close the SOCKS request(s) for the descriptor
  * fetch that ended up with this given status code. */
 static int
 fetch_status_should_close_socks(hs_client_fetch_status_t status)
@@ -101,12 +102,51 @@ fetch_status_should_close_socks(hs_client_fetch_status_t status)
   return 1;
 }
 
+/* Return a newly allocated list of all the entry connections that matches the
+ * given service identity pk. If service_identity_pk is NULL, all entry
+ * connections with an hs_ident are returned.
+ *
+ * Caller must free the returned list but does NOT have ownership of the
+ * object inside thus they have to remain untouched. */
+static smartlist_t *
+find_entry_conns(const ed25519_public_key_t *service_identity_pk)
+{
+  time_t now = time(NULL);
+  smartlist_t *conns = NULL, *entry_conns = NULL;
+
+  entry_conns = smartlist_new();
+
+  conns = connection_list_by_type_state(CONN_TYPE_AP,
+                                        AP_CONN_STATE_RENDDESC_WAIT);
+  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
+    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+
+    /* Only consider the entry connections that matches the service for which
+     * we just fetched its descriptor. */
+    if (!edge_conn->hs_ident ||
+        (service_identity_pk &&
+         !ed25519_pubkey_eq(service_identity_pk,
+                            &edge_conn->hs_ident->identity_pk))) {
+      continue;
+    }
+    assert_connection_ok(base_conn, now);
+
+    /* Validated! Add the entry connection to the list. */
+    smartlist_add(entry_conns, entry_conn);
+  } SMARTLIST_FOREACH_END(base_conn);
+
+  /* We don't have ownership of the objects in this list. */
+  smartlist_free(conns);
+  return entry_conns;
+}
+
 /* Cancel all descriptor fetches currently in progress. */
 static void
 cancel_descriptor_fetches(void)
 {
   smartlist_t *conns =
-    connection_list_by_type_state(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
+    connection_list_by_type_purpose(CONN_TYPE_DIR, DIR_PURPOSE_FETCH_HSDESC);
   SMARTLIST_FOREACH_BEGIN(conns, connection_t *, conn) {
     const hs_ident_dir_conn_t *ident = TO_DIR_CONN(conn)->hs_ident;
     if (BUG(ident == NULL)) {
@@ -125,7 +165,7 @@ cancel_descriptor_fetches(void)
   log_info(LD_REND, "Hidden service client descriptor fetches cancelled.");
 }
 
-/* Get all connections that are waiting on a circuit and flag them back to
+/** Get all connections that are waiting on a circuit and flag them back to
  * waiting for a hidden service descriptor for the given service key
  * service_identity_pk. */
 static void
@@ -152,7 +192,7 @@ flag_all_conn_wait_desc(const ed25519_public_key_t *service_identity_pk)
   smartlist_free(conns);
 }
 
-/* Remove tracked HSDir requests from our history for this hidden service
+/** Remove tracked HSDir requests from our history for this hidden service
  * identity public key. */
 static void
 purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
@@ -168,14 +208,12 @@ purge_hid_serv_request(const ed25519_public_key_t *identity_pk)
    * some point and we don't care about those anymore. */
   hs_build_blinded_pubkey(identity_pk, NULL, 0,
                           hs_get_time_period_num(0), &blinded_pk);
-  if (BUG(ed25519_public_to_base64(base64_blinded_pk, &blinded_pk) < 0)) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   /* Purge last hidden service request from cache for this blinded key. */
   hs_purge_hid_serv_from_last_hid_serv_requests(base64_blinded_pk);
 }
 
-/* Return true iff there is at least one pending directory descriptor request
+/** Return true iff there is at least one pending directory descriptor request
  * for the service identity_pk. */
 static int
 directory_request_is_pending(const ed25519_public_key_t *identity_pk)
@@ -203,7 +241,7 @@ directory_request_is_pending(const ed25519_public_key_t *identity_pk)
   return ret;
 }
 
-/* Helper function that changes the state of an entry connection to waiting
+/** Helper function that changes the state of an entry connection to waiting
  * for a circuit. For this to work properly, the connection timestamps are set
  * to now and the connection is then marked as pending for a circuit. */
 static void
@@ -223,7 +261,7 @@ mark_conn_as_waiting_for_circuit(connection_t *conn, time_t now)
   connection_ap_mark_as_pending_circuit(TO_ENTRY_CONN(conn));
 }
 
-/* We failed to fetch a descriptor for the service with <b>identity_pk</b>
+/** We failed to fetch a descriptor for the service with <b>identity_pk</b>
  * because of <b>status</b>. Find all pending SOCKS connections for this
  * service that are waiting on the descriptor and close them with
  * <b>reason</b>. */
@@ -233,26 +271,13 @@ close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
                                        int reason)
 {
   unsigned int count = 0;
-  time_t now = approx_time();
-  smartlist_t *conns =
-    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+  smartlist_t *entry_conns = find_entry_conns(identity_pk);
 
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
-    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
-
-    /* Only consider the entry connections that matches the service for which
-     * we tried to get the descriptor */
-    if (!edge_conn->hs_ident ||
-        !ed25519_pubkey_eq(identity_pk,
-                           &edge_conn->hs_ident->identity_pk)) {
-      continue;
-    }
-    assert_connection_ok(base_conn, now);
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
     /* Unattach the entry connection which will close for the reason. */
     connection_mark_unattached_ap(entry_conn, reason);
     count++;
-  } SMARTLIST_FOREACH_END(base_conn);
+  } SMARTLIST_FOREACH_END(entry_conn);
 
   if (count > 0) {
     char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
@@ -265,26 +290,26 @@ close_all_socks_conns_waiting_for_desc(const ed25519_public_key_t *identity_pk,
   }
 
   /* No ownership of the object(s) in this list. */
-  smartlist_free(conns);
+  smartlist_free(entry_conns);
 }
 
-/* Find all pending SOCKS connection waiting for a descriptor and retry them
+/** Find all pending SOCKS connection waiting for a descriptor and retry them
  * all. This is called when the directory information changed. */
 STATIC void
 retry_all_socks_conn_waiting_for_desc(void)
 {
-  smartlist_t *conns =
-    connection_list_by_type_state(CONN_TYPE_AP, AP_CONN_STATE_RENDDESC_WAIT);
+  smartlist_t *entry_conns = find_entry_conns(NULL);
 
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
     hs_client_fetch_status_t status;
-    const edge_connection_t *edge_conn =
-      ENTRY_TO_EDGE_CONN(TO_ENTRY_CONN(base_conn));
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+    connection_t *base_conn = &edge_conn->base_;
 
     /* Ignore non HS or non v3 connection. */
     if (edge_conn->hs_ident == NULL) {
       continue;
     }
+
     /* In this loop, we will possibly try to fetch a descriptor for the
      * pending connections because we just got more directory information.
      * However, the refetch process can cleanup all SOCKS request to the same
@@ -318,13 +343,13 @@ retry_all_socks_conn_waiting_for_desc(void)
      * closed or we are still missing directory information. Leave the
      * connection in renddesc wait state so when we get more info, we'll be
      * able to try it again. */
-  } SMARTLIST_FOREACH_END(base_conn);
+  } SMARTLIST_FOREACH_END(entry_conn);
 
   /* We don't have ownership of those objects. */
-  smartlist_free(conns);
+  smartlist_free(entry_conns);
 }
 
-/* A v3 HS circuit successfully connected to the hidden service. Update the
+/** A v3 HS circuit successfully connected to the hidden service. Update the
  * stream state at <b>hs_conn_ident</b> appropriately. */
 static void
 note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
@@ -346,7 +371,7 @@ note_connection_attempt_succeeded(const hs_ident_edge_conn_t *hs_conn_ident)
    * will be reset and thus possible to be retried. */
 }
 
-/* Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
+/** Given the pubkey of a hidden service in <b>onion_identity_pk</b>, fetch its
  * descriptor by launching a dir connection to <b>hsdir</b>. Return a
  * hs_client_fetch_status_t status code depending on how it went. */
 static hs_client_fetch_status_t
@@ -357,7 +382,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   ed25519_public_key_t blinded_pubkey;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   hs_ident_dir_conn_t hs_conn_dir_ident;
-  int retval;
 
   tor_assert(hsdir);
   tor_assert(onion_identity_pk);
@@ -366,10 +390,7 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return HS_CLIENT_FETCH_ERROR;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Copy onion pk to a dir_ident so that we attach it to the dir conn */
   hs_ident_dir_conn_init(onion_identity_pk, &blinded_pubkey,
@@ -408,7 +429,6 @@ directory_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
 STATIC routerstatus_t *
 pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 {
-  int retval;
   char base64_blinded_pubkey[ED25519_BASE64_LEN + 1];
   uint64_t current_time_period = hs_get_time_period_num(0);
   smartlist_t *responsible_hsdirs = NULL;
@@ -421,10 +441,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
   hs_build_blinded_pubkey(onion_identity_pk, NULL, 0,
                           current_time_period, &blinded_pubkey);
   /* ...and base64 it. */
-  retval = ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
-  if (BUG(retval < 0)) {
-    return NULL;
-  }
+  ed25519_public_to_base64(base64_blinded_pubkey, &blinded_pubkey);
 
   /* Get responsible hsdirs of service for this time period */
   responsible_hsdirs = smartlist_new();
@@ -437,7 +454,7 @@ pick_hsdir_v3(const ed25519_public_key_t *onion_identity_pk)
 
   /* Pick an HSDir from the responsible ones. The ownership of
    * responsible_hsdirs is given to this function so no need to free it. */
-  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey);
+  hsdir_rs = hs_pick_hsdir(responsible_hsdirs, base64_blinded_pubkey, NULL);
 
   return hsdir_rs;
 }
@@ -462,7 +479,25 @@ fetch_v3_desc, (const ed25519_public_key_t *onion_identity_pk))
   return directory_launch_v3_desc_fetch(onion_identity_pk, hsdir_rs);
 }
 
-/* Make sure that the given v3 origin circuit circ is a valid correct
+/** With a given <b>onion_identity_pk</b>, fetch its descriptor. If
+ * <b>hsdirs</b> is specified, use the directory servers specified in the list.
+ * Else, use a random server. */
+void
+hs_client_launch_v3_desc_fetch(const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  if (hsdirs != NULL) {
+    SMARTLIST_FOREACH_BEGIN(hsdirs, const routerstatus_t *, hsdir) {
+      directory_launch_v3_desc_fetch(onion_identity_pk, hsdir);
+    } SMARTLIST_FOREACH_END(hsdir);
+  } else {
+    fetch_v3_desc(onion_identity_pk);
+  }
+}
+
+/** Make sure that the given v3 origin circuit circ is a valid correct
  * introduction circuit. This will BUG() on any problems and hard assert if
  * the anonymity of the circuit is not ok. Return 0 on success else -1 where
  * the circuit should be mark for closed immediately. */
@@ -491,7 +526,7 @@ intro_circ_is_ok(const origin_circuit_t *circ)
   return ret;
 }
 
-/* Find a descriptor intro point object that matches the given ident in the
+/** Find a descriptor intro point object that matches the given ident in the
  * given descriptor desc. Return NULL if not found. */
 static const hs_desc_intro_point_t *
 find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
@@ -514,7 +549,7 @@ find_desc_intro_point_by_ident(const hs_ident_circuit_t *ident,
   return intro_point;
 }
 
-/* Find a descriptor intro point object from the descriptor object desc that
+/** Find a descriptor intro point object from the descriptor object desc that
  * matches the given legacy identity digest in legacy_id. Return NULL if not
  * found. */
 static hs_desc_intro_point_t *
@@ -531,13 +566,15 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id,
   SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                           hs_desc_intro_point_t *, ip) {
     SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                            const hs_desc_link_specifier_t *, lspec) {
+                            const link_specifier_t *, lspec) {
       /* Not all tor node have an ed25519 identity key so we still rely on the
        * legacy identity digest. */
-      if (lspec->type != LS_LEGACY_ID) {
+      if (link_specifier_get_ls_type(lspec) != LS_LEGACY_ID) {
         continue;
       }
-      if (fast_memneq(legacy_id, lspec->u.legacy_id, DIGEST_LEN)) {
+      if (fast_memneq(legacy_id,
+                      link_specifier_getconstarray_un_legacy_id(lspec),
+                      DIGEST_LEN)) {
         break;
       }
       /* Found it. */
@@ -550,7 +587,7 @@ find_desc_intro_point_by_legacy_id(const char *legacy_id,
   return ret_ip;
 }
 
-/* Send an INTRODUCE1 cell along the intro circuit and populate the rend
+/** Send an INTRODUCE1 cell along the intro circuit and populate the rend
  * circuit identifier with the needed key material for the e2e encryption.
  * Return 0 on success, -1 if there is a transient error such that an action
  * has been taken to recover and -2 if there is a permanent error indicating
@@ -610,7 +647,7 @@ send_introduce1(origin_circuit_t *intro_circ,
 
   /* Send the INTRODUCE1 cell. */
   if (hs_circ_send_introduce1(intro_circ, rend_circ, ip,
-                              desc->subcredential) < 0) {
+                              &desc->subcredential) < 0) {
     if (TO_CIRCUIT(intro_circ)->marked_for_close) {
       /* If the introduction circuit was closed, we were unable to send the
        * cell for some reasons. In any case, the intro circuit has to be
@@ -667,9 +704,12 @@ send_introduce1(origin_circuit_t *intro_circ,
   return status;
 }
 
-/* Using the introduction circuit circ, setup the authentication key of the
- * intro point this circuit has extended to. */
-static void
+/** Using the introduction circuit circ, setup the authentication key of the
+ * intro point this circuit has extended to.
+ *
+ * Return 0 if everything went well, otherwise return -1 in the case of errors.
+ */
+static int
 setup_intro_circ_auth_key(origin_circuit_t *circ)
 {
   const hs_descriptor_t *desc;
@@ -683,30 +723,31 @@ setup_intro_circ_auth_key(origin_circuit_t *circ)
      * and the client descriptor cache that gets purged (NEWNYM) or the
      * cleaned up because it expired. Mark the circuit for close so a new
      * descriptor fetch can occur. */
-    circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
-    goto end;
+    goto err;
   }
 
   /* We will go over every intro point and try to find which one is linked to
    * that circuit. Those lists are small so it's not that expensive. */
   ip = find_desc_intro_point_by_legacy_id(
                        circ->build_state->chosen_exit->identity_digest, desc);
-  if (ip) {
-    /* We got it, copy its authentication key to the identifier. */
-    ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
-                        &ip->auth_key_cert->signed_key);
-    goto end;
+  if (!ip) {
+    /* Reaching this point means we didn't find any intro point for this
+     * circuit which is not supposed to happen. */
+    log_info(LD_REND,"Could not match opened intro circuit with intro point.");
+    goto err;
   }
 
-  /* Reaching this point means we didn't find any intro point for this circuit
-   * which is not suppose to happen. */
-  tor_assert_nonfatal_unreached();
+  /* We got it, copy its authentication key to the identifier. */
+  ed25519_pubkey_copy(&circ->hs_ident->intro_auth_pk,
+                      &ip->auth_key_cert->signed_key);
+  return 0;
 
- end:
-  return;
+ err:
+  circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_INTERNAL);
+  return -1;
 }
 
-/* Called when an introduction circuit has opened. */
+/** Called when an introduction circuit has opened. */
 static void
 client_intro_circ_has_opened(origin_circuit_t *circ)
 {
@@ -718,12 +759,14 @@ client_intro_circ_has_opened(origin_circuit_t *circ)
   /* This is an introduction circuit so we'll attach the correct
    * authentication key to the circuit identifier so it can be identified
    * properly later on. */
-  setup_intro_circ_auth_key(circ);
+  if (setup_intro_circ_auth_key(circ) < 0) {
+    return;
+  }
 
   connection_ap_attach_pending(1);
 }
 
-/* Called when a rendezvous circuit has opened. */
+/** Called when a rendezvous circuit has opened. */
 static void
 client_rendezvous_circ_has_opened(origin_circuit_t *circ)
 {
@@ -736,10 +779,16 @@ client_rendezvous_circ_has_opened(origin_circuit_t *circ)
    * the v3 rendezvous protocol */
   if (rp_ei) {
     const node_t *rp_node = node_get_by_id(rp_ei->identity_digest);
-    if (rp_node) {
-      if (BUG(!node_supports_v3_rendezvous_point(rp_node))) {
-        return;
-      }
+    if (rp_node && !node_supports_v3_rendezvous_point(rp_node)) {
+      /* Even tho we checked that this node supported v3 when we created the
+         rendezvous circuit, there is a chance that we might think it does
+         not support v3 anymore. This might happen if we got a new consensus
+         in the meanwhile, where the relay is still listed but its listed
+         descriptor digest has changed and hence we can't access its 'ri' or
+         'md'. */
+      log_info(LD_REND, "Rendezvous node %s did not support v3 after circuit "
+               "has opened.", safe_str_client(extend_info_describe(rp_ei)));
+      return;
     }
   }
 
@@ -757,7 +806,7 @@ client_rendezvous_circ_has_opened(origin_circuit_t *circ)
   }
 }
 
-/* This is an helper function that convert a descriptor intro point object ip
+/** This is an helper function that convert a descriptor intro point object ip
  * to a newly allocated extend_info_t object fully initialized. Return NULL if
  * we can't convert it for which chances are that we are missing or malformed
  * link specifiers. */
@@ -765,28 +814,17 @@ STATIC extend_info_t *
 desc_intro_point_to_extend_info(const hs_desc_intro_point_t *ip)
 {
   extend_info_t *ei;
-  smartlist_t *lspecs = smartlist_new();
 
   tor_assert(ip);
 
-  /* We first encode the descriptor link specifiers into the binary
-   * representation which is a trunnel object. */
-  SMARTLIST_FOREACH_BEGIN(ip->link_specifiers,
-                          const hs_desc_link_specifier_t *, desc_lspec) {
-    link_specifier_t *lspec = hs_desc_lspec_to_trunnel(desc_lspec);
-    smartlist_add(lspecs, lspec);
-  } SMARTLIST_FOREACH_END(desc_lspec);
-
   /* Explicitly put the direct connection option to 0 because this is client
    * side and there is no such thing as a non anonymous client. */
-  ei = hs_get_extend_info_from_lspecs(lspecs, &ip->onion_key, 0);
+  ei = hs_get_extend_info_from_lspecs(ip->link_specifiers, &ip->onion_key, 0);
 
-  SMARTLIST_FOREACH(lspecs, link_specifier_t *, ls, link_specifier_free(ls));
-  smartlist_free(lspecs);
   return ei;
 }
 
-/* Return true iff the intro point ip for the service service_pk is usable.
+/** Return true iff the intro point ip for the service service_pk is usable.
  * This function checks if the intro point is in the client intro state cache
  * and checks at the failures. It is considered usable if:
  *   - No error happened (INTRO_POINT_FAILURE_GENERIC)
@@ -831,7 +869,7 @@ intro_point_is_usable(const ed25519_public_key_t *service_pk,
   return 0;
 }
 
-/* Using a descriptor desc, return a newly allocated extend_info_t object of a
+/** Using a descriptor desc, return a newly allocated extend_info_t object of a
  * randomly picked introduction point from its list. Return NULL if none are
  * usable. */
 STATIC extend_info_t *
@@ -936,7 +974,88 @@ client_get_random_intro(const ed25519_public_key_t *service_pk)
   return ei;
 }
 
-/* For this introduction circuit, we'll look at if we have any usable
+/** Return true iff all intro points for the given service have timed out. */
+static bool
+intro_points_all_timed_out(const ed25519_public_key_t *service_pk)
+{
+  bool ret = false;
+
+  tor_assert(service_pk);
+
+  const hs_descriptor_t *desc = hs_cache_lookup_as_client(service_pk);
+  if (BUG(!desc)) {
+    /* We can't introduce without a descriptor so ending up here means somehow
+     * between the introduction failure and this, the cache entry was removed
+     * which shouldn't be possible in theory. */
+    goto end;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          const hs_desc_intro_point_t *, ip) {
+    const hs_cache_intro_state_t *state =
+      hs_cache_client_intro_state_find(service_pk,
+                                       &ip->auth_key_cert->signed_key);
+    if (!state || !state->timed_out) {
+      /* No state or if this intro point has not timed out, we are done since
+       * clearly not all of them have timed out. */
+      goto end;
+    }
+  } SMARTLIST_FOREACH_END(ip);
+
+  /* Exiting the loop here means that all intro points we've looked at have
+   * timed out. Note that we can _not_ have a descriptor without intro points
+   * in the client cache. */
+  ret = true;
+
+ end:
+  return ret;
+}
+
+/** Called when a rendezvous circuit has timed out. Every stream attached to
+ * the circuit will get set with the SOCKS5_HS_REND_FAILED (0xF3) extended
+ * error code so if the connection to the rendezvous point ends up not
+ * working, this code could be sent back as a reason. */
+static void
+socks_mark_rend_circuit_timed_out(const origin_circuit_t *rend_circ)
+{
+  tor_assert(rend_circ);
+
+  /* For each entry connection attached to this rendezvous circuit, report
+   * the error. */
+  for (edge_connection_t *edge = rend_circ->p_streams; edge;
+       edge = edge->next_stream) {
+     entry_connection_t *entry = EDGE_TO_ENTRY_CONN(edge);
+     if (entry->socks_request) {
+       entry->socks_request->socks_extended_error_code =
+         SOCKS5_HS_REND_FAILED;
+     }
+  }
+}
+
+/** Called when introduction has failed meaning there is no more usable
+ * introduction points to be used (either NACKed or failed) for the given
+ * entry connection.
+ *
+ * This function only reports back the SOCKS5_HS_INTRO_FAILED (0xF2) code or
+ * SOCKS5_HS_INTRO_TIMEDOUT (0xF7) if all intros have timed out. The caller
+ * has to make sure to close the entry connections. */
+static void
+socks_mark_introduction_failed(entry_connection_t *conn,
+                                 const ed25519_public_key_t *identity_pk)
+{
+  socks5_reply_status_t code = SOCKS5_HS_INTRO_FAILED;
+
+  tor_assert(conn);
+  tor_assert(conn->socks_request);
+  tor_assert(identity_pk);
+
+  if (intro_points_all_timed_out(identity_pk)) {
+    code = SOCKS5_HS_INTRO_TIMEDOUT;
+  }
+  conn->socks_request->socks_extended_error_code = code;
+}
+
+/** For this introduction circuit, we'll look at if we have any usable
  * introduction point left for this service. If so, we'll use the circuit to
  * re-extend to a new intro point. Else, we'll close the circuit and its
  * corresponding rendezvous circuit. Return 0 if we are re-extending else -1
@@ -953,8 +1072,10 @@ close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
   tor_assert(intro_circ);
 
   desc = hs_cache_lookup_as_client(&intro_circ->hs_ident->identity_pk);
-  if (BUG(desc == NULL)) {
-    /* We can't continue without a descriptor. */
+  if (desc == NULL) {
+    /* We can't continue without a descriptor. This is possible if the cache
+     * was cleaned up between the intro point established and the reception of
+     * the introduce ack. */
     goto close;
   }
   /* We still have the descriptor, great! Let's try to see if we can
@@ -993,7 +1114,7 @@ close_or_reextend_intro_circ(origin_circuit_t *intro_circ)
   return ret;
 }
 
-/* Called when we get an INTRODUCE_ACK success status code. Do the appropriate
+/** Called when we get an INTRODUCE_ACK success status code. Do the appropriate
  * actions for the rendezvous point and finally close intro_circ. */
 static void
 handle_introduce_ack_success(origin_circuit_t *intro_circ)
@@ -1039,7 +1160,7 @@ handle_introduce_ack_success(origin_circuit_t *intro_circ)
   return;
 }
 
-/* Called when we get an INTRODUCE_ACK failure status code. Depending on our
+/** Called when we get an INTRODUCE_ACK failure status code. Depending on our
  * failure cache status, either close the circuit or re-extend to a new
  * introduction point. */
 static void
@@ -1061,7 +1182,7 @@ handle_introduce_ack_bad(origin_circuit_t *circ, int status)
                                    INTRO_POINT_FAILURE_GENERIC);
 }
 
-/* Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
+/** Called when we get an INTRODUCE_ACK on the intro circuit circ. The encoded
  * cell is in payload of length payload_len. Return 0 on success else a
  * negative value. The circuit is either close or reuse to re-extend to a new
  * introduction point. */
@@ -1100,7 +1221,7 @@ handle_introduce_ack(origin_circuit_t *circ, const uint8_t *payload,
   return ret;
 }
 
-/* Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
+/** Called when we get a RENDEZVOUS2 cell on the rendezvous circuit circ. The
  * encoded cell is in payload of length payload_len. Return 0 on success or a
  * negative value on error. On error, the circuit is marked for close. */
 STATIC int
@@ -1162,7 +1283,7 @@ handle_rendezvous2(origin_circuit_t *circ, const uint8_t *payload,
   return ret;
 }
 
-/* Return true iff the client can fetch a descriptor for this service public
+/** Return true iff the client can fetch a descriptor for this service public
  * identity key and status_out if not NULL is untouched. If the client can
  * _not_ fetch the descriptor and if status_out is not NULL, it is set with
  * the fetch status code. */
@@ -1230,7 +1351,27 @@ can_client_refetch_desc(const ed25519_public_key_t *identity_pk,
   return 0;
 }
 
-/* Return the client auth in the map using the service identity public key.
+/** Purge the client authorization cache of all ephemeral entries that is the
+ * entries that are not flagged with CLIENT_AUTH_FLAG_IS_PERMANENT.
+ *
+ * This is called from the hs_client_purge_state() used by a SIGNEWNYM. */
+STATIC void
+purge_ephemeral_client_auth(void)
+{
+  DIGEST256MAP_FOREACH_MODIFY(client_auths, key,
+                              hs_client_service_authorization_t *, auth) {
+    /* Cleanup every entry that are _NOT_ permanent that is ephemeral. */
+    if (!(auth->flags & CLIENT_AUTH_FLAG_IS_PERMANENT)) {
+      MAP_DEL_CURRENT(key);
+      client_service_authorization_free(auth);
+    }
+  } DIGESTMAP_FOREACH_END;
+
+  log_info(LD_REND, "Client onion service ephemeral authorization "
+                    "cache has been purged.");
+}
+
+/** Return the client auth in the map using the service identity public key.
  * Return NULL if it does not exist in the map. */
 static hs_client_service_authorization_t *
 find_client_auth(const ed25519_public_key_t *service_identity_pk)
@@ -1243,10 +1384,565 @@ find_client_auth(const ed25519_public_key_t *service_identity_pk)
   return digest256map_get(client_auths, service_identity_pk->pubkey);
 }
 
+/** This is called when a descriptor has arrived following a fetch request and
+ * has been stored in the client cache. The given entry connections, matching
+ * the service identity key, will get attached to the service circuit. */
+static void
+client_desc_has_arrived(const smartlist_t *entry_conns)
+{
+  time_t now = time(NULL);
+
+  tor_assert(entry_conns);
+
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
+    const hs_descriptor_t *desc;
+    edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
+    const ed25519_public_key_t *identity_pk =
+      &edge_conn->hs_ident->identity_pk;
+
+    /* We were just called because we stored the descriptor for this service
+     * so not finding a descriptor means we have a bigger problem. */
+    desc = hs_cache_lookup_as_client(identity_pk);
+    if (BUG(desc == NULL)) {
+      goto end;
+    }
+
+    if (!hs_client_any_intro_points_usable(identity_pk, desc)) {
+      log_info(LD_REND, "Hidden service descriptor is unusable. "
+                        "Closing streams.");
+      /* Report the extended socks error code that we were unable to introduce
+       * to the service. */
+      socks_mark_introduction_failed(entry_conn, identity_pk);
+
+      connection_mark_unattached_ap(entry_conn,
+                                    END_STREAM_REASON_RESOLVEFAILED);
+      /* We are unable to use the descriptor so remove the directory request
+       * from the cache so the next connection can try again. */
+      note_connection_attempt_succeeded(edge_conn->hs_ident);
+      continue;
+    }
+
+    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
+
+    /* Mark connection as waiting for a circuit since we do have a usable
+     * descriptor now. */
+    mark_conn_as_waiting_for_circuit(&edge_conn->base_, now);
+  } SMARTLIST_FOREACH_END(entry_conn);
+
+ end:
+  return;
+}
+
+/** This is called when a descriptor fetch was successful but the descriptor
+ * couldn't be decrypted due to missing or bad client authorization. */
+static void
+client_desc_missing_bad_client_auth(const smartlist_t *entry_conns,
+                                    hs_desc_decode_status_t status)
+{
+  tor_assert(entry_conns);
+
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
+    socks5_reply_status_t code;
+    if (status == HS_DESC_DECODE_BAD_CLIENT_AUTH) {
+      code = SOCKS5_HS_BAD_CLIENT_AUTH;
+    } else if (status == HS_DESC_DECODE_NEED_CLIENT_AUTH) {
+      code = SOCKS5_HS_MISSING_CLIENT_AUTH;
+    } else {
+      /* We should not be called with another type of status. Recover by
+       * sending a generic error. */
+      tor_assert_nonfatal_unreached();
+      code = SOCKS5_GENERAL_ERROR;
+    }
+    entry_conn->socks_request->socks_extended_error_code = code;
+    connection_mark_unattached_ap(entry_conn, END_STREAM_REASON_MISC);
+  } SMARTLIST_FOREACH_END(entry_conn);
+}
+
+/** Called when we get a 200 directory fetch status code. */
+static void
+client_dir_fetch_200(dir_connection_t *dir_conn,
+                     const smartlist_t *entry_conns, const char *body)
+{
+  hs_desc_decode_status_t decode_status;
+
+  tor_assert(dir_conn);
+  tor_assert(entry_conns);
+  tor_assert(body);
+
+  /* We got something: Try storing it in the cache. */
+  decode_status = hs_cache_store_as_client(body,
+                                           &dir_conn->hs_ident->identity_pk);
+  switch (decode_status) {
+  case HS_DESC_DECODE_OK:
+  case HS_DESC_DECODE_NEED_CLIENT_AUTH:
+  case HS_DESC_DECODE_BAD_CLIENT_AUTH:
+    log_info(LD_REND, "Stored hidden service descriptor successfully.");
+    TO_CONN(dir_conn)->purpose = DIR_PURPOSE_HAS_FETCHED_HSDESC;
+    if (decode_status == HS_DESC_DECODE_OK) {
+      client_desc_has_arrived(entry_conns);
+    } else {
+      /* This handles both client auth decode status. */
+      client_desc_missing_bad_client_auth(entry_conns, decode_status);
+      log_info(LD_REND, "Stored hidden service descriptor requires "
+                         "%s client authorization.",
+               decode_status == HS_DESC_DECODE_NEED_CLIENT_AUTH ? "missing"
+                                                                : "new");
+    }
+    /* Fire control port RECEIVED event. */
+    hs_control_desc_event_received(dir_conn->hs_ident,
+                                   dir_conn->identity_digest);
+    hs_control_desc_event_content(dir_conn->hs_ident,
+                                  dir_conn->identity_digest, body);
+    break;
+  case HS_DESC_DECODE_ENCRYPTED_ERROR:
+  case HS_DESC_DECODE_SUPERENC_ERROR:
+  case HS_DESC_DECODE_PLAINTEXT_ERROR:
+  case HS_DESC_DECODE_GENERIC_ERROR:
+  default:
+    log_info(LD_REND, "Failed to store hidden service descriptor. "
+                      "Descriptor decoding status: %d", decode_status);
+    /* Fire control port FAILED event. */
+    hs_control_desc_event_failed(dir_conn->hs_ident,
+                                 dir_conn->identity_digest, "BAD_DESC");
+    hs_control_desc_event_content(dir_conn->hs_ident,
+                                  dir_conn->identity_digest, NULL);
+    break;
+  }
+}
+
+/** Called when we get a 404 directory fetch status code. */
+static void
+client_dir_fetch_404(dir_connection_t *dir_conn,
+                     const smartlist_t *entry_conns)
+{
+  tor_assert(entry_conns);
+
+  /* Not there. We'll retry when connection_about_to_close_connection() tries
+   * to clean this conn up. */
+  log_info(LD_REND, "Fetching hidden service v3 descriptor not found: "
+                    "Retrying at another directory.");
+  /* Fire control port FAILED event. */
+  hs_control_desc_event_failed(dir_conn->hs_ident, dir_conn->identity_digest,
+                               "NOT_FOUND");
+  hs_control_desc_event_content(dir_conn->hs_ident, dir_conn->identity_digest,
+                                NULL);
+
+  /* Flag every entry connections that the descriptor was not found. */
+  SMARTLIST_FOREACH_BEGIN(entry_conns, entry_connection_t *, entry_conn) {
+    entry_conn->socks_request->socks_extended_error_code =
+      SOCKS5_HS_NOT_FOUND;
+  } SMARTLIST_FOREACH_END(entry_conn);
+}
+
+/** Called when we get a 400 directory fetch status code. */
+static void
+client_dir_fetch_400(dir_connection_t *dir_conn, const char *reason)
+{
+  tor_assert(dir_conn);
+
+  log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                    "http status 400 (%s). Dirserver didn't like our "
+                    "query? Retrying at another directory.",
+           escaped(reason));
+
+  /* Fire control port FAILED event. */
+  hs_control_desc_event_failed(dir_conn->hs_ident, dir_conn->identity_digest,
+                               "QUERY_REJECTED");
+  hs_control_desc_event_content(dir_conn->hs_ident, dir_conn->identity_digest,
+                                NULL);
+}
+
+/** Called when we get an unexpected directory fetch status code. */
+static void
+client_dir_fetch_unexpected(dir_connection_t *dir_conn, const char *reason,
+                            const int status_code)
+{
+  tor_assert(dir_conn);
+
+  log_warn(LD_REND, "Fetching v3 hidden service descriptor failed: "
+                    "http status %d (%s) response unexpected from HSDir "
+                    "server '%s:%d'. Retrying at another directory.",
+           status_code, escaped(reason), TO_CONN(dir_conn)->address,
+           TO_CONN(dir_conn)->port);
+  /* Fire control port FAILED event. */
+  hs_control_desc_event_failed(dir_conn->hs_ident, dir_conn->identity_digest,
+                               "UNEXPECTED");
+  hs_control_desc_event_content(dir_conn->hs_ident, dir_conn->identity_digest,
+                                NULL);
+}
+
+/** Get the full filename for storing the client auth credentials for the
+ *  service in <b>onion_address</b>. The base directory is <b>dir</b>.
+ *  This function never returns NULL. */
+static char *
+get_client_auth_creds_filename(const char *onion_address,
+                               const char *dir)
+{
+  char *full_fname = NULL;
+  char *fname;
+
+  tor_asprintf(&fname, "%s.auth_private", onion_address);
+  full_fname = hs_path_from_filename(dir, fname);
+  tor_free(fname);
+
+  return full_fname;
+}
+
+/** Permanently store the credentials in <b>creds</b> to disk.
+ *
+ *  Return -1 if there was an error while storing the credentials, otherwise
+ *  return 0.
+ */
+static int
+store_permanent_client_auth_credentials(
+                              const hs_client_service_authorization_t *creds)
+{
+  const or_options_t *options = get_options();
+  char *full_fname = NULL;
+  char *file_contents = NULL;
+  char priv_key_b32[BASE32_NOPAD_LEN(CURVE25519_PUBKEY_LEN)+1];
+  int retval = -1;
+
+  tor_assert(creds->flags & CLIENT_AUTH_FLAG_IS_PERMANENT);
+
+  /* We need ClientOnionAuthDir to be set, otherwise we can't proceed */
+  if (!options->ClientOnionAuthDir) {
+    log_warn(LD_GENERAL, "Can't register permanent client auth credentials "
+             "for %s without ClientOnionAuthDir option. Discarding.",
+             creds->onion_address);
+    goto err;
+  }
+
+  /* Make sure the directory exists and is private enough. */
+  if (check_private_dir(options->ClientOnionAuthDir, 0, options->User) < 0) {
+    goto err;
+  }
+
+  /* Get filename that we should store the credentials */
+  full_fname = get_client_auth_creds_filename(creds->onion_address,
+                                              options->ClientOnionAuthDir);
+
+  /* Encode client private key */
+  base32_encode(priv_key_b32, sizeof(priv_key_b32),
+                (char*)creds->enc_seckey.secret_key,
+                sizeof(creds->enc_seckey.secret_key));
+
+  /* Get the full file contents and write it to disk! */
+  tor_asprintf(&file_contents, "%s:descriptor:x25519:%s",
+               creds->onion_address, priv_key_b32);
+  if (write_str_to_file(full_fname, file_contents, 0) < 0) {
+    log_warn(LD_GENERAL, "Failed to write client auth creds file for %s!",
+             creds->onion_address);
+    goto err;
+  }
+
+  retval = 0;
+
+ err:
+  tor_free(file_contents);
+  tor_free(full_fname);
+
+  return retval;
+}
+
+/** Register the credential <b>creds</b> as part of the client auth subsystem.
+ *
+ * Takes ownership of <b>creds</b>.
+ **/
+hs_client_register_auth_status_t
+hs_client_register_auth_credentials(hs_client_service_authorization_t *creds)
+{
+  ed25519_public_key_t service_identity_pk;
+  hs_client_service_authorization_t *old_creds = NULL;
+  hs_client_register_auth_status_t retval = REGISTER_SUCCESS;
+
+  tor_assert(creds);
+
+  if (!client_auths) {
+    client_auths = digest256map_new();
+  }
+
+  if (hs_parse_address(creds->onion_address, &service_identity_pk,
+                       NULL, NULL) < 0) {
+    client_service_authorization_free(creds);
+    return REGISTER_FAIL_BAD_ADDRESS;
+  }
+
+  /* If we reach this point, the credentials will be stored one way or another:
+   * Make them permanent if the user asked us to. */
+  if (creds->flags & CLIENT_AUTH_FLAG_IS_PERMANENT) {
+    if (store_permanent_client_auth_credentials(creds) < 0) {
+      client_service_authorization_free(creds);
+      return REGISTER_FAIL_PERMANENT_STORAGE;
+    }
+  }
+
+  old_creds = digest256map_get(client_auths, service_identity_pk.pubkey);
+  if (old_creds) {
+    digest256map_remove(client_auths, service_identity_pk.pubkey);
+    client_service_authorization_free(old_creds);
+    retval = REGISTER_SUCCESS_ALREADY_EXISTS;
+  }
+
+  digest256map_set(client_auths, service_identity_pk.pubkey, creds);
+
+  /** Now that we set the new credentials, also try to decrypt any cached
+   *  descriptors. */
+  if (hs_cache_client_new_auth_parse(&service_identity_pk)) {
+    retval = REGISTER_SUCCESS_AND_DECRYPTED;
+  }
+
+  return retval;
+}
+
+/** Load a client authorization file with <b>filename</b> that is stored under
+ *  the global client auth directory, and return a newly-allocated credentials
+ *  object if it parsed well. Otherwise, return NULL.
+ */
+static hs_client_service_authorization_t *
+get_creds_from_client_auth_filename(const char *filename,
+                                    const or_options_t *options)
+{
+  hs_client_service_authorization_t *auth = NULL;
+  char *client_key_file_path = NULL;
+  char *client_key_str = NULL;
+
+  log_info(LD_REND, "Loading a client authorization key file %s...",
+           filename);
+
+  if (!auth_key_filename_is_valid(filename)) {
+    log_notice(LD_REND, "Client authorization unrecognized filename %s. "
+               "File must end in .auth_private. Ignoring.",
+               filename);
+    goto err;
+  }
+
+  /* Create a full path for a file. */
+  client_key_file_path = hs_path_from_filename(options->ClientOnionAuthDir,
+                                               filename);
+
+  client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
+  if (!client_key_str) {
+    log_warn(LD_REND, "The file %s cannot be read.", filename);
+    goto err;
+  }
+
+  auth = parse_auth_file_content(client_key_str);
+  if (!auth) {
+    goto err;
+  }
+
+ err:
+  tor_free(client_key_str);
+  tor_free(client_key_file_path);
+
+  return auth;
+}
+
+/*
+ * Remove the file in <b>filename</b> under the global client auth credential
+ * storage.
+ */
+static void
+remove_client_auth_creds_file(const char *filename)
+{
+  char *creds_file_path = NULL;
+  const or_options_t *options = get_options();
+
+  creds_file_path = hs_path_from_filename(options->ClientOnionAuthDir,
+                                          filename);
+  if (tor_unlink(creds_file_path) != 0) {
+    log_warn(LD_REND, "Failed to remove client auth file (%s).",
+             creds_file_path);
+    goto end;
+  }
+
+  log_warn(LD_REND, "Successfuly removed client auth file (%s).",
+           creds_file_path);
+
+ end:
+  tor_free(creds_file_path);
+}
+
+/**
+ * Find the filesystem file corresponding to the permanent client auth
+ * credentials in <b>cred</b> and remove it.
+ */
+static void
+find_and_remove_client_auth_creds_file(
+                                 const hs_client_service_authorization_t *cred)
+{
+  smartlist_t *file_list = NULL;
+  const or_options_t *options = get_options();
+
+  tor_assert(cred->flags & CLIENT_AUTH_FLAG_IS_PERMANENT);
+
+  if (!options->ClientOnionAuthDir) {
+    log_warn(LD_REND, "Found permanent credential but no ClientOnionAuthDir "
+             "configured. There is no file to be removed.");
+    goto end;
+  }
+
+  file_list = tor_listdir(options->ClientOnionAuthDir);
+  if (file_list == NULL) {
+    log_warn(LD_REND, "Client authorization key directory %s can't be listed.",
+             options->ClientOnionAuthDir);
+    goto end;
+  }
+
+  SMARTLIST_FOREACH_BEGIN(file_list, const char *, filename) {
+    hs_client_service_authorization_t *tmp_cred = NULL;
+
+    tmp_cred = get_creds_from_client_auth_filename(filename, options);
+    if (!tmp_cred) {
+      continue;
+    }
+
+    /* Find the right file for this credential */
+    if (!strcmp(tmp_cred->onion_address, cred->onion_address)) {
+      /* Found it! Remove the file! */
+      remove_client_auth_creds_file(filename);
+      /* cleanup and get out of here */
+      client_service_authorization_free(tmp_cred);
+      break;
+    }
+
+    client_service_authorization_free(tmp_cred);
+  } SMARTLIST_FOREACH_END(filename);
+
+ end:
+  if (file_list) {
+    SMARTLIST_FOREACH(file_list, char *, s, tor_free(s));
+    smartlist_free(file_list);
+  }
+}
+
+/** Remove client auth credentials for the service <b>hs_address</b>. */
+hs_client_removal_auth_status_t
+hs_client_remove_auth_credentials(const char *hsaddress)
+{
+  ed25519_public_key_t service_identity_pk;
+
+  if (!client_auths) {
+    return REMOVAL_SUCCESS_NOT_FOUND;
+  }
+
+  if (hs_parse_address(hsaddress, &service_identity_pk, NULL, NULL) < 0) {
+    return REMOVAL_BAD_ADDRESS;
+  }
+
+  hs_client_service_authorization_t *cred = NULL;
+  cred = digest256map_remove(client_auths, service_identity_pk.pubkey);
+
+  /* digestmap_remove() returns the previously stored data if there were any */
+  if (cred) {
+    if (cred->flags & CLIENT_AUTH_FLAG_IS_PERMANENT) {
+      /* These creds are stored on disk: remove the corresponding file. */
+      find_and_remove_client_auth_creds_file(cred);
+    }
+
+    /* Remove associated descriptor if any. */
+    hs_cache_remove_as_client(&service_identity_pk);
+
+    client_service_authorization_free(cred);
+    return REMOVAL_SUCCESS;
+  }
+
+  return REMOVAL_SUCCESS_NOT_FOUND;
+}
+
+/** Get the HS client auth map. */
+digest256map_t *
+get_hs_client_auths_map(void)
+{
+  return client_auths;
+}
+
 /* ========== */
 /* Public API */
 /* ========== */
 
+/** Called when a circuit was just cleaned up. This is done right before the
+ * circuit is marked for close. */
+void
+hs_client_circuit_cleanup_on_close(const circuit_t *circ)
+{
+  bool has_timed_out;
+
+  tor_assert(circ);
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+
+  has_timed_out =
+    (circ->marked_for_close_orig_reason == END_CIRC_REASON_TIMEOUT);
+
+  switch (circ->purpose) {
+  case CIRCUIT_PURPOSE_C_ESTABLISH_REND:
+  case CIRCUIT_PURPOSE_C_REND_READY:
+  case CIRCUIT_PURPOSE_C_REND_READY_INTRO_ACKED:
+  case CIRCUIT_PURPOSE_C_REND_JOINED:
+    /* Report extended SOCKS error code when a rendezvous circuit times out.
+     * This MUST be done on_close() because it is possible the entry
+     * connection would get closed before the circuit is freed and thus
+     * would fail to report the error code. */
+    if (has_timed_out) {
+      socks_mark_rend_circuit_timed_out(CONST_TO_ORIGIN_CIRCUIT(circ));
+    }
+    break;
+  default:
+    break;
+  }
+}
+
+/** Called when a circuit was just cleaned up. This is done right before the
+ * circuit is freed. */
+void
+hs_client_circuit_cleanup_on_free(const circuit_t *circ)
+{
+  bool has_timed_out;
+  rend_intro_point_failure_t failure = INTRO_POINT_FAILURE_GENERIC;
+  const origin_circuit_t *orig_circ = NULL;
+
+  tor_assert(circ);
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+
+  orig_circ = CONST_TO_ORIGIN_CIRCUIT(circ);
+  tor_assert(orig_circ->hs_ident);
+
+  has_timed_out =
+    (circ->marked_for_close_orig_reason == END_CIRC_REASON_TIMEOUT);
+  if (has_timed_out) {
+    failure = INTRO_POINT_FAILURE_TIMEOUT;
+  }
+
+  switch (circ->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
+    log_info(LD_REND, "Failed v3 intro circ for service %s to intro point %s "
+                      "(awaiting ACK). Failure code: %d",
+        safe_str_client(ed25519_fmt(&orig_circ->hs_ident->identity_pk)),
+        safe_str_client(build_state_get_exit_nickname(orig_circ->build_state)),
+        failure);
+    hs_cache_client_intro_state_note(&orig_circ->hs_ident->identity_pk,
+                                     &orig_circ->hs_ident->intro_auth_pk,
+                                     failure);
+    break;
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+    if (has_timed_out || !orig_circ->build_state) {
+      break;
+    }
+    failure = INTRO_POINT_FAILURE_UNREACHABLE;
+    log_info(LD_REND, "Failed v3 intro circ for service %s to intro point %s "
+                      "(while building circuit). Marking as unreachable.",
+       safe_str_client(ed25519_fmt(&orig_circ->hs_ident->identity_pk)),
+       safe_str_client(build_state_get_exit_nickname(orig_circ->build_state)));
+    hs_cache_client_intro_state_note(&orig_circ->hs_ident->identity_pk,
+                                     &orig_circ->hs_ident->intro_auth_pk,
+                                     failure);
+    break;
+  default:
+    break;
+  }
+}
+
 /** A circuit just finished connecting to a hidden service that the stream
  *  <b>conn</b> has been waiting for. Let the HS subsystem know about this. */
 void
@@ -1268,18 +1964,20 @@ hs_client_note_connection_attempt_succeeded(const edge_connection_t *conn)
   }
 }
 
-/* With the given encoded descriptor in desc_str and the service key in
+/** With the given encoded descriptor in desc_str and the service key in
  * service_identity_pk, decode the descriptor and set the desc pointer with a
  * newly allocated descriptor object.
  *
- * Return 0 on success else a negative value and desc is set to NULL. */
-int
+ * On success, HS_DESC_DECODE_OK is returned and desc is set to the decoded
+ * descriptor. On error, desc is set to NULL and a decoding error status is
+ * returned depending on what was the issue. */
+hs_desc_decode_status_t
 hs_client_decode_descriptor(const char *desc_str,
                             const ed25519_public_key_t *service_identity_pk,
                             hs_descriptor_t **desc)
 {
-  int ret;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_desc_decode_status_t ret;
+  hs_subcredential_t subcredential;
   ed25519_public_key_t blinded_pubkey;
   hs_client_service_authorization_t *client_auth = NULL;
   curve25519_secret_key_t *client_auth_sk = NULL;
@@ -1299,14 +1997,14 @@ hs_client_decode_descriptor(const char *desc_str,
     uint64_t current_time_period = hs_get_time_period_num(0);
     hs_build_blinded_pubkey(service_identity_pk, NULL, 0, current_time_period,
                             &blinded_pubkey);
-    hs_get_subcredential(service_identity_pk, &blinded_pubkey, subcredential);
+    hs_get_subcredential(service_identity_pk, &blinded_pubkey, &subcredential);
   }
 
   /* Parse descriptor */
-  ret = hs_desc_decode_descriptor(desc_str, subcredential,
+  ret = hs_desc_decode_descriptor(desc_str, &subcredential,
                                   client_auth_sk, desc);
-  memwipe(subcredential, 0, sizeof(subcredential));
-  if (ret < 0) {
+  memwipe(&subcredential, 0, sizeof(subcredential));
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
@@ -1319,15 +2017,16 @@ hs_client_decode_descriptor(const char *desc_str,
     log_warn(LD_GENERAL, "Descriptor signing key certificate signature "
              "doesn't validate with computed blinded key: %s",
              tor_cert_describe_signature_status(cert));
+    ret = HS_DESC_DECODE_GENERIC_ERROR;
     goto err;
   }
 
-  return 0;
+  return HS_DESC_DECODE_OK;
  err:
-  return -1;
+  return ret;
 }
 
-/* Return true iff there are at least one usable intro point in the service
+/** Return true iff there are at least one usable intro point in the service
  * descriptor desc. */
 int
 hs_client_any_intro_points_usable(const ed25519_public_key_t *service_pk,
@@ -1376,7 +2075,7 @@ hs_client_refetch_hsdesc(const ed25519_public_key_t *identity_pk)
   return status;
 }
 
-/* This is called when we are trying to attach an AP connection to these
+/** This is called when we are trying to attach an AP connection to these
  * hidden service circuits from connection_ap_handshake_attach_circuit().
  * Return 0 on success, -1 for a transient error that is actions were
  * triggered to recover or -2 for a permenent error where both circuits will
@@ -1392,7 +2091,7 @@ hs_client_send_introduce1(origin_circuit_t *intro_circ,
                                                                 rend_circ);
 }
 
-/* Called when the client circuit circ has been established. It can be either
+/** Called when the client circuit circ has been established. It can be either
  * an introduction or rendezvous circuit. This function handles all hidden
  * service versions. */
 void
@@ -1422,7 +2121,7 @@ hs_client_circuit_has_opened(origin_circuit_t *circ)
   }
 }
 
-/* Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
+/** Called when we receive a RENDEZVOUS_ESTABLISHED cell. Change the state of
  * the circuit to CIRCUIT_PURPOSE_C_REND_READY. Return 0 on success else a
  * negative value and the circuit marked for close. */
 int
@@ -1464,16 +2163,16 @@ hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
   return -1;
 }
 
-#define client_service_authorization_free(auth)                      \
-  FREE_AND_NULL(hs_client_service_authorization_t,                   \
-                client_service_authorization_free_, (auth))
-
-static void
+void
 client_service_authorization_free_(hs_client_service_authorization_t *auth)
 {
-  if (auth) {
-    memwipe(auth, 0, sizeof(*auth));
+  if (!auth) {
+    return;
   }
+
+  tor_free(auth->client_name);
+
+  memwipe(auth, 0, sizeof(*auth));
   tor_free(auth);
 }
 
@@ -1493,7 +2192,7 @@ client_service_authorization_free_all(void)
   digest256map_free(client_auths, client_service_authorization_free_void);
 }
 
-/* Check if the auth key file name is valid or not. Return 1 if valid,
+/** Check if the auth key file name is valid or not. Return 1 if valid,
  * otherwise return 0. */
 STATIC int
 auth_key_filename_is_valid(const char *filename)
@@ -1515,6 +2214,13 @@ auth_key_filename_is_valid(const char *filename)
   return ret;
 }
 
+/** Parse the client auth credentials off a string in <b>client_key_str</b>
+ *  based on the file format documented in the "Client side configuration"
+ *  section of rend-spec-v3.txt.
+ *
+ *  Return NULL if there was an error, otherwise return a newly allocated
+ *  hs_client_service_authorization_t structure.
+ */
 STATIC hs_client_service_authorization_t *
 parse_auth_file_content(const char *client_key_str)
 {
@@ -1545,7 +2251,7 @@ parse_auth_file_content(const char *client_key_str)
     goto err;
   }
 
-  if (strlen(seckey_b32) != BASE32_NOPAD_LEN(CURVE25519_PUBKEY_LEN)) {
+  if (strlen(seckey_b32) != BASE32_NOPAD_LEN(CURVE25519_SECKEY_LEN)) {
     log_warn(LD_REND, "Client authorization encoded base32 private key "
                       "length is invalid: %s", seckey_b32);
     goto err;
@@ -1554,11 +2260,24 @@ parse_auth_file_content(const char *client_key_str)
   auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
   if (base32_decode((char *) auth->enc_seckey.secret_key,
                     sizeof(auth->enc_seckey.secret_key),
-                    seckey_b32, strlen(seckey_b32)) < 0) {
+                    seckey_b32, strlen(seckey_b32)) !=
+      sizeof(auth->enc_seckey.secret_key)) {
+    log_warn(LD_REND, "Client authorization encoded base32 private key "
+                      "can't be decoded: %s", seckey_b32);
     goto err;
   }
+
+  if (fast_mem_is_zero((const char*)auth->enc_seckey.secret_key,
+                       sizeof(auth->enc_seckey.secret_key))) {
+    log_warn(LD_REND, "Client authorization private key can't be all-zeroes");
+    goto err;
+  }
+
   strncpy(auth->onion_address, onion_address, HS_SERVICE_ADDR_LEN_BASE32);
 
+  /* We are reading this from the disk, so set the permanent flag anyway. */
+  auth->flags |= CLIENT_AUTH_FLAG_IS_PERMANENT;
+
   /* Success. */
   goto done;
 
@@ -1575,7 +2294,7 @@ parse_auth_file_content(const char *client_key_str)
   return auth;
 }
 
-/* From a set of <b>options</b>, setup every client authorization detail
+/** From a set of <b>options</b>, setup every client authorization detail
  * found. Return 0 on success or -1 on failure. If <b>validate_only</b>
  * is set, parse, warn and return as normal, but don't actually change
  * the configuration. */
@@ -1585,10 +2304,7 @@ hs_config_client_authorization(const or_options_t *options,
 {
   int ret = -1;
   digest256map_t *auths = digest256map_new();
-  char *key_dir = NULL;
   smartlist_t *file_list = NULL;
-  char *client_key_str = NULL;
-  char *client_key_file_path = NULL;
 
   tor_assert(options);
 
@@ -1599,82 +2315,54 @@ hs_config_client_authorization(const or_options_t *options,
     goto end;
   }
 
-  key_dir = tor_strdup(options->ClientOnionAuthDir);
-
   /* Make sure the directory exists and is private enough. */
-  if (check_private_dir(key_dir, 0, options->User) < 0) {
+  if (check_private_dir(options->ClientOnionAuthDir, 0, options->User) < 0) {
     goto end;
   }
 
-  file_list = tor_listdir(key_dir);
+  file_list = tor_listdir(options->ClientOnionAuthDir);
   if (file_list == NULL) {
     log_warn(LD_REND, "Client authorization key directory %s can't be listed.",
-             key_dir);
+             options->ClientOnionAuthDir);
     goto end;
   }
 
-  SMARTLIST_FOREACH_BEGIN(file_list, char *, filename) {
-
+  SMARTLIST_FOREACH_BEGIN(file_list, const char *, filename) {
     hs_client_service_authorization_t *auth = NULL;
     ed25519_public_key_t identity_pk;
-    log_info(LD_REND, "Loading a client authorization key file %s...",
-             filename);
 
-    if (!auth_key_filename_is_valid(filename)) {
-      log_notice(LD_REND, "Client authorization unrecognized filename %s. "
-                          "File must end in .auth_private. Ignoring.",
-                 filename);
+    auth = get_creds_from_client_auth_filename(filename, options);
+    if (!auth) {
       continue;
     }
 
-    /* Create a full path for a file. */
-    client_key_file_path = hs_path_from_filename(key_dir, filename);
-    client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
-    /* Free the file path immediately after using it. */
-    tor_free(client_key_file_path);
-
-    /* If we cannot read the file, continue with the next file. */
-    if (!client_key_str) {
-      log_warn(LD_REND, "The file %s cannot be read.", filename);
+    /* Parse the onion address to get an identity public key and use it
+     * as a key of global map in the future. */
+    if (hs_parse_address(auth->onion_address, &identity_pk,
+                         NULL, NULL) < 0) {
+      log_warn(LD_REND, "The onion address \"%s\" is invalid in "
+               "file %s", filename, auth->onion_address);
+      client_service_authorization_free(auth);
       continue;
     }
 
-    auth = parse_auth_file_content(client_key_str);
-    /* Free immediately after using it. */
-    tor_free(client_key_str);
-
-    if (auth) {
-      /* Parse the onion address to get an identity public key and use it
-       * as a key of global map in the future. */
-      if (hs_parse_address(auth->onion_address, &identity_pk,
-                           NULL, NULL) < 0) {
-        log_warn(LD_REND, "The onion address \"%s\" is invalid in "
-                          "file %s", filename, auth->onion_address);
-        client_service_authorization_free(auth);
-        continue;
-      }
-
-      if (digest256map_get(auths, identity_pk.pubkey)) {
+    if (digest256map_get(auths, identity_pk.pubkey)) {
         log_warn(LD_REND, "Duplicate authorization for the same hidden "
-                          "service address %s.",
+                 "service address %s.",
                  safe_str_client_opts(options, auth->onion_address));
         client_service_authorization_free(auth);
         goto end;
-      }
-
-      digest256map_set(auths, identity_pk.pubkey, auth);
-      log_info(LD_REND, "Loaded a client authorization key file %s.",
-               filename);
     }
+
+    digest256map_set(auths, identity_pk.pubkey, auth);
+    log_info(LD_REND, "Loaded a client authorization key file %s.",
+             filename);
   } SMARTLIST_FOREACH_END(filename);
 
   /* Success. */
   ret = 0;
 
  end:
-  tor_free(key_dir);
-  tor_free(client_key_str);
-  tor_free(client_key_file_path);
   if (file_list) {
     SMARTLIST_FOREACH(file_list, char *, s, tor_free(s));
     smartlist_free(file_list);
@@ -1690,65 +2378,48 @@ hs_config_client_authorization(const or_options_t *options,
   return ret;
 }
 
-/* This is called when a descriptor has arrived following a fetch request and
- * has been stored in the client cache. Every entry connection that matches
- * the service identity key in the ident will get attached to the hidden
- * service circuit. */
+/** Called when a descriptor directory fetch is done.
+ *
+ * Act accordingly on all entry connections depending on the HTTP status code
+ * we got. In case of an error, the SOCKS error is set (if ExtendedErrors is
+ * set).
+ *
+ * The reason is a human readable string returned by the directory server
+ * which can describe the status of the request. The body is the response
+ * content, on 200 code it is the descriptor itself. Finally, the status_code
+ * is the HTTP code returned by the directory server. */
 void
-hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident)
+hs_client_dir_fetch_done(dir_connection_t *dir_conn, const char *reason,
+                         const char *body, const int status_code)
 {
-  time_t now = time(NULL);
-  smartlist_t *conns = NULL;
+  smartlist_t *entry_conns;
 
-  tor_assert(ident);
+  tor_assert(dir_conn);
+  tor_assert(body);
 
-  conns = connection_list_by_type_state(CONN_TYPE_AP,
-                                        AP_CONN_STATE_RENDDESC_WAIT);
-  SMARTLIST_FOREACH_BEGIN(conns, connection_t *, base_conn) {
-    const hs_descriptor_t *desc;
-    entry_connection_t *entry_conn = TO_ENTRY_CONN(base_conn);
-    const edge_connection_t *edge_conn = ENTRY_TO_EDGE_CONN(entry_conn);
-
-    /* Only consider the entry connections that matches the service for which
-     * we just fetched its descriptor. */
-    if (!edge_conn->hs_ident ||
-        !ed25519_pubkey_eq(&ident->identity_pk,
-                           &edge_conn->hs_ident->identity_pk)) {
-      continue;
-    }
-    assert_connection_ok(base_conn, now);
-
-    /* We were just called because we stored the descriptor for this service
-     * so not finding a descriptor means we have a bigger problem. */
-    desc = hs_cache_lookup_as_client(&ident->identity_pk);
-    if (BUG(desc == NULL)) {
-      goto end;
-    }
+  /* Get all related entry connections. */
+  entry_conns = find_entry_conns(&dir_conn->hs_ident->identity_pk);
 
-    if (!hs_client_any_intro_points_usable(&ident->identity_pk, desc)) {
-      log_info(LD_REND, "Hidden service descriptor is unusable. "
-                        "Closing streams.");
-      connection_mark_unattached_ap(entry_conn,
-                                    END_STREAM_REASON_RESOLVEFAILED);
-      /* We are unable to use the descriptor so remove the directory request
-       * from the cache so the next connection can try again. */
-      note_connection_attempt_succeeded(edge_conn->hs_ident);
-      continue;
-    }
-
-    log_info(LD_REND, "Descriptor has arrived. Launching circuits.");
-
-    /* Mark connection as waiting for a circuit since we do have a usable
-     * descriptor now. */
-    mark_conn_as_waiting_for_circuit(base_conn, now);
-  } SMARTLIST_FOREACH_END(base_conn);
+  switch (status_code) {
+  case 200:
+    client_dir_fetch_200(dir_conn, entry_conns, body);
+    break;
+  case 404:
+    client_dir_fetch_404(dir_conn, entry_conns);
+    break;
+  case 400:
+    client_dir_fetch_400(dir_conn, reason);
+    break;
+  default:
+    client_dir_fetch_unexpected(dir_conn, reason, status_code);
+    break;
+  }
 
- end:
   /* We don't have ownership of the objects in this list. */
-  smartlist_free(conns);
+  smartlist_free(entry_conns);
 }
 
-/* Return a newly allocated extend_info_t for a randomly chosen introduction
+/** Return a newly allocated extend_info_t for a randomly chosen introduction
  * point for the given edge connection identifier ident. Return NULL if we
  * can't pick any usable introduction points. */
 extend_info_t *
@@ -1761,7 +2432,7 @@ hs_client_get_random_intro_from_edge(const edge_connection_t *edge_conn)
     rend_client_get_random_intro(edge_conn->rend_data);
 }
 
-/* Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
+/** Called when get an INTRODUCE_ACK cell on the introduction circuit circ.
  * Return 0 on success else a negative value is returned. The circuit will be
  * closed or reuse to extend again to another intro point. */
 int
@@ -1790,7 +2461,7 @@ hs_client_receive_introduce_ack(origin_circuit_t *circ,
   return ret;
 }
 
-/* Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
+/** Called when get a RENDEZVOUS2 cell on the rendezvous circuit circ.  Return
  * 0 on success else a negative value is returned. The circuit will be closed
  * on error. */
 int
@@ -1823,7 +2494,7 @@ hs_client_receive_rendezvous2(origin_circuit_t *circ,
   return ret;
 }
 
-/* Extend the introduction circuit circ to another valid introduction point
+/** Extend the introduction circuit circ to another valid introduction point
  * for the hidden service it is trying to connect to, or mark it and launch a
  * new circuit if we can't extend it.  Return 0 on success or possible
  * success. Return -1 and mark the introduction circuit for close on permanent
@@ -1873,7 +2544,7 @@ hs_client_reextend_intro_circuit(origin_circuit_t *circ)
   return ret;
 }
 
-/* Close all client introduction circuits related to the given descriptor.
+/** Close all client introduction circuits related to the given descriptor.
  * This is called with a descriptor that is about to get replaced in the
  * client cache.
  *
@@ -1905,7 +2576,7 @@ hs_client_close_intro_circuits_from_desc(const hs_descriptor_t *desc)
   }
 }
 
-/* Release all the storage held by the client subsystem. */
+/** Release all the storage held by the client subsystem. */
 void
 hs_client_free_all(void)
 {
@@ -1914,7 +2585,7 @@ hs_client_free_all(void)
   client_service_authorization_free_all();
 }
 
-/* Purge all potentially remotely-detectable state held in the hidden
+/** Purge all potentially remotely-detectable state held in the hidden
  * service client code. Called on SIGNAL NEWNYM. */
 void
 hs_client_purge_state(void)
@@ -1931,11 +2602,13 @@ hs_client_purge_state(void)
   hs_cache_purge_as_client();
   /* Purge the last hidden service request cache. */
   hs_purge_last_hid_serv_requests();
+  /* Purge ephemeral client authorization. */
+  purge_ephemeral_client_auth();
 
   log_info(LD_REND, "Hidden service client state has been purged.");
 }
 
-/* Called when our directory information has changed. */
+/** Called when our directory information has changed. */
 void
 hs_client_dir_info_changed(void)
 {
@@ -1947,10 +2620,10 @@ hs_client_dir_info_changed(void)
 
 #ifdef TOR_UNIT_TESTS
 
-STATIC digest256map_t *
-get_hs_client_auths_map(void)
+STATIC void
+set_hs_client_auths_map(digest256map_t *map)
 {
-  return client_auths;
+  client_auths = map;
 }
 
 #endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/feature/hs/hs_client.h b/src/feature/hs/hs_client.h
index dadfa024b8..88dede8126 100644
--- a/src/feature/hs/hs_client.h
+++ b/src/feature/hs/hs_client.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,41 +10,97 @@
 #define TOR_HS_CLIENT_H
 
 #include "lib/crypt_ops/crypto_ed25519.h"
+
+#include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_descriptor.h"
 #include "feature/hs/hs_ident.h"
 
-/* Status code of a descriptor fetch request. */
+/** Status code of a descriptor fetch request. */
 typedef enum {
-  /* Something internally went wrong. */
+  /** Something internally went wrong. */
   HS_CLIENT_FETCH_ERROR        = -1,
-  /* The fetch request has been launched successfully. */
+  /** The fetch request has been launched successfully. */
   HS_CLIENT_FETCH_LAUNCHED     = 0,
-  /* We already have a usable descriptor. No fetch. */
+  /** We already have a usable descriptor. No fetch. */
   HS_CLIENT_FETCH_HAVE_DESC    = 1,
-  /* No more HSDir available to query. */
+  /** No more HSDir available to query. */
   HS_CLIENT_FETCH_NO_HSDIRS    = 2,
-  /* The fetch request is not allowed. */
+  /** The fetch request is not allowed. */
   HS_CLIENT_FETCH_NOT_ALLOWED  = 3,
-  /* We are missing information to be able to launch a request. */
+  /** We are missing information to be able to launch a request. */
   HS_CLIENT_FETCH_MISSING_INFO = 4,
-  /* There is a pending fetch for the requested service. */
+  /** There is a pending fetch for the requested service. */
   HS_CLIENT_FETCH_PENDING      = 5,
 } hs_client_fetch_status_t;
 
-/** Client-side configuration of authorization for a service. */
+/* Status code of client auth credential registration */
+typedef enum {
+  /* We successfuly registered these credentials */
+  REGISTER_SUCCESS,
+  /* We successfully registered these credentials, but had to replace some
+   * existing ones. */
+  REGISTER_SUCCESS_ALREADY_EXISTS,
+  /* We successfuly registered these credentials, and also decrypted a cached
+   * descriptor. */
+  REGISTER_SUCCESS_AND_DECRYPTED,
+  /* We failed to register these credentials, because of a bad HS address. */
+  REGISTER_FAIL_BAD_ADDRESS,
+  /* We failed to store these credentials in a persistent file on disk. */
+  REGISTER_FAIL_PERMANENT_STORAGE,
+} hs_client_register_auth_status_t;
+
+/* Status code of client auth credential removal */
+typedef enum {
+  /* We successfuly removed these credentials */
+  REMOVAL_SUCCESS,
+  /* No need to remove those credentials, because they were not there. */
+  REMOVAL_SUCCESS_NOT_FOUND,
+  /* We failed to register these credentials, because of a bad HS address. */
+  REMOVAL_BAD_ADDRESS,
+} hs_client_removal_auth_status_t;
+
+/** Flag to set when a client auth is permanent (saved on disk). */
+#define CLIENT_AUTH_FLAG_IS_PERMANENT (1<<0)
+
+/** Client-side configuration of client authorization */
 typedef struct hs_client_service_authorization_t {
-  /* An curve25519 secret key used to compute decryption keys that
+  /** An curve25519 secret key used to compute decryption keys that
    * allow the client to decrypt the hidden service descriptor. */
   curve25519_secret_key_t enc_seckey;
 
-  /* An onion address that is used to connect to the onion service. */
+  /** An onion address that is used to connect to the onion service. */
   char onion_address[HS_SERVICE_ADDR_LEN_BASE32+1];
+
+  /** An client name used to connect to the onion service. */
+  char *client_name;
+
+  /* Optional flags for this client. */
+  int flags;
 } hs_client_service_authorization_t;
 
+hs_client_register_auth_status_t
+hs_client_register_auth_credentials(hs_client_service_authorization_t *creds);
+
+hs_client_removal_auth_status_t
+hs_client_remove_auth_credentials(const char *hsaddress);
+
+digest256map_t *get_hs_client_auths_map(void);
+
+#define client_service_authorization_free(auth)                      \
+  FREE_AND_NULL(hs_client_service_authorization_t,                   \
+                client_service_authorization_free_, (auth))
+
+void
+client_service_authorization_free_(hs_client_service_authorization_t *auth);
+
 void hs_client_note_connection_attempt_succeeded(
                                        const edge_connection_t *conn);
 
-int hs_client_decode_descriptor(
+void hs_client_launch_v3_desc_fetch(
+                               const ed25519_public_key_t *onion_identity_pk,
+                               const smartlist_t *hsdirs);
+
+hs_desc_decode_status_t hs_client_decode_descriptor(
                      const char *desc_str,
                      const ed25519_public_key_t *service_identity_pk,
                      hs_descriptor_t **desc);
@@ -57,6 +113,8 @@ int hs_client_send_introduce1(origin_circuit_t *intro_circ,
                               origin_circuit_t *rend_circ);
 
 void hs_client_circuit_has_opened(origin_circuit_t *circ);
+void hs_client_circuit_cleanup_on_close(const circuit_t *circ);
+void hs_client_circuit_cleanup_on_free(const circuit_t *circ);
 
 int hs_client_receive_rendezvous_acked(origin_circuit_t *circ,
                                        const uint8_t *payload,
@@ -68,7 +126,8 @@ int hs_client_receive_rendezvous2(origin_circuit_t *circ,
                                   const uint8_t *payload,
                                   size_t payload_len);
 
-void hs_client_desc_has_arrived(const hs_ident_dir_conn_t *ident);
+void hs_client_dir_fetch_done(dir_connection_t *dir_conn, const char *reason,
+                              const char *body, const int status_code);
 
 extend_info_t *hs_client_get_random_intro_from_edge(
                                           const edge_connection_t *edge_conn);
@@ -107,13 +166,14 @@ MOCK_DECL(STATIC hs_client_fetch_status_t,
 
 STATIC void retry_all_socks_conn_waiting_for_desc(void);
 
+STATIC void purge_ephemeral_client_auth(void);
+
 #ifdef TOR_UNIT_TESTS
 
-STATIC digest256map_t *get_hs_client_auths_map(void);
+STATIC void set_hs_client_auths_map(digest256map_t *map);
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
 #endif /* defined(HS_CLIENT_PRIVATE) */
 
 #endif /* !defined(TOR_HS_CLIENT_H) */
-
diff --git a/src/feature/hs/hs_common.c b/src/feature/hs/hs_common.c
index de653037d1..86d3fcab7d 100644
--- a/src/feature/hs/hs_common.c
+++ b/src/feature/hs/hs_common.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,6 +21,8 @@
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_dos.h"
+#include "feature/hs/hs_ob.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_service.h"
 #include "feature/hs_common/shared_random_client.h"
@@ -31,6 +33,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendservice.h"
+#include "feature/relay/routermode.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
@@ -43,7 +46,7 @@
 /* Trunnel */
 #include "trunnel/ed25519_cert.h"
 
-/* Ed25519 Basepoint value. Taken from section 5 of
+/** Ed25519 Basepoint value. Taken from section 5 of
  * https://tools.ietf.org/html/draft-josefsson-eddsa-ed25519-03 */
 static const char *str_ed25519_basepoint =
   "(15112221349535400772501151409588531511"
@@ -85,7 +88,7 @@ set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
   return 0;
 }
 
-#else /* !(defined(HAVE_SYS_UN_H)) */
+#else /* !defined(HAVE_SYS_UN_H) */
 
 static int
 set_unix_port(edge_connection_t *conn, rend_service_port_config_t *p)
@@ -105,7 +108,7 @@ add_unix_port(smartlist_t *ports, rend_service_port_config_t *p)
 
 #endif /* defined(HAVE_SYS_UN_H) */
 
-/* Helper function: The key is a digest that we compare to a node_t object
+/** Helper function: The key is a digest that we compare to a node_t object
  * current hsdir_index. */
 static int
 compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
@@ -115,7 +118,7 @@ compare_digest_to_fetch_hsdir_index(const void *_key, const void **_member)
   return tor_memcmp(key, node->hsdir_index.fetch, DIGEST256_LEN);
 }
 
-/* Helper function: The key is a digest that we compare to a node_t object
+/** Helper function: The key is a digest that we compare to a node_t object
  * next hsdir_index. */
 static int
 compare_digest_to_store_first_hsdir_index(const void *_key,
@@ -126,7 +129,7 @@ compare_digest_to_store_first_hsdir_index(const void *_key,
   return tor_memcmp(key, node->hsdir_index.store_first, DIGEST256_LEN);
 }
 
-/* Helper function: The key is a digest that we compare to a node_t object
+/** Helper function: The key is a digest that we compare to a node_t object
  * next hsdir_index. */
 static int
 compare_digest_to_store_second_hsdir_index(const void *_key,
@@ -137,7 +140,7 @@ compare_digest_to_store_second_hsdir_index(const void *_key,
   return tor_memcmp(key, node->hsdir_index.store_second, DIGEST256_LEN);
 }
 
-/* Helper function: Compare two node_t objects current hsdir_index. */
+/** Helper function: Compare two node_t objects current hsdir_index. */
 static int
 compare_node_fetch_hsdir_index(const void **a, const void **b)
 {
@@ -148,7 +151,7 @@ compare_node_fetch_hsdir_index(const void **a, const void **b)
                     DIGEST256_LEN);
 }
 
-/* Helper function: Compare two node_t objects next hsdir_index. */
+/** Helper function: Compare two node_t objects next hsdir_index. */
 static int
 compare_node_store_first_hsdir_index(const void **a, const void **b)
 {
@@ -159,7 +162,7 @@ compare_node_store_first_hsdir_index(const void **a, const void **b)
                     DIGEST256_LEN);
 }
 
-/* Helper function: Compare two node_t objects next hsdir_index. */
+/** Helper function: Compare two node_t objects next hsdir_index. */
 static int
 compare_node_store_second_hsdir_index(const void **a, const void **b)
 {
@@ -170,7 +173,7 @@ compare_node_store_second_hsdir_index(const void **a, const void **b)
                     DIGEST256_LEN);
 }
 
-/* Allocate and return a string containing the path to filename in directory.
+/** Allocate and return a string containing the path to filename in directory.
  * This function will never return NULL. The caller must free this path. */
 char *
 hs_path_from_filename(const char *directory, const char *filename)
@@ -184,8 +187,9 @@ hs_path_from_filename(const char *directory, const char *filename)
   return file_path;
 }
 
-/* Make sure that the directory for <b>service</b> is private, using the config
- * <b>username</b>.
+/** Make sure that the directory for <b>service</b> is private, using the
+ * config <b>username</b>.
+ *
  * If <b>create</b> is true:
  *  - if the directory exists, change permissions if needed,
  *  - if the directory does not exist, create it with the correct permissions.
@@ -305,18 +309,18 @@ hs_get_next_time_period_num(time_t now)
   return hs_get_time_period_num(now) + 1;
 }
 
-/* Get the number of the _previous_ HS time period, given that the current time
- * is <b>now</b>. If <b>now</b> is not set, we try to get the time from a live
- * consensus. */
+/** Get the number of the _previous_ HS time period, given that the current
+ * time is <b>now</b>. If <b>now</b> is not set, we try to get the time from a
+ * live consensus. */
 uint64_t
 hs_get_previous_time_period_num(time_t now)
 {
   return hs_get_time_period_num(now) - 1;
 }
 
-/* Return the start time of the upcoming time period based on <b>now</b>. If
-   <b>now</b> is not set, we try to get the time ourselves from a live
-   consensus. */
+/** Return the start time of the upcoming time period based on <b>now</b>. If
+ * <b>now</b> is not set, we try to get the time ourselves from a live
+ * consensus. */
 time_t
 hs_get_start_time_of_next_time_period(time_t now)
 {
@@ -331,7 +335,7 @@ hs_get_start_time_of_next_time_period(time_t now)
   return (time_t)(start_of_next_tp_in_mins * 60 + time_period_rotation_offset);
 }
 
-/* Create a new rend_data_t for a specific given <b>version</b>.
+/** Create a new rend_data_t for a specific given <b>version</b>.
  * Return a pointer to the newly allocated data structure. */
 static rend_data_t *
 rend_data_alloc(uint32_t version)
@@ -380,7 +384,7 @@ rend_data_free_(rend_data_t *data)
   }
 }
 
-/* Allocate and return a deep copy of <b>data</b>. */
+/** Allocate and return a deep copy of <b>data</b>. */
 rend_data_t *
 rend_data_dup(const rend_data_t *data)
 {
@@ -410,7 +414,7 @@ rend_data_dup(const rend_data_t *data)
   return data_dup;
 }
 
-/* Compute the descriptor ID for each HS descriptor replica and save them. A
+/** Compute the descriptor ID for each HS descriptor replica and save them. A
  * valid onion address must be present in the <b>rend_data</b>.
  *
  * Return 0 on success else -1. */
@@ -448,7 +452,7 @@ compute_desc_id(rend_data_t *rend_data)
   return ret;
 }
 
-/* Allocate and initialize a rend_data_t object for a service using the
+/** Allocate and initialize a rend_data_t object for a service using the
  * provided arguments. All arguments are optional (can be NULL), except from
  * <b>onion_address</b> which MUST be set. The <b>pk_digest</b> is the hash of
  * the service private key. The <b>cookie</b> is the rendezvous cookie and
@@ -480,7 +484,7 @@ rend_data_service_create(const char *onion_address, const char *pk_digest,
   return rend_data;
 }
 
-/* Allocate and initialize a rend_data_t object for a client request using the
+/** Allocate and initialize a rend_data_t object for a client request using the
  * given arguments. Either an onion address or a descriptor ID is needed. Both
  * can be given but in this case only the onion address will be used to make
  * the descriptor fetch. The <b>cookie</b> is the rendezvous cookie and
@@ -521,7 +525,7 @@ rend_data_client_create(const char *onion_address, const char *desc_id,
   return NULL;
 }
 
-/* Return the onion address from the rend data. Depending on the version,
+/** Return the onion address from the rend data. Depending on the version,
  * the size of the address can vary but it's always NUL terminated. */
 const char *
 rend_data_get_address(const rend_data_t *rend_data)
@@ -537,7 +541,7 @@ rend_data_get_address(const rend_data_t *rend_data)
   }
 }
 
-/* Return the descriptor ID for a specific replica number from the rend
+/** Return the descriptor ID for a specific replica number from the rend
  * data. The returned data is a binary digest and depending on the version its
  * size can vary. The size of the descriptor ID is put in <b>len_out</b> if
  * non NULL. */
@@ -560,7 +564,7 @@ rend_data_get_desc_id(const rend_data_t *rend_data, uint8_t replica,
   }
 }
 
-/* Return the public key digest using the given <b>rend_data</b>. The size of
+/** Return the public key digest using the given <b>rend_data</b>. The size of
  * the digest is put in <b>len_out</b> (if set) which can differ depending on
  * the version. */
 const uint8_t *
@@ -583,7 +587,7 @@ rend_data_get_pk_digest(const rend_data_t *rend_data, size_t *len_out)
   }
 }
 
-/* Using the given time period number, compute the disaster shared random
+/** Using the given time period number, compute the disaster shared random
  * value and put it in srv_out. It MUST be at least DIGEST256_LEN bytes. */
 static void
 compute_disaster_srv(uint64_t time_period_num, uint8_t *srv_out)
@@ -671,7 +675,7 @@ get_second_cached_disaster_srv(void)
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
-/* When creating a blinded key, we need a parameter which construction is as
+/** When creating a blinded key, we need a parameter which construction is as
  * follow: H(pubkey | [secret] | ed25519-basepoint | nonce).
  *
  * The nonce has a pre-defined format which uses the time period number
@@ -725,7 +729,7 @@ build_blinded_key_param(const ed25519_public_key_t *pubkey,
   memwipe(nonce, 0, sizeof(nonce));
 }
 
-/* Using an ed25519 public key and version to build the checksum of an
+/** Using an ed25519 public key and version to build the checksum of an
  * address. Put in checksum_out. Format is:
  *    SHA3-256(".onion checksum" || PUBKEY || VERSION)
  *
@@ -752,7 +756,7 @@ build_hs_checksum(const ed25519_public_key_t *key, uint8_t version,
                    DIGEST_SHA3_256);
 }
 
-/* Using an ed25519 public key, checksum and version to build the binary
+/** Using an ed25519 public key, checksum and version to build the binary
  * representation of a service address. Put in addr_out. Format is:
  *    addr_out = PUBKEY || CHECKSUM || VERSION
  *
@@ -775,7 +779,7 @@ build_hs_address(const ed25519_public_key_t *key, const uint8_t *checksum,
   tor_assert(offset == HS_SERVICE_ADDR_LEN);
 }
 
-/* Helper for hs_parse_address(): Using a binary representation of a service
+/** Helper for hs_parse_address(): Using a binary representation of a service
  * address, parse its content into the key_out, checksum_out and version_out.
  * Any out variable can be NULL in case the caller would want only one field.
  * checksum_out MUST at least be 2 bytes long. address must be at least
@@ -807,13 +811,13 @@ hs_parse_address_impl(const char *address, ed25519_public_key_t *key_out,
   tor_assert(offset == HS_SERVICE_ADDR_LEN);
 }
 
-/* Using the given identity public key and a blinded public key, compute the
- * subcredential and put it in subcred_out (must be of size DIGEST256_LEN).
+/** Using the given identity public key and a blinded public key, compute the
+ * subcredential and put it in subcred_out.
  * This can't fail. */
 void
 hs_get_subcredential(const ed25519_public_key_t *identity_pk,
                      const ed25519_public_key_t *blinded_pk,
-                     uint8_t *subcred_out)
+                     hs_subcredential_t *subcred_out)
 {
   uint8_t credential[DIGEST256_LEN];
   crypto_digest_t *digest;
@@ -841,13 +845,14 @@ hs_get_subcredential(const ed25519_public_key_t *identity_pk,
                           sizeof(credential));
   crypto_digest_add_bytes(digest, (const char *) blinded_pk->pubkey,
                           ED25519_PUBKEY_LEN);
-  crypto_digest_get_digest(digest, (char *) subcred_out, DIGEST256_LEN);
+  crypto_digest_get_digest(digest, (char *) subcred_out->subcred,
+                           SUBCRED_LEN);
   crypto_digest_free(digest);
 
   memwipe(credential, 0, sizeof(credential));
 }
 
-/* From the given list of hidden service ports, find the ones that match the
+/** From the given list of hidden service ports, find the ones that match the
  * given edge connection conn, pick one at random and use it to set the
  * connection address. Return 0 on success or -1 if none. */
 int
@@ -904,34 +909,40 @@ hs_set_conn_addr_port(const smartlist_t *ports, edge_connection_t *conn)
   return (chosen_port) ? 0 : -1;
 }
 
-/* Using a base32 representation of a service address, parse its content into
+/** Using a base32 representation of a service address, parse its content into
  * the key_out, checksum_out and version_out. Any out variable can be NULL in
  * case the caller would want only one field. checksum_out MUST at least be 2
  * bytes long.
  *
- * Return 0 if parsing went well; return -1 in case of error. */
+ * Return 0 if parsing went well; return -1 in case of error and if errmsg is
+ * non NULL, a human readable string message is set. */
 int
-hs_parse_address(const char *address, ed25519_public_key_t *key_out,
-                 uint8_t *checksum_out, uint8_t *version_out)
+hs_parse_address_no_log(const char *address, ed25519_public_key_t *key_out,
+                        uint8_t *checksum_out, uint8_t *version_out,
+                        const char **errmsg)
 {
   char decoded[HS_SERVICE_ADDR_LEN];
 
   tor_assert(address);
 
+  if (errmsg) {
+    *errmsg = NULL;
+  }
+
   /* Obvious length check. */
   if (strlen(address) != HS_SERVICE_ADDR_LEN_BASE32) {
-    log_warn(LD_REND, "Service address %s has an invalid length. "
-                      "Expected %lu but got %lu.",
-             escaped_safe_str(address),
-             (unsigned long) HS_SERVICE_ADDR_LEN_BASE32,
-             (unsigned long) strlen(address));
+    if (errmsg) {
+      *errmsg = "Invalid length";
+    }
     goto invalid;
   }
 
   /* Decode address so we can extract needed fields. */
-  if (base32_decode(decoded, sizeof(decoded), address, strlen(address)) < 0) {
-    log_warn(LD_REND, "Service address %s can't be decoded.",
-             escaped_safe_str(address));
+  if (base32_decode(decoded, sizeof(decoded), address, strlen(address))
+      != sizeof(decoded)) {
+    if (errmsg) {
+      *errmsg = "Unable to base32 decode";
+    }
     goto invalid;
   }
 
@@ -943,7 +954,23 @@ hs_parse_address(const char *address, ed25519_public_key_t *key_out,
   return -1;
 }
 
-/* Validate a given onion address. The length, the base32 decoding and
+/** Same has hs_parse_address_no_log() but emits a log warning on parsing
+ * failure. */
+int
+hs_parse_address(const char *address, ed25519_public_key_t *key_out,
+                 uint8_t *checksum_out, uint8_t *version_out)
+{
+  const char *errmsg = NULL;
+  int ret = hs_parse_address_no_log(address, key_out, checksum_out,
+                                    version_out, &errmsg);
+  if (ret < 0) {
+    log_warn(LD_REND, "Service address %s failed to be parsed: %s",
+             escaped_safe_str(address), errmsg);
+  }
+  return ret;
+}
+
+/** Validate a given onion address. The length, the base32 decoding, and
  * checksum are validated. Return 1 if valid else 0. */
 int
 hs_address_is_valid(const char *address)
@@ -958,7 +985,7 @@ hs_address_is_valid(const char *address)
     goto invalid;
   }
 
-  /* Get the checksum it's suppose to be and compare it with what we have
+  /* Get the checksum it's supposed to be and compare it with what we have
    * encoded in the address. */
   build_hs_checksum(&service_pubkey, version, target_checksum);
   if (tor_memcmp(checksum, target_checksum, sizeof(checksum))) {
@@ -982,11 +1009,11 @@ hs_address_is_valid(const char *address)
   return 0;
 }
 
-/* Build a service address using an ed25519 public key and a given version.
+/** Build a service address using an ed25519 public key and a given version.
  * The returned address is base32 encoded and put in addr_out. The caller MUST
  * make sure the addr_out is at least HS_SERVICE_ADDR_LEN_BASE32 + 1 long.
  *
- * Format is as follow:
+ * Format is as follows:
  *     base32(PUBKEY || CHECKSUM || VERSION)
  *     CHECKSUM = H(".onion checksum" || PUBKEY || VERSION)
  * */
@@ -1012,25 +1039,7 @@ hs_build_address(const ed25519_public_key_t *key, uint8_t version,
   tor_assert(hs_address_is_valid(addr_out));
 }
 
-/* Return a newly allocated copy of lspec. */
-link_specifier_t *
-hs_link_specifier_dup(const link_specifier_t *lspec)
-{
-  link_specifier_t *result = link_specifier_new();
-  memcpy(result, lspec, sizeof(*result));
-  /* The unrecognized field is a dynamic array so make sure to copy its
-   * content and not the pointer. */
-  link_specifier_setlen_un_unrecognized(
-                  result, link_specifier_getlen_un_unrecognized(lspec));
-  if (link_specifier_getlen_un_unrecognized(result)) {
-    memcpy(link_specifier_getarray_un_unrecognized(result),
-           link_specifier_getconstarray_un_unrecognized(lspec),
-           link_specifier_getlen_un_unrecognized(result));
-  }
-  return result;
-}
-
-/* From a given ed25519 public key pk and an optional secret, compute a
+/** From a given ed25519 public key pk and an optional secret, compute a
  * blinded public key and put it in blinded_pk_out. This is only useful to
  * the client side because the client only has access to the identity public
  * key of the service. */
@@ -1045,7 +1054,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
 
   tor_assert(pk);
   tor_assert(blinded_pk_out);
-  tor_assert(!tor_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) pk, ED25519_PUBKEY_LEN));
 
   build_blinded_key_param(pk, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1054,7 +1063,7 @@ hs_build_blinded_pubkey(const ed25519_public_key_t *pk,
   memwipe(param, 0, sizeof(param));
 }
 
-/* From a given ed25519 keypair kp and an optional secret, compute a blinded
+/** From a given ed25519 keypair kp and an optional secret, compute a blinded
  * keypair for the current time period and put it in blinded_kp_out. This is
  * only useful by the service side because the client doesn't have access to
  * the identity secret key. */
@@ -1070,8 +1079,8 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp,
   tor_assert(kp);
   tor_assert(blinded_kp_out);
   /* Extra safety. A zeroed key is bad. */
-  tor_assert(!tor_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
-  tor_assert(!tor_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->pubkey, ED25519_PUBKEY_LEN));
+  tor_assert(!fast_mem_is_zero((char *) &kp->seckey, ED25519_SECKEY_LEN));
 
   build_blinded_key_param(&kp->pubkey, secret, secret_len,
                           time_period_num, get_time_period_length(), param);
@@ -1080,7 +1089,7 @@ hs_build_blinded_keypair(const ed25519_keypair_t *kp,
   memwipe(param, 0, sizeof(param));
 }
 
-/* Return true if we are currently in the time segment between a new time
+/** Return true if we are currently in the time segment between a new time
  * period and a new SRV (in the real network that happens between 12:00 and
  * 00:00 UTC). Here is a diagram showing exactly when this returns true:
  *
@@ -1121,7 +1130,7 @@ hs_in_period_between_tp_and_srv,(const networkstatus_t *consensus, time_t now))
   return 1;
 }
 
-/* Return 1 if any virtual port in ports needs a circuit with good uptime.
+/** Return 1 if any virtual port in ports needs a circuit with good uptime.
  * Else return 0. */
 int
 hs_service_requires_uptime_circ(const smartlist_t *ports)
@@ -1137,7 +1146,7 @@ hs_service_requires_uptime_circ(const smartlist_t *ports)
   return 0;
 }
 
-/* Build hs_index which is used to find the responsible hsdirs. This index
+/** Build hs_index which is used to find the responsible hsdirs. This index
  * value is used to select the responsible HSDir where their hsdir_index is
  * closest to this value.
  *    SHA3-256("store-at-idx" | blinded_public_key |
@@ -1179,7 +1188,7 @@ hs_build_hs_index(uint64_t replica, const ed25519_public_key_t *blinded_pk,
   crypto_digest_free(digest);
 }
 
-/* Build hsdir_index which is used to find the responsible hsdirs. This is the
+/** Build hsdir_index which is used to find the responsible hsdirs. This is the
  * index value that is compare to the hs_index when selecting an HSDir.
  *    SHA3-256("node-idx" | node_identity |
  *             shared_random_value | INT_8(period_length) | INT_8(period_num) )
@@ -1220,7 +1229,7 @@ hs_build_hsdir_index(const ed25519_public_key_t *identity_pk,
   crypto_digest_free(digest);
 }
 
-/* Return a newly allocated buffer containing the current shared random value
+/** Return a newly allocated buffer containing the current shared random value
  * or if not present, a disaster value is computed using the given time period
  * number. If a consensus is provided in <b>ns</b>, use it to get the SRV
  * value. This function can't fail. */
@@ -1239,7 +1248,7 @@ hs_get_current_srv(uint64_t time_period_num, const networkstatus_t *ns)
   return sr_value;
 }
 
-/* Return a newly allocated buffer containing the previous shared random
+/** Return a newly allocated buffer containing the previous shared random
  * value or if not present, a disaster value is computed using the given time
  * period number. This function can't fail. */
 uint8_t *
@@ -1257,7 +1266,7 @@ hs_get_previous_srv(uint64_t time_period_num, const networkstatus_t *ns)
   return sr_value;
 }
 
-/* Return the number of replicas defined by a consensus parameter or the
+/** Return the number of replicas defined by a consensus parameter or the
  * default value. */
 int32_t
 hs_get_hsdir_n_replicas(void)
@@ -1267,7 +1276,7 @@ hs_get_hsdir_n_replicas(void)
                                  HS_DEFAULT_HSDIR_N_REPLICAS, 1, 16);
 }
 
-/* Return the spread fetch value defined by a consensus parameter or the
+/** Return the spread fetch value defined by a consensus parameter or the
  * default value. */
 int32_t
 hs_get_hsdir_spread_fetch(void)
@@ -1277,7 +1286,7 @@ hs_get_hsdir_spread_fetch(void)
                                  HS_DEFAULT_HSDIR_SPREAD_FETCH, 1, 128);
 }
 
-/* Return the spread store value defined by a consensus parameter or the
+/** Return the spread store value defined by a consensus parameter or the
  * default value. */
 int32_t
 hs_get_hsdir_spread_store(void)
@@ -1304,15 +1313,15 @@ node_has_hsdir_index(const node_t *node)
 
   /* At this point, since the node has a desc, this node must also have an
    * hsdir index. If not, something went wrong, so BUG out. */
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.fetch,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.fetch,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_first,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_first,
                           DIGEST256_LEN))) {
     return 0;
   }
-  if (BUG(tor_mem_is_zero((const char*)node->hsdir_index.store_second,
+  if (BUG(fast_mem_is_zero((const char*)node->hsdir_index.store_second,
                           DIGEST256_LEN))) {
     return 0;
   }
@@ -1320,7 +1329,7 @@ node_has_hsdir_index(const node_t *node)
   return 1;
 }
 
-/* For a given blinded key and time period number, get the responsible HSDir
+/** For a given blinded key and time period number, get the responsible HSDir
  * and put their routerstatus_t object in the responsible_dirs list. If
  * 'use_second_hsdir_index' is true, use the second hsdir_index of the node_t
  * is used. If 'for_fetching' is true, the spread fetch consensus parameter is
@@ -1612,20 +1621,25 @@ hs_purge_last_hid_serv_requests(void)
 /** Given the list of responsible HSDirs in <b>responsible_dirs</b>, pick the
  *  one that we should use to fetch a descriptor right now. Take into account
  *  previous failed attempts at fetching this descriptor from HSDirs using the
- *  string identifier <b>req_key_str</b>.
+ *  string identifier <b>req_key_str</b>. We return whether we are rate limited
+ *  into *<b>is_rate_limited_out</b> if it is not NULL.
  *
  *  Steals ownership of <b>responsible_dirs</b>.
  *
  *  Return the routerstatus of the chosen HSDir if successful, otherwise return
  *  NULL if no HSDirs are worth trying right now. */
 routerstatus_t *
-hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
+hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str,
+              bool *is_rate_limited_out)
 {
   smartlist_t *usable_responsible_dirs = smartlist_new();
   const or_options_t *options = get_options();
   routerstatus_t *hs_dir;
   time_t now = time(NULL);
   int excluded_some;
+  bool rate_limited = false;
+  int rate_limited_count = 0;
+  int responsible_dirs_count = smartlist_len(responsible_dirs);
 
   tor_assert(req_key_str);
 
@@ -1645,6 +1659,7 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     if (last + hs_hsdir_requery_period(options) >= now ||
         !node || !node_has_preferred_descriptor(node, 0)) {
       SMARTLIST_DEL_CURRENT(responsible_dirs, dir);
+      rate_limited_count++;
       continue;
     }
     if (!routerset_contains_node(options->ExcludeNodes, node)) {
@@ -1652,6 +1667,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     }
   } SMARTLIST_FOREACH_END(dir);
 
+  if (rate_limited_count > 0 || responsible_dirs_count > 0) {
+    rate_limited = rate_limited_count == responsible_dirs_count;
+  }
+
   excluded_some =
     smartlist_len(usable_responsible_dirs) < smartlist_len(responsible_dirs);
 
@@ -1663,9 +1682,10 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
   smartlist_free(responsible_dirs);
   smartlist_free(usable_responsible_dirs);
   if (!hs_dir) {
+    const char *warn_str = (rate_limited) ? "we are rate limited." :
+                              "we requested them all recently without success";
     log_info(LD_REND, "Could not pick one of the responsible hidden "
-                      "service directories, because we requested them all "
-                      "recently without success.");
+                      "service directories, because %s.", warn_str);
     if (options->StrictNodes && excluded_some) {
       log_warn(LD_REND, "Could not pick a hidden service directory for the "
                "requested hidden service: they are all either down or "
@@ -1677,17 +1697,23 @@ hs_pick_hsdir(smartlist_t *responsible_dirs, const char *req_key_str)
     hs_lookup_last_hid_serv_request(hs_dir, req_key_str, now, 1);
   }
 
+  if (is_rate_limited_out != NULL) {
+    *is_rate_limited_out = rate_limited;
+  }
+
   return hs_dir;
 }
 
-/* From a list of link specifier, an onion key and if we are requesting a
- * direct connection (ex: single onion service), return a newly allocated
- * extend_info_t object. This function always returns an extend info with
- * an IPv4 address, or NULL.
+/** Given a list of link specifiers lspecs, a curve 25519 onion_key, and
+ * a direct connection boolean direct_conn (true for single onion services),
+ * return a newly allocated extend_info_t object.
+ *
+ * This function always returns an extend info with a valid IP address and
+ * ORPort, or NULL. If direct_conn is false, the IP address is always IPv4.
  *
  * It performs the following checks:
- *  if either IPv4 or legacy ID is missing, return NULL.
- *  if direct_conn, and we can't reach the IPv4 address, return NULL.
+ *  if there is no usable IP address, or legacy ID is missing, return NULL.
+ *  if direct_conn, and we can't reach any IP address, return NULL.
  */
 extend_info_t *
 hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
@@ -1696,21 +1722,40 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
 {
   int have_v4 = 0, have_legacy_id = 0, have_ed25519_id = 0;
   char legacy_id[DIGEST_LEN] = {0};
-  uint16_t port_v4 = 0;
-  tor_addr_t addr_v4;
   ed25519_public_key_t ed25519_pk;
   extend_info_t *info = NULL;
+  tor_addr_port_t ap;
 
-  tor_assert(lspecs);
+  tor_addr_make_null(&ap.addr, AF_UNSPEC);
+  ap.port = 0;
+
+  if (lspecs == NULL) {
+    log_warn(LD_BUG, "Specified link specifiers is null");
+    goto done;
+  }
+
+  if (onion_key == NULL) {
+    log_warn(LD_BUG, "Specified onion key is null");
+    goto done;
+  }
+
+  if (smartlist_len(lspecs) == 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND, "Empty link specifier list.");
+    /* Return NULL. */
+    goto done;
+  }
 
   SMARTLIST_FOREACH_BEGIN(lspecs, const link_specifier_t *, ls) {
     switch (link_specifier_get_ls_type(ls)) {
     case LS_IPV4:
-      /* Skip if we already seen a v4. */
-      if (have_v4) continue;
-      tor_addr_from_ipv4h(&addr_v4,
+      /* Skip if we already seen a v4. If direct_conn is true, we skip this
+       * block because fascist_firewall_choose_address_ls() will set ap. If
+       * direct_conn is false, set ap to the first IPv4 address and port in
+       * the link specifiers.*/
+      if (have_v4 || direct_conn) continue;
+      tor_addr_from_ipv4h(&ap.addr,
                           link_specifier_get_un_ipv4_addr(ls));
-      port_v4 = link_specifier_get_un_ipv4_port(ls);
+      ap.port = link_specifier_get_un_ipv4_port(ls);
       have_v4 = 1;
       break;
     case LS_LEGACY_ID:
@@ -1734,52 +1779,45 @@ hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
     }
   } SMARTLIST_FOREACH_END(ls);
 
-  /* Legacy ID is mandatory, and we require IPv4. */
-  if (!have_v4 || !have_legacy_id) {
+  /* Choose a preferred address first, but fall back to an allowed address. */
+  if (direct_conn)
+    fascist_firewall_choose_address_ls(lspecs, 0, &ap);
+
+  /* Legacy ID is mandatory, and we require an IP address. */
+  if (!tor_addr_port_is_valid_ap(&ap, 0)) {
+    /* If we're missing the IP address, log a warning and return NULL. */
+    log_info(LD_NET, "Unreachable or invalid IP address in link state");
     goto done;
   }
-
-  /* We know we have IPv4, because we just checked. */
-  if (!direct_conn) {
-    /* All clients can extend to any IPv4 via a 3-hop path. */
-    goto validate;
-  } else if (direct_conn &&
-             fascist_firewall_allows_address_addr(&addr_v4, port_v4,
-                                                  FIREWALL_OR_CONNECTION,
-                                                  0, 0)) {
-    /* Direct connection and we can reach it in IPv4 so go for it. */
-    goto validate;
-
-    /* We will add support for falling back to a 3-hop path in a later
-     * release. */
-  } else {
-    /* If we can't reach IPv4, return NULL. */
+  if (!have_legacy_id) {
+    /* If we're missing the legacy ID, log a warning and return NULL. */
+    log_warn(LD_PROTOCOL, "Missing Legacy ID in link state");
     goto done;
   }
 
-  /* We will add support for IPv6 in a later release. */
+  /* We will add support for falling back to a 3-hop path in a later
+   * release. */
 
- validate:
   /* We'll validate now that the address we've picked isn't a private one. If
-   * it is, are we allowing to extend to private address? */
-  if (!extend_info_addr_is_allowed(&addr_v4)) {
+   * it is, are we allowed to extend to private addresses? */
+  if (!extend_info_addr_is_allowed(&ap.addr)) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Requested address is private and we are not allowed to extend to "
-           "it: %s:%u", fmt_addr(&addr_v4), port_v4);
+           "it: %s:%u", fmt_addr(&ap.addr), ap.port);
     goto done;
   }
 
   /* We do have everything for which we think we can connect successfully. */
   info = extend_info_new(NULL, legacy_id,
                          (have_ed25519_id) ? &ed25519_pk : NULL, NULL,
-                         onion_key, &addr_v4, port_v4);
+                         onion_key, &ap.addr, ap.port);
  done:
   return info;
 }
 
 /***********************************************************************/
 
-/* Initialize the entire HS subsytem. This is called in tor_init() before any
+/** Initialize the entire HS subsytem. This is called in tor_init() before any
  * torrc options are loaded. Only for >= v3. */
 void
 hs_init(void)
@@ -1789,7 +1827,7 @@ hs_init(void)
   hs_cache_init();
 }
 
-/* Release and cleanup all memory of the HS subsystem (all version). This is
+/** Release and cleanup all memory of the HS subsystem (all version). This is
  * called by tor_free_all(). */
 void
 hs_free_all(void)
@@ -1798,9 +1836,10 @@ hs_free_all(void)
   hs_service_free_all();
   hs_cache_free_all();
   hs_client_free_all();
+  hs_ob_free_all();
 }
 
-/* For the given origin circuit circ, decrement the number of rendezvous
+/** For the given origin circuit circ, decrement the number of rendezvous
  * stream counter. This handles every hidden service version. */
 void
 hs_dec_rdv_stream_counter(origin_circuit_t *circ)
@@ -1817,7 +1856,7 @@ hs_dec_rdv_stream_counter(origin_circuit_t *circ)
   }
 }
 
-/* For the given origin circuit circ, increment the number of rendezvous
+/** For the given origin circuit circ, increment the number of rendezvous
  * stream counter. This handles every hidden service version. */
 void
 hs_inc_rdv_stream_counter(origin_circuit_t *circ)
@@ -1833,3 +1872,42 @@ hs_inc_rdv_stream_counter(origin_circuit_t *circ)
     tor_assert_nonfatal_unreached();
   }
 }
+
+/** Return a newly allocated link specifier object that is a copy of dst. */
+link_specifier_t *
+link_specifier_dup(const link_specifier_t *src)
+{
+  link_specifier_t *dup = NULL;
+  uint8_t *buf = NULL;
+
+  if (BUG(!src)) {
+    goto err;
+  }
+
+  ssize_t encoded_len_alloc = link_specifier_encoded_len(src);
+  if (BUG(encoded_len_alloc < 0)) {
+    goto err;
+  }
+
+  buf = tor_malloc_zero(encoded_len_alloc);
+  ssize_t encoded_len_data = link_specifier_encode(buf,
+                                                   encoded_len_alloc,
+                                                   src);
+  if (BUG(encoded_len_data < 0)) {
+    goto err;
+  }
+
+  ssize_t parsed_len = link_specifier_parse(&dup, buf, encoded_len_alloc);
+  if (BUG(parsed_len < 0)) {
+    goto err;
+  }
+
+  goto done;
+
+ err:
+  dup = NULL;
+
+ done:
+  tor_free(buf);
+  return dup;
+}
diff --git a/src/feature/hs/hs_common.h b/src/feature/hs/hs_common.h
index a44505930a..997b7298a6 100644
--- a/src/feature/hs/hs_common.h
+++ b/src/feature/hs/hs_common.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,13 +19,14 @@ struct ed25519_keypair_t;
 /* Trunnel */
 #include "trunnel/ed25519_cert.h"
 
-/* Protocol version 2. Use this instead of hardcoding "2" in the code base,
+/** Protocol version 2. Use this instead of hardcoding "2" in the code base,
  * this adds a clearer semantic to the value when used. */
 #define HS_VERSION_TWO 2
-/* Version 3 of the protocol (prop224). */
+/** Version 3 of the protocol (prop224). */
 #define HS_VERSION_THREE 3
-/* Earliest and latest version we support. */
+/** Earliest version we support. */
 #define HS_VERSION_MIN HS_VERSION_TWO
+/** Latest version we support. */
 #define HS_VERSION_MAX HS_VERSION_THREE
 
 /** Try to maintain this many intro points per service by default. */
@@ -48,94 +49,95 @@ struct ed25519_keypair_t;
  * rendezvous point before giving up? */
 #define MAX_REND_TIMEOUT 30
 
-/* String prefix for the signature of ESTABLISH_INTRO */
+/** String prefix for the signature of ESTABLISH_INTRO */
 #define ESTABLISH_INTRO_SIG_PREFIX "Tor establish-intro cell v1"
 
-/* The default HS time period length */
+/** The default HS time period length */
 #define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
 
-/* Prefix of the onion address checksum. */
+/** Prefix of the onion address checksum. */
 #define HS_SERVICE_ADDR_CHECKSUM_PREFIX ".onion checksum"
-/* Length of the checksum prefix minus the NUL terminated byte. */
+/** Length of the checksum prefix minus the NUL terminated byte. */
 #define HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN \
   (sizeof(HS_SERVICE_ADDR_CHECKSUM_PREFIX) - 1)
-/* Length of the resulting checksum of the address. The construction of this
+/** Length of the resulting checksum of the address. The construction of this
  * checksum looks like:
  *   CHECKSUM = ".onion checksum" || PUBKEY || VERSION
  * where VERSION is 1 byte. This is pre-hashing. */
 #define HS_SERVICE_ADDR_CHECKSUM_INPUT_LEN \
   (HS_SERVICE_ADDR_CHECKSUM_PREFIX_LEN + ED25519_PUBKEY_LEN + sizeof(uint8_t))
-/* The amount of bytes we use from the address checksum. */
+/** The amount of bytes we use from the address checksum. */
 #define HS_SERVICE_ADDR_CHECKSUM_LEN_USED 2
-/* Length of the binary encoded service address which is of course before the
+/** Length of the binary encoded service address which is of course before the
  * base32 encoding. Construction is:
  *    PUBKEY || CHECKSUM || VERSION
  * with 1 byte VERSION and 2 bytes CHECKSUM. The following is 35 bytes. */
 #define HS_SERVICE_ADDR_LEN \
   (ED25519_PUBKEY_LEN + HS_SERVICE_ADDR_CHECKSUM_LEN_USED + sizeof(uint8_t))
-/* Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
+/** Length of 'y' portion of 'y.onion' URL. This is base32 encoded and the
  * length ends up to 56 bytes (not counting the terminated NUL byte.) */
 #define HS_SERVICE_ADDR_LEN_BASE32 \
   (CEIL_DIV(HS_SERVICE_ADDR_LEN * 8, 5))
 
-/* The default HS time period length */
+/** The default HS time period length */
 #define HS_TIME_PERIOD_LENGTH_DEFAULT 1440 /* 1440 minutes == one day */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MIN 30 /* minutes */
-/* The minimum time period length as seen in prop224 section [TIME-PERIODS] */
+/** The minimum time period length as seen in prop224 section [TIME-PERIODS] */
 #define HS_TIME_PERIOD_LENGTH_MAX (60 * 24 * 10) /* 10 days or 14400 minutes */
-/* The time period rotation offset as seen in prop224 section [TIME-PERIODS] */
+/** The time period rotation offset as seen in prop224 section
+ * [TIME-PERIODS] */
 #define HS_TIME_PERIOD_ROTATION_OFFSET (12 * 60) /* minutes */
 
-/* Keyblinding parameter construction is as follow:
+/** Keyblinding parameter construction is as follow:
  *    "key-blind" || INT_8(period_num) || INT_8(start_period_sec) */
 #define HS_KEYBLIND_NONCE_PREFIX "key-blind"
 #define HS_KEYBLIND_NONCE_PREFIX_LEN (sizeof(HS_KEYBLIND_NONCE_PREFIX) - 1)
 #define HS_KEYBLIND_NONCE_LEN \
   (HS_KEYBLIND_NONCE_PREFIX_LEN + sizeof(uint64_t) + sizeof(uint64_t))
 
-/* Credential and subcredential prefix value. */
+/** Credential and subcredential prefix value. */
 #define HS_CREDENTIAL_PREFIX "credential"
 #define HS_CREDENTIAL_PREFIX_LEN (sizeof(HS_CREDENTIAL_PREFIX) - 1)
 #define HS_SUBCREDENTIAL_PREFIX "subcredential"
 #define HS_SUBCREDENTIAL_PREFIX_LEN (sizeof(HS_SUBCREDENTIAL_PREFIX) - 1)
 
-/* Node hidden service stored at index prefix value. */
+/** Node hidden service stored at index prefix value. */
 #define HS_INDEX_PREFIX "store-at-idx"
 #define HS_INDEX_PREFIX_LEN (sizeof(HS_INDEX_PREFIX) - 1)
 
-/* Node hidden service directory index prefix value. */
+/** Node hidden service directory index prefix value. */
 #define HSDIR_INDEX_PREFIX "node-idx"
 #define HSDIR_INDEX_PREFIX_LEN (sizeof(HSDIR_INDEX_PREFIX) - 1)
 
-/* Prefix of the shared random value disaster mode. */
+/** Prefix of the shared random value disaster mode. */
 #define HS_SRV_DISASTER_PREFIX "shared-random-disaster"
 #define HS_SRV_DISASTER_PREFIX_LEN (sizeof(HS_SRV_DISASTER_PREFIX) - 1)
 
-/* Default value of number of hsdir replicas (hsdir_n_replicas). */
+/** Default value of number of hsdir replicas (hsdir_n_replicas). */
 #define HS_DEFAULT_HSDIR_N_REPLICAS 2
-/* Default value of hsdir spread store (hsdir_spread_store). */
+/** Default value of hsdir spread store (hsdir_spread_store). */
 #define HS_DEFAULT_HSDIR_SPREAD_STORE 4
-/* Default value of hsdir spread fetch (hsdir_spread_fetch). */
+/** Default value of hsdir spread fetch (hsdir_spread_fetch). */
 #define HS_DEFAULT_HSDIR_SPREAD_FETCH 3
 
-/* The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
+/** The size of a legacy RENDEZVOUS1 cell which adds up to 168 bytes. It is
  * bigger than the 84 bytes needed for version 3 so we need to pad up to that
  * length so it is indistinguishable between versions. */
 #define HS_LEGACY_RENDEZVOUS_CELL_SIZE \
   (REND_COOKIE_LEN + DH1024_KEY_LEN + DIGEST_LEN)
 
-/* Type of authentication key used by an introduction point. */
+/** Type of authentication key used by an introduction point. */
 typedef enum {
   HS_AUTH_KEY_TYPE_LEGACY  = 1,
   HS_AUTH_KEY_TYPE_ED25519 = 2,
 } hs_auth_key_type_t;
 
-/* Return value when adding an ephemeral service through the ADD_ONION
+/** Return value when adding an ephemeral service through the ADD_ONION
  * control port command. Both v2 and v3 share these. */
 typedef enum {
   RSAE_BADAUTH     = -5, /**< Invalid auth_type/auth_clients */
@@ -146,18 +148,18 @@ typedef enum {
   RSAE_OKAY        = 0   /**< Service added as expected */
 } hs_service_add_ephemeral_status_t;
 
-/* Represents the mapping from a virtual port of a rendezvous service to a
+/** Represents the mapping from a virtual port of a rendezvous service to a
  * real port on some IP. */
 typedef struct rend_service_port_config_t {
-  /* The incoming HS virtual port we're mapping */
+  /** The incoming HS virtual port we're mapping */
   uint16_t virtual_port;
-  /* Is this an AF_UNIX port? */
+  /** Is this an AF_UNIX port? */
   unsigned int is_unix_addr:1;
-  /* The outgoing TCP port to use, if !is_unix_addr */
+  /** The outgoing TCP port to use, if !is_unix_addr */
   uint16_t real_port;
-  /* The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
+  /** The outgoing IPv4 or IPv6 address to use, if !is_unix_addr */
   tor_addr_t real_addr;
-  /* The socket path to connect to, if is_unix_addr */
+  /** The socket path to connect to, if is_unix_addr */
   char unix_addr[FLEXIBLE_ARRAY_MEMBER];
 } rend_service_port_config_t;
 
@@ -177,6 +179,10 @@ void hs_build_address(const struct ed25519_public_key_t *key, uint8_t version,
 int hs_address_is_valid(const char *address);
 int hs_parse_address(const char *address, struct ed25519_public_key_t *key_out,
                      uint8_t *checksum_out, uint8_t *version_out);
+int hs_parse_address_no_log(const char *address,
+                            struct ed25519_public_key_t *key_out,
+                            uint8_t *checksum_out, uint8_t *version_out,
+                            const char **errmsg);
 
 void hs_build_blinded_pubkey(const struct ed25519_public_key_t *pubkey,
                              const uint8_t *secret, size_t secret_len,
@@ -208,17 +214,16 @@ const uint8_t *rend_data_get_pk_digest(const rend_data_t *rend_data,
 
 routerstatus_t *pick_hsdir(const char *desc_id, const char *desc_id_base32);
 
+struct hs_subcredential_t;
 void hs_get_subcredential(const struct ed25519_public_key_t *identity_pk,
                           const struct ed25519_public_key_t *blinded_pk,
-                          uint8_t *subcred_out);
+                          struct hs_subcredential_t *subcred_out);
 
 uint64_t hs_get_previous_time_period_num(time_t now);
 uint64_t hs_get_time_period_num(time_t now);
 uint64_t hs_get_next_time_period_num(time_t now);
 time_t hs_get_start_time_of_next_time_period(time_t now);
 
-link_specifier_t *hs_link_specifier_dup(const link_specifier_t *lspec);
-
 MOCK_DECL(int, hs_in_period_between_tp_and_srv,
           (const networkstatus_t *consensus, time_t now));
 
@@ -243,7 +248,8 @@ void hs_get_responsible_hsdirs(const struct ed25519_public_key_t *blinded_pk,
                               int use_second_hsdir_index,
                               int for_fetching, smartlist_t *responsible_dirs);
 routerstatus_t *hs_pick_hsdir(smartlist_t *responsible_dirs,
-                              const char *req_key_str);
+                              const char *req_key_str,
+                              bool *is_rate_limited_out);
 
 time_t hs_hsdir_requery_period(const or_options_t *options);
 time_t hs_lookup_last_hid_serv_request(routerstatus_t *hs_dir,
@@ -262,6 +268,8 @@ extend_info_t *hs_get_extend_info_from_lspecs(const smartlist_t *lspecs,
                           const struct curve25519_public_key_t *onion_key,
                           int direct_conn);
 
+link_specifier_t *link_specifier_dup(const link_specifier_t *src);
+
 #ifdef HS_COMMON_PRIVATE
 
 STATIC void get_disaster_srv(uint64_t time_period_num, uint8_t *srv_out);
diff --git a/src/feature/hs/hs_config.c b/src/feature/hs/hs_config.c
index ee4499ef5b..0dad8dd6d8 100644
--- a/src/feature/hs/hs_config.c
+++ b/src/feature/hs/hs_config.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,18 +23,72 @@
  * every option that is common to all version (config_generic_service).
  **/
 
-#define HS_CONFIG_PRIVATE
-
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_client.h"
+#include "feature/hs/hs_ob.h"
 #include "feature/hs/hs_service.h"
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
 #include "lib/encoding/confline.h"
+#include "lib/conf/confdecl.h"
+#include "lib/confmgt/confmgt.h"
+
+#include "feature/hs/hs_opts_st.h"
 #include "app/config/or_options_st.h"
 
-/* Using the given list of services, stage them into our global state. Every
+/* Declare the table mapping hs options to hs_opts_t */
+#define CONF_CONTEXT TABLE
+#include "feature/hs/hs_options.inc"
+#undef CONF_CONTEXT
+
+/** Magic number for hs_opts_t. */
+#define HS_OPTS_MAGIC 0x6f6e796e
+
+static const config_format_t hs_opts_fmt = {
+  .size = sizeof(hs_opts_t),
+  .magic = { "hs_opts_t",
+             HS_OPTS_MAGIC,
+             offsetof(hs_opts_t, magic) },
+  .vars = hs_opts_t_vars,
+};
+
+/** Global configuration manager to handle HS sections*/
+static config_mgr_t *hs_opts_mgr = NULL;
+
+/**
+ * Return a configuration manager for the hs_opts_t configuration type.
+ **/
+static const config_mgr_t *
+get_hs_opts_mgr(void)
+{
+  if (PREDICT_UNLIKELY(hs_opts_mgr == NULL)) {
+    hs_opts_mgr = config_mgr_new(&hs_opts_fmt);
+    config_mgr_freeze(hs_opts_mgr);
+  }
+  return hs_opts_mgr;
+}
+
+/**
+ * Allocate, initialize, and return a new hs_opts_t.
+ **/
+static hs_opts_t *
+hs_opts_new(void)
+{
+  const config_mgr_t *mgr = get_hs_opts_mgr();
+  hs_opts_t *r = config_new(mgr);
+  tor_assert(r);
+  config_init(mgr, r);
+  return r;
+}
+
+/**
+ * Free an hs_opts_t.
+ **/
+#define hs_opts_free(opts) \
+  config_free(get_hs_opts_mgr(), (opts))
+
+/** Using the given list of services, stage them into our global state. Every
  * service version are handled. This function can remove entries in the given
  * service_list.
  *
@@ -70,7 +124,7 @@ stage_services(smartlist_t *service_list)
   hs_service_stage_services(service_list);
 }
 
-/* Validate the given service against all service in the given list. If the
+/** Validate the given service against all service in the given list. If the
  * service is ephemeral, this function ignores it. Services with the same
  * directory path aren't allowed and will return an error. If a duplicate is
  * found, 1 is returned else 0 if none found. */
@@ -118,33 +172,27 @@ service_is_duplicate_in_list(const smartlist_t *service_list,
   return ret;
 }
 
-/* Helper function: Given an configuration option name, its value, a minimum
- * min and a maxium max, parse the value as a uint64_t. On success, ok is set
- * to 1 and ret is the parsed value. On error, ok is set to 0 and ret must be
- * ignored. This function logs both on error and success. */
-static uint64_t
-helper_parse_uint64(const char *opt, const char *value, uint64_t min,
-                    uint64_t max, int *ok)
+/** Check whether an integer <b>i</b> is out of bounds (not between <b>low</b>
+ * and <b>high</b> incusive).  If it is, then log a warning about the option
+ * <b>name</b>, and return true. Otherwise return false. */
+static bool
+check_value_oob(int i, const char *name, int low, int high)
 {
-  uint64_t ret = 0;
-
-  tor_assert(opt);
-  tor_assert(value);
-  tor_assert(ok);
-
-  *ok = 0;
-  ret = tor_parse_uint64(value, 10, min, max, ok, NULL);
-  if (!*ok) {
-    log_warn(LD_CONFIG, "%s must be between %" PRIu64 " and %"PRIu64
-                        ", not %s.",
-             opt, min, max, value);
-    goto err;
+  if (i < low || i > high) {
+    log_warn(LD_CONFIG, "%s must be between %d and %d, not %d.",
+             name, low, high, i);
+    return true;
   }
-  log_info(LD_CONFIG, "%s was parsed to %" PRIu64, opt, ret);
- err:
-  return ret;
+  return false;
 }
 
+/**
+ * Helper: check whether the integer value called <b>name</b> in <b>opts</b>
+ * is out-of-bounds.
+ **/
+#define CHECK_OOB(opts, name, low, high)      \
+  check_value_oob((opts)->name, #name, (low), (high))
+
 /** Helper function: Given a configuration option and its value, parse the
  * value as a hs_circuit_id_protocol_t. On success, ok is set to 1 and ret is
  * the parse value. On error, ok is set to 0 and the "none"
@@ -173,7 +221,7 @@ helper_parse_circuit_id_protocol(const char *key, const char *value, int *ok)
   return ret;
 }
 
-/* Return the service version by trying to learn it from the key on disk if
+/** Return the service version by trying to learn it from the key on disk if
  * any. If nothing is found, the current service configured version is
  * returned. */
 static int
@@ -191,7 +239,13 @@ config_learn_service_version(hs_service_t *service)
   return version;
 }
 
-/* Return true iff the given options starting at line_ for a hidden service
+/**
+ * Header key indicating the start of a new hidden service configuration
+ * block.
+ **/
+static const char SECTION_HEADER[] = "HiddenServiceDir";
+
+/** Return true iff the given options starting at line_ for a hidden service
  * contains at least one invalid option. Each hidden service option don't
  * apply to all versions so this function can find out. The line_ MUST start
  * right after the HiddenServiceDir line of this service.
@@ -218,6 +272,10 @@ config_has_invalid_options(const config_line_t *line_,
 
   const char *opts_exclude_v2[] = {
     "HiddenServiceExportCircuitID",
+    "HiddenServiceEnableIntroDoSDefense",
+    "HiddenServiceEnableIntroDoSRatePerSec",
+    "HiddenServiceEnableIntroDoSBurstPerSec",
+    "HiddenServiceOnionBalanceInstance",
     NULL /* End marker. */
   };
 
@@ -241,8 +299,11 @@ config_has_invalid_options(const config_line_t *line_,
   for (int i = 0; optlist[i]; i++) {
     const char *opt = optlist[i];
     for (line = line_; line; line = line->next) {
-      if (!strcasecmp(line->key, "HiddenServiceDir")) {
-        /* We just hit the next hidden service, stop right now. */
+      if (!strcasecmp(line->key, SECTION_HEADER)) {
+        /* We just hit the next hidden service, stop right now.
+         * (This shouldn't be possible, now that we have partitioned the list
+         * into sections.) */
+        tor_assert_nonfatal_unreached();
         goto end;
       }
       if (!strcasecmp(line->key, opt)) {
@@ -250,6 +311,16 @@ config_has_invalid_options(const config_line_t *line_,
                             "version %" PRIu32 " of service in %s",
                  opt, service->config.version,
                  service->config.directory_path);
+
+        if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
+          /* Special case this v2 option so that we can offer alternatives.
+           * If more such special cases appear, it would be good to
+           * generalize the exception mechanism here. */
+          log_warn(LD_CONFIG, "For v3 onion service client authorization, "
+                   "please read the 'CLIENT AUTHORIZATION' section in the "
+                   "manual.");
+        }
+
         ret = 1;
         /* Continue the loop so we can find all possible options. */
         continue;
@@ -260,7 +331,7 @@ config_has_invalid_options(const config_line_t *line_,
   return ret;
 }
 
-/* Validate service configuration. This is used when loading the configuration
+/** Validate service configuration. This is used when loading the configuration
  * and once we've setup a service object, it's config object is passed to this
  * function for further validation. This does not validate service key
  * material. Return 0 if valid else -1 if invalid. */
@@ -276,63 +347,83 @@ config_validate_service(const hs_service_config_t *config)
     goto invalid;
   }
 
+  /* DoS validation values. */
+  if (config->has_dos_defense_enabled &&
+      (config->intro_dos_burst_per_sec < config->intro_dos_rate_per_sec)) {
+    log_warn(LD_CONFIG, "Hidden service DoS defenses burst (%" PRIu32 ") can "
+                        "not be smaller than the rate value (%" PRIu32 ").",
+             config->intro_dos_burst_per_sec, config->intro_dos_rate_per_sec);
+    goto invalid;
+  }
+
   /* Valid. */
   return 0;
  invalid:
   return -1;
 }
 
-/* Configuration funcion for a version 3 service. The line_ must be pointing
- * to the directive directly after a HiddenServiceDir. That way, when hitting
- * the next HiddenServiceDir line or reaching the end of the list of lines, we
- * know that we have to stop looking for more options. The given service
+/** Configuration funcion for a version 3 service. The given service
  * object must be already allocated and passed through
  * config_generic_service() prior to calling this function.
  *
  * Return 0 on success else a negative value. */
 static int
-config_service_v3(const config_line_t *line_,
+config_service_v3(const hs_opts_t *hs_opts,
                   hs_service_config_t *config)
 {
-  int have_num_ip = 0;
-  bool export_circuit_id = false; /* just to detect duplicate options */
-  const char *dup_opt_seen = NULL;
-  const config_line_t *line;
-
   tor_assert(config);
+  tor_assert(hs_opts);
 
-  for (line = line_; line; line = line->next) {
-    int ok = 0;
-    if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* We just hit the next hidden service, stop right now. */
-      break;
-    }
-    /* Number of introduction points. */
-    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
-      config->num_intro_points =
-        (unsigned int) helper_parse_uint64(line->key, line->value,
-                                           NUM_INTRO_POINTS_DEFAULT,
-                                           HS_CONFIG_V3_MAX_INTRO_POINTS,
-                                           &ok);
-      if (!ok || have_num_ip) {
-        if (have_num_ip)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_num_ip = 1;
-      continue;
+  /* Number of introduction points. */
+  if (CHECK_OOB(hs_opts, HiddenServiceNumIntroductionPoints,
+                NUM_INTRO_POINTS_DEFAULT,
+                HS_CONFIG_V3_MAX_INTRO_POINTS)) {
+    goto err;
+  }
+  config->num_intro_points = hs_opts->HiddenServiceNumIntroductionPoints;
+
+  /* Circuit ID export setting. */
+  if (hs_opts->HiddenServiceExportCircuitID) {
+    int ok;
+    config->circuit_id_protocol =
+      helper_parse_circuit_id_protocol("HiddenServcieExportCircuitID",
+                                       hs_opts->HiddenServiceExportCircuitID,
+                                       &ok);
+    if (!ok) {
+      goto err;
     }
-    if (!strcasecmp(line->key, "HiddenServiceExportCircuitID")) {
-      config->circuit_id_protocol =
-        helper_parse_circuit_id_protocol(line->key, line->value, &ok);
-      if (!ok || export_circuit_id) {
-        if (export_circuit_id) {
-          dup_opt_seen = line->key;
-        }
-        goto err;
-      }
-      export_circuit_id = true;
-      continue;
+  }
+
+  /* Is the DoS defense enabled? */
+  config->has_dos_defense_enabled =
+    hs_opts->HiddenServiceEnableIntroDoSDefense;
+
+  /* Rate for DoS defense */
+  if (CHECK_OOB(hs_opts, HiddenServiceEnableIntroDoSRatePerSec,
+                 HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN,
+                 HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX)) {
+    goto err;
+  }
+  config->intro_dos_rate_per_sec =
+    hs_opts->HiddenServiceEnableIntroDoSRatePerSec;
+  log_info(LD_REND, "Service INTRO2 DoS defenses rate set to: %" PRIu32,
+           config->intro_dos_rate_per_sec);
+
+  if (CHECK_OOB(hs_opts, HiddenServiceEnableIntroDoSBurstPerSec,
+                HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN,
+                HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX)) {
+    goto err;
+  }
+  config->intro_dos_burst_per_sec =
+    hs_opts->HiddenServiceEnableIntroDoSBurstPerSec;
+  log_info(LD_REND, "Service INTRO2 DoS defenses burst set to: %" PRIu32,
+           config->intro_dos_burst_per_sec);
+
+  /* Is this an onionbalance instance? */
+  if (hs_opts->HiddenServiceOnionBalanceInstance) {
+    /* Option is enabled, parse config file. */
+    if (! hs_ob_parse_config_file(config)) {
+      goto err;
     }
   }
 
@@ -347,13 +438,10 @@ config_service_v3(const config_line_t *line_,
 
   return 0;
  err:
-  if (dup_opt_seen) {
-    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
-  }
   return -1;
 }
 
-/* Configure a service using the given options in line_ and options. This is
+/** Configure a service using the given options in hs_opts and options. This is
  * called for any service regardless of its version which means that all
  * directives in this function are generic to any service version. This
  * function will also check the validity of the service directory path.
@@ -365,168 +453,98 @@ config_service_v3(const config_line_t *line_,
  *
  * Return 0 on success else -1. */
 static int
-config_generic_service(const config_line_t *line_,
+config_generic_service(const hs_opts_t *hs_opts,
                        const or_options_t *options,
                        hs_service_t *service)
 {
-  int dir_seen = 0;
-  const config_line_t *line;
   hs_service_config_t *config;
-  /* If this is set, we've seen a duplicate of this option. Keep the string
-   * so we can log the directive. */
-  const char *dup_opt_seen = NULL;
-  /* These variables will tell us if we ever have duplicate. */
-  int have_version = 0, have_allow_unknown_ports = 0;
-  int have_dir_group_read = 0, have_max_streams = 0;
-  int have_max_streams_close = 0;
-
-  tor_assert(line_);
+
+  tor_assert(hs_opts);
   tor_assert(options);
   tor_assert(service);
 
   /* Makes thing easier. */
   config = &service->config;
 
-  /* The first line starts with HiddenServiceDir so we consider what's next is
-   * the configuration of the service. */
-  for (line = line_; line ; line = line->next) {
-    int ok = 0;
-
-    /* This indicate that we have a new service to configure. */
-    if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* This function only configures one service at a time so if we've
-       * already seen one, stop right now. */
-      if (dir_seen) {
-        break;
+  /* Directory where the service's keys are stored. */
+  tor_assert(hs_opts->HiddenServiceDir);
+  config->directory_path = tor_strdup(hs_opts->HiddenServiceDir);
+  log_info(LD_CONFIG, "%s=%s. Configuring...",
+           SECTION_HEADER, escaped(config->directory_path));
+
+  /* Protocol version for the service. */
+  if (hs_opts->HiddenServiceVersion == -1) {
+    /* No value was set; stay with the default. */
+  } else if (CHECK_OOB(hs_opts, HiddenServiceVersion,
+                       HS_VERSION_MIN, HS_VERSION_MAX)) {
+    goto err;
+  } else {
+    config->hs_version_explicitly_set = 1;
+    config->version = hs_opts->HiddenServiceVersion;
+  }
+
+  /* Virtual port. */
+  for (const config_line_t *portline = hs_opts->HiddenServicePort;
+       portline; portline = portline->next) {
+    char *err_msg = NULL;
+    /* XXX: Can we rename this? */
+    rend_service_port_config_t *portcfg =
+      rend_service_parse_port_config(portline->value, " ", &err_msg);
+    if (!portcfg) {
+      if (err_msg) {
+        log_warn(LD_CONFIG, "%s", err_msg);
       }
-      /* Ok, we've seen one and we are about to configure it. */
-      dir_seen = 1;
-      config->directory_path = tor_strdup(line->value);
-      log_info(LD_CONFIG, "HiddenServiceDir=%s. Configuring...",
-               escaped(config->directory_path));
-      continue;
-    }
-    if (BUG(!dir_seen)) {
+      tor_free(err_msg);
       goto err;
     }
-    /* Version of the service. */
-    if (!strcasecmp(line->key, "HiddenServiceVersion")) {
-      service->config.version =
-        (uint32_t) helper_parse_uint64(line->key, line->value, HS_VERSION_MIN,
-                                       HS_VERSION_MAX, &ok);
-      if (!ok || have_version) {
-        if (have_version)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_version = service->config.hs_version_explicitly_set = 1;
-      continue;
-    }
-    /* Virtual port. */
-    if (!strcasecmp(line->key, "HiddenServicePort")) {
-      char *err_msg = NULL;
-      /* XXX: Can we rename this? */
-      rend_service_port_config_t *portcfg =
-        rend_service_parse_port_config(line->value, " ", &err_msg);
-      if (!portcfg) {
-        if (err_msg) {
-          log_warn(LD_CONFIG, "%s", err_msg);
-        }
-        tor_free(err_msg);
-        goto err;
-      }
-      tor_assert(!err_msg);
-      smartlist_add(config->ports, portcfg);
-      log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
-               line->value, escaped(config->directory_path));
-      continue;
-    }
-    /* Do we allow unknown ports. */
-    if (!strcasecmp(line->key, "HiddenServiceAllowUnknownPorts")) {
-      config->allow_unknown_ports =
-        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
-      if (!ok || have_allow_unknown_ports) {
-        if (have_allow_unknown_ports)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_allow_unknown_ports = 1;
-      continue;
-    }
-    /* Directory group readable. */
-    if (!strcasecmp(line->key, "HiddenServiceDirGroupReadable")) {
-      config->dir_group_readable =
-        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
-      if (!ok || have_dir_group_read) {
-        if (have_dir_group_read)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_dir_group_read = 1;
-      continue;
-    }
-    /* Maximum streams per circuit. */
-    if (!strcasecmp(line->key, "HiddenServiceMaxStreams")) {
-      config->max_streams_per_rdv_circuit =
-        helper_parse_uint64(line->key, line->value, 0,
-                            HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT, &ok);
-      if (!ok || have_max_streams) {
-        if (have_max_streams)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_max_streams = 1;
-      continue;
-    }
-    /* Maximum amount of streams before we close the circuit. */
-    if (!strcasecmp(line->key, "HiddenServiceMaxStreamsCloseCircuit")) {
-      config->max_streams_close_circuit =
-        (unsigned int) helper_parse_uint64(line->key, line->value, 0, 1, &ok);
-      if (!ok || have_max_streams_close) {
-        if (have_max_streams_close)
-          dup_opt_seen = line->key;
-        goto err;
-      }
-      have_max_streams_close = 1;
-      continue;
-    }
+    tor_assert(!err_msg);
+    smartlist_add(config->ports, portcfg);
+    log_info(LD_CONFIG, "HiddenServicePort=%s for %s",
+             portline->value, escaped(config->directory_path));
+  }
+
+  /* Do we allow unknown ports? */
+  config->allow_unknown_ports = hs_opts->HiddenServiceAllowUnknownPorts;
+
+  /* Directory group readable. */
+  config->dir_group_readable = hs_opts->HiddenServiceDirGroupReadable;
+
+  /* Maximum streams per circuit. */
+  if (CHECK_OOB(hs_opts, HiddenServiceMaxStreams,
+                0, HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT)) {
+    goto err;
   }
+  config->max_streams_per_rdv_circuit = hs_opts->HiddenServiceMaxStreams;
+
+  /* Maximum amount of streams before we close the circuit. */
+  config->max_streams_close_circuit =
+    hs_opts->HiddenServiceMaxStreamsCloseCircuit;
 
   /* Check if we are configured in non anonymous mode meaning every service
    * becomes a single onion service. */
   if (rend_service_non_anonymous_mode_enabled(options)) {
     config->is_single_onion = 1;
-    /* We will add support for IPv6-only v3 single onion services in a future
-     * Tor version. This won't catch "ReachableAddresses reject *4", but that
-     * option doesn't work anyway. */
-    if (options->ClientUseIPv4 == 0 && config->version == HS_VERSION_THREE) {
-      log_warn(LD_CONFIG, "IPv6-only v3 single onion services are not "
-               "supported. Set HiddenServiceSingleHopMode 0 and "
-               "HiddenServiceNonAnonymousMode 0, or set ClientUseIPv4 1.");
-      goto err;
-    }
   }
 
   /* Success */
   return 0;
  err:
-  if (dup_opt_seen) {
-    log_warn(LD_CONFIG, "Duplicate directive %s.", dup_opt_seen);
-  }
   return -1;
 }
 
-/* Configure a service using the given line and options. This function will
+/** Configure a service using the given line and options. This function will
  * call the corresponding configuration function for a specific service
  * version and validate the service against the other ones. On success, add
  * the service to the given list and return 0. On error, nothing is added to
  * the list and a negative value is returned. */
 static int
-config_service(const config_line_t *line, const or_options_t *options,
+config_service(config_line_t *line, const or_options_t *options,
                smartlist_t *service_list)
 {
   int ret;
   hs_service_t *service = NULL;
+  hs_opts_t *hs_opts = NULL;
+  char *msg = NULL;
 
   tor_assert(line);
   tor_assert(options);
@@ -535,9 +553,25 @@ config_service(const config_line_t *line, const or_options_t *options,
   /* We have a new hidden service. */
   service = hs_service_new(options);
 
+  /* Try to validate and parse the configuration lines into 'hs_opts' */
+  hs_opts = hs_opts_new();
+  ret = config_assign(get_hs_opts_mgr(), hs_opts, line, 0, &msg);
+  if (ret < 0) {
+    log_warn(LD_REND, "Can't parse configuration for onion service: %s", msg);
+    goto err;
+  }
+  tor_assert_nonfatal(msg == NULL);
+  validation_status_t vs = config_validate(get_hs_opts_mgr(), NULL,
+                                           hs_opts, &msg);
+  if (vs < 0) {
+    log_warn(LD_REND, "Bad configuration for onion service: %s", msg);
+    goto err;
+  }
+  tor_assert_nonfatal(msg == NULL);
+
   /* We'll configure that service as a generic one and then pass it to a
    * specific function according to the configured version number. */
-  if (config_generic_service(line, options, service) < 0) {
+  if (config_generic_service(hs_opts, options, service) < 0) {
     goto err;
   }
 
@@ -572,10 +606,10 @@ config_service(const config_line_t *line, const or_options_t *options,
    * directory line, the function knows that it has to stop parsing. */
   switch (service->config.version) {
   case HS_VERSION_TWO:
-    ret = rend_config_service(line->next, options, &service->config);
+    ret = rend_config_service(hs_opts, options, &service->config);
     break;
   case HS_VERSION_THREE:
-    ret = config_service_v3(line->next, &service->config);
+    ret = config_service_v3(hs_opts, &service->config);
     break;
   default:
     /* We do validate before if we support the parsed version. */
@@ -594,22 +628,25 @@ config_service(const config_line_t *line, const or_options_t *options,
 
   /* Passes, add it to the given list. */
   smartlist_add(service_list, service);
+  hs_opts_free(hs_opts);
 
   return 0;
 
  err:
   hs_service_free(service);
+  hs_opts_free(hs_opts);
+  tor_free(msg);
   return -1;
 }
 
-/* From a set of <b>options</b>, setup every hidden service found. Return 0 on
+/** From a set of <b>options</b>, setup every hidden service found. Return 0 on
  * success or -1 on failure. If <b>validate_only</b> is set, parse, warn and
  * return as normal, but don't actually change the configured services. */
 int
 hs_config_service_all(const or_options_t *options, int validate_only)
 {
-  int dir_option_seen = 0, ret = -1;
-  const config_line_t *line;
+  int ret = -1;
+  config_line_t *remaining = NULL;
   smartlist_t *new_service_list = NULL;
 
   tor_assert(options);
@@ -618,23 +655,24 @@ hs_config_service_all(const or_options_t *options, int validate_only)
    * validation and staging for >= v3. */
   new_service_list = smartlist_new();
 
-  for (line = options->RendConfigLines; line; line = line->next) {
-    /* Ignore all directives that aren't the start of a service. */
-    if (strcasecmp(line->key, "HiddenServiceDir")) {
-      if (!dir_option_seen) {
-        log_warn(LD_CONFIG, "%s with no preceding HiddenServiceDir directive",
-                 line->key);
-        goto err;
-      }
-      continue;
-    }
-    /* Flag that we've seen a directory directive and we'll use it to make
-     * sure that the torrc options ordering is actually valid. */
-    dir_option_seen = 1;
+  /* We need to start with a HiddenServiceDir line */
+  if (options->RendConfigLines &&
+      strcasecmp(options->RendConfigLines->key, SECTION_HEADER)) {
+    log_warn(LD_CONFIG, "%s with no preceding %s directive",
+             options->RendConfigLines->key, SECTION_HEADER);
+    goto err;
+  }
+
+  remaining = config_lines_dup(options->RendConfigLines);
+  while (remaining) {
+    config_line_t *section = remaining;
+    remaining = config_lines_partition(section, SECTION_HEADER);
 
     /* Try to configure this service now. On success, it will be added to the
      * list and validated against the service in that same list. */
-    if (config_service(line, options, new_service_list) < 0) {
+    int rv = config_service(section, options, new_service_list);
+    config_free_lines(section);
+    if (rv < 0) {
       goto err;
     }
   }
@@ -670,7 +708,7 @@ hs_config_service_all(const or_options_t *options, int validate_only)
   return ret;
 }
 
-/* From a set of <b>options</b>, setup every client authorization found.
+/** From a set of <b>options</b>, setup every client authorization found.
  * Return 0 on success or -1 on failure. If <b>validate_only</b> is set,
  * parse, warn and return as normal, but don't actually change the
  * configured state. */
@@ -694,3 +732,12 @@ hs_config_client_auth_all(const or_options_t *options, int validate_only)
  done:
   return ret;
 }
+
+/**
+ * Free all resources held by the hs_config.c module.
+ **/
+void
+hs_config_free_all(void)
+{
+  config_mgr_free(hs_opts_mgr);
+}
diff --git a/src/feature/hs/hs_config.h b/src/feature/hs/hs_config.h
index 040e451f13..c60b4fbb5d 100644
--- a/src/feature/hs/hs_config.h
+++ b/src/feature/hs/hs_config.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,11 +15,21 @@
 #define HS_CONFIG_MAX_STREAMS_PER_RDV_CIRCUIT 65535
 /* Maximum number of intro points per version 3 services. */
 #define HS_CONFIG_V3_MAX_INTRO_POINTS 20
+/* Default value for the introduction DoS defenses. The MIN/MAX are inclusive
+ * meaning they can be used as valid values. */
+#define HS_CONFIG_V3_DOS_DEFENSE_DEFAULT 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT 25
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX INT32_MAX
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT 200
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN 0
+#define HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX INT32_MAX
 
 /* API */
 
 int hs_config_service_all(const or_options_t *options, int validate_only);
 int hs_config_client_auth_all(const or_options_t *options, int validate_only);
 
-#endif /* !defined(TOR_HS_CONFIG_H) */
+void hs_config_free_all(void);
 
+#endif /* !defined(TOR_HS_CONFIG_H) */
diff --git a/src/feature/hs/hs_control.c b/src/feature/hs/hs_control.c
index 9970fdd123..78b0735c29 100644
--- a/src/feature/hs/hs_control.c
+++ b/src/feature/hs/hs_control.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -7,9 +7,10 @@
  **/
 
 #include "core/or/or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_util.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_control.h"
 #include "feature/hs/hs_descriptor.h"
@@ -19,7 +20,7 @@
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
-/* Send on the control port the "HS_DESC REQUESTED [...]" event.
+/** Send on the control port the "HS_DESC REQUESTED [...]" event.
  *
  * The onion_pk is the onion service public key, base64_blinded_pk is the
  * base64 encoded blinded key for the service and hsdir_rs is the routerstatus
@@ -56,7 +57,7 @@ hs_control_desc_event_requested(const ed25519_public_key_t *onion_pk,
   memwipe(onion_address, 0, sizeof(onion_address));
 }
 
-/* Send on the control port the "HS_DESC FAILED [...]" event.
+/** Send on the control port the "HS_DESC FAILED [...]" event.
  *
  * Using a directory connection identifier, the HSDir identity digest and a
  * reason for the failure. None can be NULL. */
@@ -73,17 +74,14 @@ hs_control_desc_event_failed(const hs_ident_dir_conn_t *ident,
   tor_assert(reason);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_failed(onion_address, base64_blinded_pk,
                                        hsdir_id_digest, reason);
 }
 
-/* Send on the control port the "HS_DESC RECEIVED [...]" event.
+/** Send on the control port the "HS_DESC RECEIVED [...]" event.
  *
  * Using a directory connection identifier and the HSDir identity digest.
  * None can be NULL. */
@@ -98,17 +96,14 @@ hs_control_desc_event_received(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hsv3_descriptor_received(onion_address, base64_blinded_pk,
                                          hsdir_id_digest);
 }
 
-/* Send on the control port the "HS_DESC CREATED [...]" event.
+/** Send on the control port the "HS_DESC CREATED [...]" event.
  *
  * Using the onion address of the descriptor's service and the blinded public
  * key of the descriptor as a descriptor ID. None can be NULL. */
@@ -122,16 +117,14 @@ hs_control_desc_event_created(const char *onion_address,
   tor_assert(blinded_pk);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   /* Version 3 doesn't use the replica number in its descriptor ID computation
    * so we pass negative value so the control port subsystem can ignore it. */
   control_event_hs_descriptor_created(onion_address, base64_blinded_pk, -1);
 }
 
-/* Send on the control port the "HS_DESC UPLOAD [...]" event.
+/** Send on the control port the "HS_DESC UPLOAD [...]" event.
  *
  * Using the onion address of the descriptor's service, the HSDir identity
  * digest, the blinded public key of the descriptor as a descriptor ID and the
@@ -150,9 +143,7 @@ hs_control_desc_event_upload(const char *onion_address,
   tor_assert(hsdir_index);
 
   /* Build base64 encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk, blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, blinded_pk);
 
   control_event_hs_descriptor_upload(onion_address, hsdir_id_digest,
                                      base64_blinded_pk,
@@ -160,7 +151,7 @@ hs_control_desc_event_upload(const char *onion_address,
                                              DIGEST256_LEN));
 }
 
-/* Send on the control port the "HS_DESC UPLOADED [...]" event.
+/** Send on the control port the "HS_DESC UPLOADED [...]" event.
  *
  * Using the directory connection identifier and the HSDir identity digest.
  * None can be NULL. */
@@ -178,7 +169,7 @@ hs_control_desc_event_uploaded(const hs_ident_dir_conn_t *ident,
   control_event_hs_descriptor_uploaded(hsdir_id_digest, onion_address);
 }
 
-/* Send on the control port the "HS_DESC_CONTENT [...]" event.
+/** Send on the control port the "HS_DESC_CONTENT [...]" event.
  *
  * Using the directory connection identifier, the HSDir identity digest and
  * the body of the descriptor (as it was received from the directory). None
@@ -195,17 +186,14 @@ hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
   tor_assert(hsdir_id_digest);
 
   /* Build onion address and encoded blinded key. */
-  IF_BUG_ONCE(ed25519_public_to_base64(base64_blinded_pk,
-                                       &ident->blinded_pk) < 0) {
-    return;
-  }
+  ed25519_public_to_base64(base64_blinded_pk, &ident->blinded_pk);
   hs_build_address(&ident->identity_pk, HS_VERSION_THREE, onion_address);
 
   control_event_hs_descriptor_content(onion_address, base64_blinded_pk,
                                       hsdir_id_digest, body);
 }
 
-/* Handle the "HSPOST [...]" command. The body is an encoded descriptor for
+/** Handle the "HSPOST [...]" command. The body is an encoded descriptor for
  * the given onion_address. The descriptor will be uploaded to each directory
  * in hsdirs_rs. If NULL, the responsible directories for the current time
  * period will be selected.
@@ -259,3 +247,16 @@ hs_control_hspost_command(const char *body, const char *onion_address,
   smartlist_free(hsdirs);
   return ret;
 }
+
+/** With a given <b>onion_identity_pk</b>, fetch its descriptor, optionally
+ * using the list of directory servers given in <b>hsdirs</b>, or a random
+ * server if it is NULL. This function calls hs_client_launch_v3_desc_fetch().
+ */
+void
+hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                           const smartlist_t *hsdirs)
+{
+  tor_assert(onion_identity_pk);
+
+  hs_client_launch_v3_desc_fetch(onion_identity_pk, hsdirs);
+}
diff --git a/src/feature/hs/hs_control.h b/src/feature/hs/hs_control.h
index f7ab642652..947b0ebf1c 100644
--- a/src/feature/hs/hs_control.h
+++ b/src/feature/hs/hs_control.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,5 +48,9 @@ void hs_control_desc_event_content(const hs_ident_dir_conn_t *ident,
 int hs_control_hspost_command(const char *body, const char *onion_address,
                               const smartlist_t *hsdirs_rs);
 
+/* Command "HSFETCH [...]" */
+void hs_control_hsfetch_command(const ed25519_public_key_t *onion_identity_pk,
+                                const smartlist_t *hsdirs);
+
 #endif /* !defined(TOR_HS_CONTROL_H) */
 
diff --git a/src/feature/hs/hs_descriptor.c b/src/feature/hs/hs_descriptor.c
index f74bb97ee2..c1e6553398 100644
--- a/src/feature/hs/hs_descriptor.c
+++ b/src/feature/hs/hs_descriptor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -56,6 +56,7 @@
 #define HS_DESCRIPTOR_PRIVATE
 
 #include "core/or/or.h"
+#include "app/config/config.h"
 #include "trunnel/ed25519_cert.h" /* Trunnel interface. */
 #include "feature/hs/hs_descriptor.h"
 #include "core/or/circuitbuild.h"
@@ -102,7 +103,7 @@
 #define str_desc_auth_client "auth-client"
 #define str_encrypted "encrypted"
 
-/* Authentication supported types. */
+/** Authentication supported types. */
 static const struct {
   hs_desc_auth_type_t type;
   const char *identifier;
@@ -112,7 +113,7 @@ static const struct {
   { 0, NULL }
 };
 
-/* Descriptor ruleset. */
+/** Descriptor ruleset. */
 static token_rule_t hs_desc_v3_token_table[] = {
   T1_START(str_hs_desc, R_HS_DESCRIPTOR, EQ(1), NO_OBJ),
   T1(str_lifetime, R3_DESC_LIFETIME, EQ(1), NO_OBJ),
@@ -123,7 +124,7 @@ static token_rule_t hs_desc_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Descriptor ruleset for the superencrypted section. */
+/** Descriptor ruleset for the superencrypted section. */
 static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
   T1_START(str_desc_auth_type, R3_DESC_AUTH_TYPE, GE(1), NO_OBJ),
   T1(str_desc_auth_key, R3_DESC_AUTH_KEY, GE(1), NO_OBJ),
@@ -132,7 +133,7 @@ static token_rule_t hs_desc_superencrypted_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Descriptor ruleset for the encrypted section. */
+/** Descriptor ruleset for the encrypted section. */
 static token_rule_t hs_desc_encrypted_v3_token_table[] = {
   T1_START(str_create2_formats, R3_CREATE2_FORMATS, CONCAT_ARGS, NO_OBJ),
   T01(str_intro_auth_required, R3_INTRO_AUTH_REQUIRED, GE(1), NO_OBJ),
@@ -140,7 +141,7 @@ static token_rule_t hs_desc_encrypted_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Descriptor ruleset for the introduction points section. */
+/** Descriptor ruleset for the introduction points section. */
 static token_rule_t hs_desc_intro_point_v3_token_table[] = {
   T1_START(str_intro_point, R3_INTRODUCTION_POINT, EQ(1), NO_OBJ),
   T1N(str_ip_onion_key, R3_INTRO_ONION_KEY, GE(2), OBJ_OK),
@@ -152,7 +153,7 @@ static token_rule_t hs_desc_intro_point_v3_token_table[] = {
   END_OF_TABLE
 };
 
-/* Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
+/** Using a key, salt and encrypted payload, build a MAC and put it in mac_out.
  * We use SHA3-256 for the MAC computation.
  * This function can't fail. */
 static void
@@ -184,7 +185,7 @@ build_mac(const uint8_t *mac_key, size_t mac_key_len,
   crypto_digest_free(digest);
 }
 
-/* Using a secret data and a given decriptor object, build the secret
+/** Using a secret data and a given decriptor object, build the secret
  * input needed for the KDF.
  *
  * secret_input = SECRET_DATA | subcredential | INT_8(revision_counter)
@@ -211,7 +212,7 @@ build_secret_input(const hs_descriptor_t *desc,
   memcpy(secret_input, secret_data, secret_data_len);
   offset += secret_data_len;
   /* Copy subcredential. */
-  memcpy(secret_input + offset, desc->subcredential, DIGEST256_LEN);
+  memcpy(secret_input + offset, desc->subcredential.subcred, DIGEST256_LEN);
   offset += DIGEST256_LEN;
   /* Copy revision counter value. */
   set_uint64(secret_input + offset,
@@ -224,7 +225,7 @@ build_secret_input(const hs_descriptor_t *desc,
   return secret_input_len;
 }
 
-/* Do the KDF construction and put the resulting data in key_out which is of
+/** Do the KDF construction and put the resulting data in key_out which is of
  * key_out_len length. It uses SHAKE-256 as specified in the spec. */
 static void
 build_kdf_key(const hs_descriptor_t *desc,
@@ -269,7 +270,7 @@ build_kdf_key(const hs_descriptor_t *desc,
   tor_free(secret_input);
 }
 
-/* Using the given descriptor, secret data, and salt, run it through our
+/** Using the given descriptor, secret data, and salt, run it through our
  * KDF function and then extract a secret key in key_out, the IV in iv_out
  * and MAC in mac_out. This function can't fail. */
 static void
@@ -308,7 +309,7 @@ build_secret_key_iv_mac(const hs_descriptor_t *desc,
 
 /* === ENCODING === */
 
-/* Encode the given link specifier objects into a newly allocated string.
+/** Encode the given link specifier objects into a newly allocated string.
  * This can't fail so caller can always assume a valid string being
  * returned. */
 STATIC char *
@@ -324,12 +325,11 @@ encode_link_specifiers(const smartlist_t *specs)
 
   link_specifier_list_set_n_spec(lslist, smartlist_len(specs));
 
-  SMARTLIST_FOREACH_BEGIN(specs, const hs_desc_link_specifier_t *,
+  SMARTLIST_FOREACH_BEGIN(specs, const link_specifier_t *,
                           spec) {
-    link_specifier_t *ls = hs_desc_lspec_to_trunnel(spec);
-    if (ls) {
-      link_specifier_list_add_spec(lslist, ls);
-    }
+    link_specifier_t *ls = link_specifier_dup(spec);
+    tor_assert(ls);
+    link_specifier_list_add_spec(lslist, ls);
   } SMARTLIST_FOREACH_END(spec);
 
   {
@@ -356,7 +356,7 @@ encode_link_specifiers(const smartlist_t *specs)
   return encoded_b64;
 }
 
-/* Encode an introduction point legacy key and certificate. Return a newly
+/** Encode an introduction point legacy key and certificate. Return a newly
  * allocated string with it. On failure, return NULL. */
 static char *
 encode_legacy_key(const hs_desc_intro_point_t *ip)
@@ -393,7 +393,7 @@ encode_legacy_key(const hs_desc_intro_point_t *ip)
   return encoded;
 }
 
-/* Encode an introduction point encryption key and certificate. Return a newly
+/** Encode an introduction point encryption key and certificate. Return a newly
  * allocated string with it. On failure, return NULL. */
 static char *
 encode_enc_key(const hs_desc_intro_point_t *ip)
@@ -404,9 +404,7 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "enc-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->enc_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->enc_key);
   if (tor_cert_encode_ed22519(ip->enc_key_cert, &encoded_cert) < 0) {
     goto done;
   }
@@ -421,8 +419,8 @@ encode_enc_key(const hs_desc_intro_point_t *ip)
   return encoded;
 }
 
-/* Encode an introduction point onion key. Return a newly allocated string
- * with it. On failure, return NULL. */
+/** Encode an introduction point onion key. Return a newly allocated string
+ * with it. Can not fail. */
 static char *
 encode_onion_key(const hs_desc_intro_point_t *ip)
 {
@@ -432,16 +430,13 @@ encode_onion_key(const hs_desc_intro_point_t *ip)
   tor_assert(ip);
 
   /* Base64 encode the encryption key for the "onion-key" field. */
-  if (curve25519_public_to_base64(key_b64, &ip->onion_key) < 0) {
-    goto done;
-  }
+  curve25519_public_to_base64(key_b64, &ip->onion_key);
   tor_asprintf(&encoded, "%s ntor %s", str_ip_onion_key, key_b64);
 
- done:
   return encoded;
 }
 
-/* Encode an introduction point object and return a newly allocated string
+/** Encode an introduction point object and return a newly allocated string
  * with it. On failure, return NULL. */
 static char *
 encode_intro_point(const ed25519_public_key_t *sig_key,
@@ -511,7 +506,7 @@ encode_intro_point(const ed25519_public_key_t *sig_key,
   return encoded_ip;
 }
 
-/* Given a source length, return the new size including padding for the
+/** Given a source length, return the new size including padding for the
  * plaintext encryption. */
 static size_t
 compute_padded_plaintext_length(size_t plaintext_len)
@@ -531,7 +526,7 @@ compute_padded_plaintext_length(size_t plaintext_len)
   return plaintext_padded_len;
 }
 
-/* Given a buffer, pad it up to the encrypted section padding requirement. Set
+/** Given a buffer, pad it up to the encrypted section padding requirement. Set
  * the newly allocated string in padded_out and return the length of the
  * padded buffer. */
 STATIC size_t
@@ -554,7 +549,7 @@ build_plaintext_padding(const char *plaintext, size_t plaintext_len,
   return padded_len;
 }
 
-/* Using a key, IV and plaintext data of length plaintext_len, create the
+/** Using a key, IV and plaintext data of length plaintext_len, create the
  * encrypted section by encrypting it and setting encrypted_out with the
  * data. Return size of the encrypted data buffer. */
 static size_t
@@ -599,7 +594,7 @@ build_encrypted(const uint8_t *key, const uint8_t *iv, const char *plaintext,
   return encrypted_len;
 }
 
-/* Encrypt the given <b>plaintext</b> buffer using <b>desc</b> and
+/** Encrypt the given <b>plaintext</b> buffer using <b>desc</b> and
  * <b>secret_data</b> to get the keys. Set encrypted_out with the encrypted
  * data and return the length of it. <b>is_superencrypted_layer</b> is set
  * if this is the outer encrypted layer of the descriptor. */
@@ -669,7 +664,7 @@ encrypt_descriptor_data(const hs_descriptor_t *desc,
   return final_blob_len;
 }
 
-/* Create and return a string containing a client-auth entry. It's the
+/** Create and return a string containing a client-auth entry. It's the
  * responsibility of the caller to free the returned string. This function
  * will never fail. */
 static char *
@@ -684,7 +679,7 @@ get_auth_client_str(const hs_desc_authorized_client_t *client)
   char encrypted_cookie_b64[HS_DESC_ENCRYPED_COOKIE_LEN * 2];
 
 #define ASSERT_AND_BASE64(field) STMT_BEGIN                        \
-  tor_assert(!tor_mem_is_zero((char *) client->field,              \
+  tor_assert(!fast_mem_is_zero((char *) client->field,              \
                               sizeof(client->field)));             \
   ret = base64_encode_nopad(field##_b64, sizeof(field##_b64),      \
                             client->field, sizeof(client->field)); \
@@ -739,7 +734,7 @@ get_all_auth_client_lines(const hs_descriptor_t *desc)
   return auth_client_lines_str;
 }
 
-/* Create the inner layer of the descriptor (which includes the intro points,
+/** Create the inner layer of the descriptor (which includes the intro points,
  * etc.). Return a newly-allocated string with the layer plaintext, or NULL if
  * an error occurred. It's the responsibility of the caller to free the
  * returned string. */
@@ -795,11 +790,11 @@ get_inner_encrypted_layer_plaintext(const hs_descriptor_t *desc)
   return encoded_str;
 }
 
-/* Create the middle layer of the descriptor, which includes the client auth
+/** Create the middle layer of the descriptor, which includes the client auth
  * data and the encrypted inner layer (provided as a base64 string at
  * <b>layer2_b64_ciphertext</b>). Return a newly-allocated string with the
- * layer plaintext, or NULL if an error occurred. It's the responsibility of
- * the caller to free the returned string. */
+ * layer plaintext. It's the responsibility of the caller to free the returned
+ * string. Can not fail. */
 static char *
 get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
                                     const char *layer2_b64_ciphertext)
@@ -815,13 +810,10 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
     const curve25519_public_key_t *ephemeral_pubkey;
 
     ephemeral_pubkey = &desc->superencrypted_data.auth_ephemeral_pubkey;
-    tor_assert(!tor_mem_is_zero((char *) ephemeral_pubkey->public_key,
+    tor_assert(!fast_mem_is_zero((char *) ephemeral_pubkey->public_key,
                                 CURVE25519_PUBKEY_LEN));
 
-    if (curve25519_public_to_base64(ephemeral_key_base64,
-                                    ephemeral_pubkey) < 0) {
-      goto done;
-    }
+    curve25519_public_to_base64(ephemeral_key_base64, ephemeral_pubkey);
     smartlist_add_asprintf(lines, "%s %s\n",
                            str_desc_auth_key, ephemeral_key_base64);
 
@@ -846,7 +838,6 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
 
   layer1_str = smartlist_join_strings(lines, "", 0, NULL);
 
- done:
   /* We need to memwipe all lines because it contains the ephemeral key */
   SMARTLIST_FOREACH(lines, char *, a, memwipe(a, 0, strlen(a)));
   SMARTLIST_FOREACH(lines, char *, a, tor_free(a));
@@ -855,7 +846,7 @@ get_outer_encrypted_layer_plaintext(const hs_descriptor_t *desc,
   return layer1_str;
 }
 
-/* Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
+/** Encrypt <b>encoded_str</b> into an encrypted blob and then base64 it before
  * returning it. <b>desc</b> is provided to derive the encryption
  * keys. <b>secret_data</b> is also proved to derive the encryption keys.
  * <b>is_superencrypted_layer</b> is set if <b>encoded_str</b> is the
@@ -888,7 +879,7 @@ encrypt_desc_data_and_base64(const hs_descriptor_t *desc,
   return enc_b64;
 }
 
-/* Generate the secret data which is used to encrypt/decrypt the descriptor.
+/** Generate the secret data which is used to encrypt/decrypt the descriptor.
  *
  * SECRET_DATA = blinded-public-key
  * SECRET_DATA = blinded-public-key | descriptor_cookie
@@ -935,7 +926,7 @@ build_secret_data(const ed25519_public_key_t *blinded_pubkey,
   return secret_data_len;
 }
 
-/* Generate and encode the superencrypted portion of <b>desc</b>. This also
+/** Generate and encode the superencrypted portion of <b>desc</b>. This also
  * involves generating the encrypted portion of the descriptor, and performing
  * the superencryption. A newly allocated NUL-terminated string pointer
  * containing the encrypted encoded blob is put in encrypted_blob_out. Return 0
@@ -1009,7 +1000,7 @@ encode_superencrypted_data(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
+/** Encode a v3 HS descriptor. Return 0 on success and set encoded_out to the
  * newly allocated string of the encoded descriptor. On error, -1 is returned
  * and encoded_out is untouched. */
 static int
@@ -1028,10 +1019,6 @@ desc_encode_v3(const hs_descriptor_t *desc,
   tor_assert(encoded_out);
   tor_assert(desc->plaintext_data.version == 3);
 
-  if (BUG(desc->subcredential == NULL)) {
-    goto err;
-  }
-
   /* Build the non-encrypted values. */
   {
     char *encoded_cert;
@@ -1092,11 +1079,7 @@ desc_encode_v3(const hs_descriptor_t *desc,
       tor_free(encoded_str);
       goto err;
     }
-    if (ed25519_signature_to_base64(ed_sig_b64, &sig) < 0) {
-      log_warn(LD_BUG, "Can't base64 encode descriptor signature!");
-      tor_free(encoded_str);
-      goto err;
-    }
+    ed25519_signature_to_base64(ed_sig_b64, &sig);
     /* Create the signature line. */
     smartlist_add_asprintf(lines, "%s %s", str_signature, ed_sig_b64);
   }
@@ -1125,7 +1108,7 @@ desc_encode_v3(const hs_descriptor_t *desc,
 
 /* === DECODING === */
 
-/* Given the token tok for an auth client, decode it as
+/** Given the token tok for an auth client, decode it as
  * hs_desc_authorized_client_t. tok->args MUST contain at least 3 elements
  * Return 0 on success else -1 on failure. */
 static int
@@ -1161,7 +1144,7 @@ decode_auth_client(const directory_token_t *tok,
   return ret;
 }
 
-/* Given an encoded string of the link specifiers, return a newly allocated
+/** Given an encoded string of the link specifiers, return a newly allocated
  * list of decoded link specifiers. Return NULL on error. */
 STATIC smartlist_t *
 decode_link_specifiers(const char *encoded)
@@ -1190,52 +1173,22 @@ decode_link_specifiers(const char *encoded)
   results = smartlist_new();
 
   for (i = 0; i < link_specifier_list_getlen_spec(specs); i++) {
-    hs_desc_link_specifier_t *hs_spec;
     link_specifier_t *ls = link_specifier_list_get_spec(specs, i);
-    tor_assert(ls);
-
-    hs_spec = tor_malloc_zero(sizeof(*hs_spec));
-    hs_spec->type = link_specifier_get_ls_type(ls);
-    switch (hs_spec->type) {
-    case LS_IPV4:
-      tor_addr_from_ipv4h(&hs_spec->u.ap.addr,
-                          link_specifier_get_un_ipv4_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv4_port(ls);
-      break;
-    case LS_IPV6:
-      tor_addr_from_ipv6_bytes(&hs_spec->u.ap.addr, (const char *)
-                               link_specifier_getarray_un_ipv6_addr(ls));
-      hs_spec->u.ap.port = link_specifier_get_un_ipv6_port(ls);
-      break;
-    case LS_LEGACY_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_legacy_id(ls) ==
-                 sizeof(hs_spec->u.legacy_id));
-      memcpy(hs_spec->u.legacy_id, link_specifier_getarray_un_legacy_id(ls),
-             sizeof(hs_spec->u.legacy_id));
-      break;
-    case LS_ED25519_ID:
-      /* Both are known at compile time so let's make sure they are the same
-       * else we can copy memory out of bound. */
-      tor_assert(link_specifier_getlen_un_ed25519_id(ls) ==
-                 sizeof(hs_spec->u.ed25519_id));
-      memcpy(hs_spec->u.ed25519_id,
-             link_specifier_getconstarray_un_ed25519_id(ls),
-             sizeof(hs_spec->u.ed25519_id));
-      break;
-    default:
-      tor_free(hs_spec);
+    if (BUG(!ls)) {
       goto err;
     }
-
-    smartlist_add(results, hs_spec);
+    link_specifier_t *ls_dup = link_specifier_dup(ls);
+    if (BUG(!ls_dup)) {
+      goto err;
+    }
+    smartlist_add(results, ls_dup);
   }
 
   goto done;
  err:
   if (results) {
-    SMARTLIST_FOREACH(results, hs_desc_link_specifier_t *, s, tor_free(s));
+    SMARTLIST_FOREACH(results, link_specifier_t *, s,
+                      link_specifier_free(s));
     smartlist_free(results);
     results = NULL;
   }
@@ -1245,7 +1198,7 @@ decode_link_specifiers(const char *encoded)
   return results;
 }
 
-/* Given a list of authentication types, decode it and put it in the encrypted
+/** Given a list of authentication types, decode it and put it in the encrypted
  * data section. Return 1 if we at least know one of the type or 0 if we know
  * none of them. */
 static int
@@ -1273,7 +1226,7 @@ decode_auth_type(hs_desc_encrypted_data_t *desc, const char *list)
   return match;
 }
 
-/* Parse a space-delimited list of integers representing CREATE2 formats into
+/** Parse a space-delimited list of integers representing CREATE2 formats into
  * the bitfield in hs_desc_encrypted_data_t. Ignore unrecognized values. */
 static void
 decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
@@ -1307,7 +1260,7 @@ decode_create2_list(hs_desc_encrypted_data_t *desc, const char *list)
   smartlist_free(tokens);
 }
 
-/* Given a certificate, validate the certificate for certain conditions which
+/** Given a certificate, validate the certificate for certain conditions which
  * are if the given type matches the cert's one, if the signing key is
  * included and if the that key was actually used to sign the certificate.
  *
@@ -1331,11 +1284,20 @@ cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
     log_warn(LD_REND, "Signing key is NOT included for %s.", log_obj_type);
     goto err;
   }
+
   /* The following will not only check if the signature matches but also the
    * expiration date and overall validity. */
   if (tor_cert_checksig(cert, &cert->signing_key, approx_time()) < 0) {
-    log_warn(LD_REND, "Invalid signature for %s: %s", log_obj_type,
-             tor_cert_describe_signature_status(cert));
+    if (cert->cert_expired) {
+      char expiration_str[ISO_TIME_LEN+1];
+      format_iso_time(expiration_str, cert->valid_until);
+      log_fn(LOG_PROTOCOL_WARN, LD_REND, "Invalid signature for %s: %s (%s)",
+             log_obj_type, tor_cert_describe_signature_status(cert),
+             expiration_str);
+    } else {
+      log_warn(LD_REND, "Invalid signature for %s: %s",
+               log_obj_type, tor_cert_describe_signature_status(cert));
+    }
     goto err;
   }
 
@@ -1344,7 +1306,7 @@ cert_is_valid(tor_cert_t *cert, uint8_t type, const char *log_obj_type)
   return 0;
 }
 
-/* Given some binary data, try to parse it to get a certificate object. If we
+/** Given some binary data, try to parse it to get a certificate object. If we
  * have a valid cert, validate it using the given wanted type. On error, print
  * a log using the err_msg has the certificate identifier adding semantic to
  * the log and cert_out is set to NULL. On success, 0 is returned and cert_out
@@ -1381,7 +1343,7 @@ cert_parse_and_validate(tor_cert_t **cert_out, const char *data,
   return -1;
 }
 
-/* Return true iff the given length of the encrypted data of a descriptor
+/** Return true iff the given length of the encrypted data of a descriptor
  * passes validation. */
 STATIC int
 encrypted_data_length_is_valid(size_t len)
@@ -1400,7 +1362,50 @@ encrypted_data_length_is_valid(size_t len)
   return 0;
 }
 
-/* Decrypt the descriptor cookie given the descriptor, the auth client,
+/** Build the KEYS component for the authorized client computation. The format
+ * of the construction is:
+ *
+ *    SECRET_SEED = x25519(sk, pk)
+ *    KEYS = KDF(subcredential | SECRET_SEED, 40)
+ *
+ * Set the <b>keys_out</b> argument to point to the buffer containing the KEYS,
+ * and return the buffer's length. The caller should wipe and free its content
+ * once done with it. This function can't fail. */
+static size_t
+build_descriptor_cookie_keys(const hs_subcredential_t *subcredential,
+                             const curve25519_secret_key_t *sk,
+                             const curve25519_public_key_t *pk,
+                             uint8_t **keys_out)
+{
+  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
+  uint8_t *keystream;
+  size_t keystream_len = HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN;
+  crypto_xof_t *xof;
+
+  tor_assert(subcredential);
+  tor_assert(sk);
+  tor_assert(pk);
+  tor_assert(keys_out);
+
+  keystream = tor_malloc_zero(keystream_len);
+
+  /* Calculate x25519(sk, pk) to get the secret seed. */
+  curve25519_handshake(secret_seed, sk, pk);
+
+  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
+  xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, subcredential->subcred, SUBCRED_LEN);
+  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
+  crypto_xof_squeeze_bytes(xof, keystream, keystream_len);
+  crypto_xof_free(xof);
+
+  memwipe(secret_seed, 0, sizeof(secret_seed));
+
+  *keys_out = keystream;
+  return keystream_len;
+}
+
+/** Decrypt the descriptor cookie given the descriptor, the auth client,
  * and the client secret key. On sucess, return 0 and a newly allocated
  * descriptor cookie descriptor_cookie_out. On error or if the client id
  * is invalid, return -1 and descriptor_cookie_out is set to
@@ -1412,33 +1417,34 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
                           uint8_t **descriptor_cookie_out)
 {
   int ret = -1;
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key = NULL;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
   uint8_t *descriptor_cookie = NULL;
+  const uint8_t *cookie_key = NULL;
   crypto_cipher_t *cipher = NULL;
-  crypto_xof_t *xof = NULL;
 
   tor_assert(desc);
   tor_assert(client);
   tor_assert(client_auth_sk);
-  tor_assert(!tor_mem_is_zero(
+  tor_assert(!fast_mem_is_zero(
         (char *) &desc->superencrypted_data.auth_ephemeral_pubkey,
         sizeof(desc->superencrypted_data.auth_ephemeral_pubkey)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_sk,
-                              sizeof(*client_auth_sk)));
-  tor_assert(!tor_mem_is_zero((char *) desc->subcredential, DIGEST256_LEN));
+  tor_assert(!fast_mem_is_zero((char *) desc->subcredential.subcred,
+                               DIGEST256_LEN));
 
-  /* Calculate x25519(client_x, hs_Y) */
-  curve25519_handshake(secret_seed, client_auth_sk,
-                       &desc->superencrypted_data.auth_ephemeral_pubkey);
+  /* Catch potential code-flow cases of an unitialized private key sneaking
+   * into this function. */
+  if (BUG(fast_mem_is_zero((char *)client_auth_sk, sizeof(*client_auth_sk)))) {
+    goto done;
+  }
 
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, desc->subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS component to derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(&desc->subcredential,
+                             client_auth_sk,
+                             &desc->superencrypted_data.auth_ephemeral_pubkey,
+                             &keystream);
+  tor_assert(keystream_length > 0);
 
   /* If the client id of auth client is not the same as the calculcated
    * client id, it means that this auth client is invaild according to the
@@ -1464,8 +1470,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
   if (cipher) {
     crypto_cipher_free(cipher);
   }
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
   return ret;
 }
 
@@ -1481,10 +1487,8 @@ decrypt_descriptor_cookie(const hs_descriptor_t *desc,
  */
 MOCK_IMPL(STATIC size_t,
 decrypt_desc_layer,(const hs_descriptor_t *desc,
-                    const uint8_t *encrypted_blob,
-                    size_t encrypted_blob_size,
                     const uint8_t *descriptor_cookie,
-                    int is_superencrypted_layer,
+                    bool is_superencrypted_layer,
                     char **decrypted_out))
 {
   uint8_t *decrypted = NULL;
@@ -1494,6 +1498,12 @@ decrypt_desc_layer,(const hs_descriptor_t *desc,
   uint8_t mac_key[DIGEST256_LEN], our_mac[DIGEST256_LEN];
   const uint8_t *salt, *encrypted, *desc_mac;
   size_t encrypted_len, result_len = 0;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
 
   tor_assert(decrypted_out);
   tor_assert(desc);
@@ -1592,7 +1602,7 @@ decrypt_desc_layer,(const hs_descriptor_t *desc,
   return result_len;
 }
 
-/* Decrypt the superencrypted section of the descriptor using the given
+/** Decrypt the superencrypted section of the descriptor using the given
  * descriptor object <b>desc</b>. A newly allocated NUL terminated string is
  * put in decrypted_out which contains the superencrypted layer of the
  * descriptor. Return the length of decrypted_out on success else 0 is
@@ -1607,9 +1617,8 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
   tor_assert(decrypted_out);
 
   superencrypted_len = decrypt_desc_layer(desc,
-                                 desc->plaintext_data.superencrypted_blob,
-                                 desc->plaintext_data.superencrypted_blob_size,
-                                 NULL, 1, &superencrypted_plaintext);
+                                          NULL,
+                                          true, &superencrypted_plaintext);
 
   if (!superencrypted_len) {
     log_warn(LD_REND, "Decrypting superencrypted desc failed.");
@@ -1625,7 +1634,7 @@ desc_decrypt_superencrypted(const hs_descriptor_t *desc, char **decrypted_out)
   return superencrypted_len;
 }
 
-/* Decrypt the encrypted section of the descriptor using the given descriptor
+/** Decrypt the encrypted section of the descriptor using the given descriptor
  * object <b>desc</b>. A newly allocated NUL terminated string is put in
  * decrypted_out which contains the encrypted layer of the descriptor.
  * Return the length of decrypted_out on success else 0 is returned and
@@ -1658,9 +1667,9 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc,
   }
 
   encrypted_len = decrypt_desc_layer(desc,
-                                 desc->superencrypted_data.encrypted_blob,
-                                 desc->superencrypted_data.encrypted_blob_size,
-                                 descriptor_cookie, 0, &encrypted_plaintext);
+                                     descriptor_cookie,
+                                     false, &encrypted_plaintext);
+
   if (!encrypted_len) {
     goto err;
   }
@@ -1678,7 +1687,7 @@ desc_decrypt_encrypted(const hs_descriptor_t *desc,
   return encrypted_len;
 }
 
-/* Given the token tok for an intro point legacy key, the list of tokens, the
+/** Given the token tok for an intro point legacy key, the list of tokens, the
  * introduction point ip being decoded and the descriptor desc from which it
  * comes from, decode the legacy key and set the intro point object. Return 0
  * on success else -1 on failure. */
@@ -1736,7 +1745,7 @@ decode_intro_legacy_key(const directory_token_t *tok,
   return -1;
 }
 
-/* Dig into the descriptor <b>tokens</b> to find the onion key we should use
+/** Dig into the descriptor <b>tokens</b> to find the onion key we should use
  * for this intro point, and set it into <b>onion_key_out</b>. Return 0 if it
  * was found and well-formed, otherwise return -1 in case of errors. */
 static int
@@ -1780,7 +1789,7 @@ set_intro_point_onion_key(curve25519_public_key_t *onion_key_out,
   return retval;
 }
 
-/* Given the start of a section and the end of it, decode a single
+/** Given the start of a section and the end of it, decode a single
  * introduction point from that section. Return a newly allocated introduction
  * point object containing the decoded data. Return NULL if the section can't
  * be decoded. */
@@ -1909,7 +1918,7 @@ decode_introduction_point(const hs_descriptor_t *desc, const char *start)
   return ip;
 }
 
-/* Given a descriptor string at <b>data</b>, decode all possible introduction
+/** Given a descriptor string at <b>data</b>, decode all possible introduction
  * points that we can find. Add the introduction point object to desc_enc as we
  * find them. This function can't fail and it is possible that zero
  * introduction points can be decoded. */
@@ -1972,7 +1981,8 @@ decode_intro_points(const hs_descriptor_t *desc,
   SMARTLIST_FOREACH(intro_points, char *, a, tor_free(a));
   smartlist_free(intro_points);
 }
-/* Return 1 iff the given base64 encoded signature in b64_sig from the encoded
+
+/** Return 1 iff the given base64 encoded signature in b64_sig from the encoded
  * descriptor in encoded_desc validates the descriptor content. */
 STATIC int
 desc_sig_is_valid(const char *b64_sig,
@@ -2031,14 +2041,14 @@ desc_sig_is_valid(const char *b64_sig,
   return ret;
 }
 
-/* Decode descriptor plaintext data for version 3. Given a list of tokens, an
+/** Decode descriptor plaintext data for version 3. Given a list of tokens, an
  * allocated plaintext object that will be populated and the encoded
  * descriptor with its length. The last one is needed for signature
  * verification. Unknown tokens are simply ignored so this won't error on
  * unknowns but requires that all v3 token be present and valid.
  *
  * Return 0 on success else a negative value. */
-static int
+static hs_desc_decode_status_t
 desc_decode_plaintext_v3(smartlist_t *tokens,
                          hs_desc_plaintext_data_t *desc,
                          const char *encoded_desc, size_t encoded_len)
@@ -2128,21 +2138,19 @@ desc_decode_plaintext_v3(smartlist_t *tokens,
     goto err;
   }
 
-  return 0;
-
+  return HS_DESC_DECODE_OK;
  err:
-  return -1;
+  return HS_DESC_DECODE_PLAINTEXT_ERROR;
 }
 
-/* Decode the version 3 superencrypted section of the given descriptor desc.
- * The desc_superencrypted_out will be populated with the decoded data.
- * Return 0 on success else -1. */
-static int
+/** Decode the version 3 superencrypted section of the given descriptor desc.
+ * The desc_superencrypted_out will be populated with the decoded data. */
+static hs_desc_decode_status_t
 desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
                               hs_desc_superencrypted_data_t *
                               desc_superencrypted_out)
 {
-  int ret = -1;
+  int ret = HS_DESC_DECODE_SUPERENC_ERROR;
   char *message = NULL;
   size_t message_len;
   memarea_t *area = NULL;
@@ -2228,11 +2236,11 @@ desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
                                               tok->object_size);
   superencrypted->encrypted_blob_size = tok->object_size;
 
-  ret = 0;
+  ret = HS_DESC_DECODE_OK;
   goto done;
 
  err:
-  tor_assert(ret < 0);
+  tor_assert(ret < HS_DESC_DECODE_OK);
   hs_desc_superencrypted_data_free_contents(desc_superencrypted_out);
 
  done:
@@ -2249,15 +2257,14 @@ desc_decode_superencrypted_v3(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Decode the version 3 encrypted section of the given descriptor desc. The
- * desc_encrypted_out will be populated with the decoded data. Return 0 on
- * success else -1. */
-static int
+/** Decode the version 3 encrypted section of the given descriptor desc. The
+ * desc_encrypted_out will be populated with the decoded data. */
+static hs_desc_decode_status_t
 desc_decode_encrypted_v3(const hs_descriptor_t *desc,
                          const curve25519_secret_key_t *client_auth_sk,
                          hs_desc_encrypted_data_t *desc_encrypted_out)
 {
-  int ret = -1;
+  int ret = HS_DESC_DECODE_ENCRYPTED_ERROR;
   char *message = NULL;
   size_t message_len;
   memarea_t *area = NULL;
@@ -2280,12 +2287,14 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc,
        * authorization is failing. */
       log_warn(LD_REND, "Client authorization for requested onion address "
                         "is invalid. Can't decrypt the descriptor.");
+      ret = HS_DESC_DECODE_BAD_CLIENT_AUTH;
     } else {
       /* Inform at notice level that the onion address requested can't be
        * reached without client authorization most likely. */
       log_notice(LD_REND, "Fail to decrypt descriptor for requested onion "
                         "address. It is likely requiring client "
                         "authorization.");
+      ret = HS_DESC_DECODE_NEED_CLIENT_AUTH;
     }
     goto err;
   }
@@ -2344,11 +2353,11 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc,
   /* NOTE: Unknown fields are allowed because this function could be used to
    * decode other descriptor version. */
 
-  ret = 0;
+  ret = HS_DESC_DECODE_OK;
   goto done;
 
  err:
-  tor_assert(ret < 0);
+  tor_assert(ret < HS_DESC_DECODE_OK);
   hs_desc_encrypted_data_free_contents(desc_encrypted_out);
 
  done:
@@ -2365,9 +2374,9 @@ desc_decode_encrypted_v3(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Table of encrypted decode function version specific. The function are
+/** Table of encrypted decode function version specific. The function are
  * indexed by the version number so v3 callback is at index 3 in the array. */
-static int
+static hs_desc_decode_status_t
   (*decode_encrypted_handlers[])(
       const hs_descriptor_t *desc,
       const curve25519_secret_key_t *client_auth_sk,
@@ -2377,15 +2386,15 @@ static int
   desc_decode_encrypted_v3,
 };
 
-/* Decode the encrypted data section of the given descriptor and store the
+/** Decode the encrypted data section of the given descriptor and store the
  * data in the given encrypted data object. Return 0 on success else a
  * negative value on error. */
-int
+hs_desc_decode_status_t
 hs_desc_decode_encrypted(const hs_descriptor_t *desc,
                          const curve25519_secret_key_t *client_auth_sk,
                          hs_desc_encrypted_data_t *desc_encrypted)
 {
-  int ret;
+  int ret = HS_DESC_DECODE_ENCRYPTED_ERROR;
   uint32_t version;
 
   tor_assert(desc);
@@ -2399,7 +2408,6 @@ hs_desc_decode_encrypted(const hs_descriptor_t *desc,
   /* Let's make sure we have a supported version as well. By correctly parsing
    * the plaintext, this should not fail. */
   if (BUG(!hs_desc_is_supported_version(version))) {
-    ret = -1;
     goto err;
   }
   /* Extra precaution. Having no handler for the supported version should
@@ -2418,9 +2426,9 @@ hs_desc_decode_encrypted(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Table of superencrypted decode function version specific. The function are
+/** Table of superencrypted decode function version specific. The function are
  * indexed by the version number so v3 callback is at index 3 in the array. */
-static int
+static hs_desc_decode_status_t
   (*decode_superencrypted_handlers[])(
       const hs_descriptor_t *desc,
       hs_desc_superencrypted_data_t *desc_superencrypted) =
@@ -2429,15 +2437,14 @@ static int
   desc_decode_superencrypted_v3,
 };
 
-/* Decode the superencrypted data section of the given descriptor and store the
- * data in the given superencrypted data object. Return 0 on success else a
- * negative value on error. */
-int
+/** Decode the superencrypted data section of the given descriptor and store
+ * the data in the given superencrypted data object. */
+hs_desc_decode_status_t
 hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
                               hs_desc_superencrypted_data_t *
                               desc_superencrypted)
 {
-  int ret;
+  int ret = HS_DESC_DECODE_SUPERENC_ERROR;
   uint32_t version;
 
   tor_assert(desc);
@@ -2451,7 +2458,6 @@ hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
   /* Let's make sure we have a supported version as well. By correctly parsing
    * the plaintext, this should not fail. */
   if (BUG(!hs_desc_is_supported_version(version))) {
-    ret = -1;
     goto err;
   }
   /* Extra precaution. Having no handler for the supported version should
@@ -2469,9 +2475,9 @@ hs_desc_decode_superencrypted(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Table of plaintext decode function version specific. The function are
+/** Table of plaintext decode function version specific. The function are
  * indexed by the version number so v3 callback is at index 3 in the array. */
-static int
+static hs_desc_decode_status_t
   (*decode_plaintext_handlers[])(
       smartlist_t *tokens,
       hs_desc_plaintext_data_t *desc,
@@ -2482,13 +2488,13 @@ static int
   desc_decode_plaintext_v3,
 };
 
-/* Fully decode the given descriptor plaintext and store the data in the
- * plaintext data object. Returns 0 on success else a negative value. */
-int
+/** Fully decode the given descriptor plaintext and store the data in the
+ * plaintext data object. */
+hs_desc_decode_status_t
 hs_desc_decode_plaintext(const char *encoded,
                          hs_desc_plaintext_data_t *plaintext)
 {
-  int ok = 0, ret = -1;
+  int ok = 0, ret = HS_DESC_DECODE_PLAINTEXT_ERROR;
   memarea_t *area = NULL;
   smartlist_t *tokens = NULL;
   size_t encoded_len;
@@ -2538,11 +2544,11 @@ hs_desc_decode_plaintext(const char *encoded,
   /* Run the version specific plaintext decoder. */
   ret = decode_plaintext_handlers[plaintext->version](tokens, plaintext,
                                                       encoded, encoded_len);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
   /* Success. Descriptor has been populated with the data. */
-  ret = 0;
+  ret = HS_DESC_DECODE_OK;
 
  err:
   if (tokens) {
@@ -2555,19 +2561,19 @@ hs_desc_decode_plaintext(const char *encoded,
   return ret;
 }
 
-/* Fully decode an encoded descriptor and set a newly allocated descriptor
+/** Fully decode an encoded descriptor and set a newly allocated descriptor
  * object in desc_out.  Client secret key is used to decrypt the "encrypted"
  * section if not NULL else it's ignored.
  *
  * Return 0 on success. A negative value is returned on error and desc_out is
  * set to NULL. */
-int
+hs_desc_decode_status_t
 hs_desc_decode_descriptor(const char *encoded,
-                          const uint8_t *subcredential,
+                          const hs_subcredential_t *subcredential,
                           const curve25519_secret_key_t *client_auth_sk,
                           hs_descriptor_t **desc_out)
 {
-  int ret = -1;
+  hs_desc_decode_status_t ret = HS_DESC_DECODE_GENERIC_ERROR;
   hs_descriptor_t *desc;
 
   tor_assert(encoded);
@@ -2576,25 +2582,25 @@ hs_desc_decode_descriptor(const char *encoded,
 
   /* Subcredentials are not optional. */
   if (BUG(!subcredential ||
-          tor_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
+          fast_mem_is_zero((char*)subcredential, DIGEST256_LEN))) {
     log_warn(LD_GENERAL, "Tried to decrypt without subcred. Impossible!");
     goto err;
   }
 
-  memcpy(desc->subcredential, subcredential, sizeof(desc->subcredential));
+  memcpy(&desc->subcredential, subcredential, sizeof(desc->subcredential));
 
   ret = hs_desc_decode_plaintext(encoded, &desc->plaintext_data);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
   ret = hs_desc_decode_superencrypted(desc, &desc->superencrypted_data);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
   ret = hs_desc_decode_encrypted(desc, client_auth_sk, &desc->encrypted_data);
-  if (ret < 0) {
+  if (ret != HS_DESC_DECODE_OK) {
     goto err;
   }
 
@@ -2615,7 +2621,7 @@ hs_desc_decode_descriptor(const char *encoded,
   return ret;
 }
 
-/* Table of encode function version specific. The functions are indexed by the
+/** Table of encode function version specific. The functions are indexed by the
  * version number so v3 callback is at index 3 in the array. */
 static int
   (*encode_handlers[])(
@@ -2628,7 +2634,7 @@ static int
   desc_encode_v3,
 };
 
-/* Encode the given descriptor desc including signing with the given key pair
+/** Encode the given descriptor desc including signing with the given key pair
  * signing_kp and encrypting with the given descriptor cookie.
  *
  * If the client authorization is enabled, descriptor_cookie must be the same
@@ -2671,9 +2677,10 @@ hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
    * symmetric only if the client auth is disabled. That is, the descriptor
    * cookie will be NULL. */
   if (!descriptor_cookie) {
-    ret = hs_desc_decode_descriptor(*encoded_out, desc->subcredential,
+    ret = hs_desc_decode_descriptor(*encoded_out, &desc->subcredential,
                                     NULL, NULL);
-    if (BUG(ret < 0)) {
+    if (BUG(ret != HS_DESC_DECODE_OK)) {
+      ret = -1;
       goto err;
     }
   }
@@ -2685,7 +2692,7 @@ hs_desc_encode_descriptor,(const hs_descriptor_t *desc,
   return ret;
 }
 
-/* Free the content of the plaintext section of a descriptor. */
+/** Free the content of the plaintext section of a descriptor. */
 void
 hs_desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
 {
@@ -2701,7 +2708,7 @@ hs_desc_plaintext_data_free_contents(hs_desc_plaintext_data_t *desc)
   memwipe(desc, 0, sizeof(*desc));
 }
 
-/* Free the content of the superencrypted section of a descriptor. */
+/** Free the content of the superencrypted section of a descriptor. */
 void
 hs_desc_superencrypted_data_free_contents(hs_desc_superencrypted_data_t *desc)
 {
@@ -2721,7 +2728,7 @@ hs_desc_superencrypted_data_free_contents(hs_desc_superencrypted_data_t *desc)
   memwipe(desc, 0, sizeof(*desc));
 }
 
-/* Free the content of the encrypted section of a descriptor. */
+/** Free the content of the encrypted section of a descriptor. */
 void
 hs_desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
 {
@@ -2741,7 +2748,7 @@ hs_desc_encrypted_data_free_contents(hs_desc_encrypted_data_t *desc)
   memwipe(desc, 0, sizeof(*desc));
 }
 
-/* Free the descriptor plaintext data object. */
+/** Free the descriptor plaintext data object. */
 void
 hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
 {
@@ -2749,7 +2756,7 @@ hs_desc_plaintext_data_free_(hs_desc_plaintext_data_t *desc)
   tor_free(desc);
 }
 
-/* Free the descriptor plaintext data object. */
+/** Free the descriptor plaintext data object. */
 void
 hs_desc_superencrypted_data_free_(hs_desc_superencrypted_data_t *desc)
 {
@@ -2757,7 +2764,7 @@ hs_desc_superencrypted_data_free_(hs_desc_superencrypted_data_t *desc)
   tor_free(desc);
 }
 
-/* Free the descriptor encrypted data object. */
+/** Free the descriptor encrypted data object. */
 void
 hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
 {
@@ -2765,7 +2772,7 @@ hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc)
   tor_free(desc);
 }
 
-/* Free the given descriptor object. */
+/** Free the given descriptor object. */
 void
 hs_descriptor_free_(hs_descriptor_t *desc)
 {
@@ -2779,7 +2786,7 @@ hs_descriptor_free_(hs_descriptor_t *desc)
   tor_free(desc);
 }
 
-/* Return the size in bytes of the given plaintext data object. A sizeof() is
+/** Return the size in bytes of the given plaintext data object. A sizeof() is
  * not enough because the object contains pointers and the encrypted blob.
  * This is particularly useful for our OOM subsystem that tracks the HSDir
  * cache size for instance. */
@@ -2791,7 +2798,7 @@ hs_desc_plaintext_obj_size(const hs_desc_plaintext_data_t *data)
           data->superencrypted_blob_size);
 }
 
-/* Return the size in bytes of the given encrypted data object. Used by OOM
+/** Return the size in bytes of the given encrypted data object. Used by OOM
  * subsystem. */
 static size_t
 hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
@@ -2811,18 +2818,20 @@ hs_desc_encrypted_obj_size(const hs_desc_encrypted_data_t *data)
   return sizeof(*data) + intro_size;
 }
 
-/* Return the size in bytes of the given descriptor object. Used by OOM
+/** Return the size in bytes of the given descriptor object. Used by OOM
  * subsystem. */
   size_t
 hs_desc_obj_size(const hs_descriptor_t *data)
 {
-  tor_assert(data);
+  if (data == NULL) {
+    return 0;
+  }
   return (hs_desc_plaintext_obj_size(&data->plaintext_data) +
           hs_desc_encrypted_obj_size(&data->encrypted_data) +
           sizeof(data->subcredential));
 }
 
-/* Return a newly allocated descriptor intro point. */
+/** Return a newly allocated descriptor intro point. */
 hs_desc_intro_point_t *
 hs_desc_intro_point_new(void)
 {
@@ -2831,7 +2840,7 @@ hs_desc_intro_point_new(void)
   return ip;
 }
 
-/* Free a descriptor intro point object. */
+/** Free a descriptor intro point object. */
 void
 hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
 {
@@ -2839,8 +2848,8 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
     return;
   }
   if (ip->link_specifiers) {
-    SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *,
-                      ls, hs_desc_link_specifier_free(ls));
+    SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *,
+                      ls, link_specifier_free(ls));
     smartlist_free(ip->link_specifiers);
   }
   tor_cert_free(ip->auth_key_cert);
@@ -2850,7 +2859,7 @@ hs_desc_intro_point_free_(hs_desc_intro_point_t *ip)
   tor_free(ip);
 }
 
-/* Allocate and build a new fake client info for the descriptor. Return a
+/** Allocate and build a new fake client info for the descriptor. Return a
  * newly allocated object. This can't fail. */
 hs_desc_authorized_client_t *
 hs_desc_build_fake_authorized_client(void)
@@ -2868,49 +2877,44 @@ hs_desc_build_fake_authorized_client(void)
   return client_auth;
 }
 
-/* Using the service's subcredential, client public key, auth ephemeral secret
+/** Using the service's subcredential, client public key, auth ephemeral secret
  * key, and descriptor cookie, build the auth client so we can then encode the
  * descriptor for publication. client_out must be already allocated. */
 void
-hs_desc_build_authorized_client(const uint8_t *subcredential,
+hs_desc_build_authorized_client(const hs_subcredential_t *subcredential,
                                 const curve25519_public_key_t *client_auth_pk,
                                 const curve25519_secret_key_t *
                                 auth_ephemeral_sk,
                                 const uint8_t *descriptor_cookie,
                                 hs_desc_authorized_client_t *client_out)
 {
-  uint8_t secret_seed[CURVE25519_OUTPUT_LEN];
-  uint8_t keystream[HS_DESC_CLIENT_ID_LEN + HS_DESC_COOKIE_KEY_LEN];
-  uint8_t *cookie_key;
+  uint8_t *keystream = NULL;
+  size_t keystream_length = 0;
+  const uint8_t *cookie_key;
   crypto_cipher_t *cipher;
-  crypto_xof_t *xof;
 
   tor_assert(client_auth_pk);
   tor_assert(auth_ephemeral_sk);
   tor_assert(descriptor_cookie);
   tor_assert(client_out);
   tor_assert(subcredential);
-  tor_assert(!tor_mem_is_zero((char *) auth_ephemeral_sk,
+  tor_assert(!fast_mem_is_zero((char *) auth_ephemeral_sk,
                               sizeof(*auth_ephemeral_sk)));
-  tor_assert(!tor_mem_is_zero((char *) client_auth_pk,
+  tor_assert(!fast_mem_is_zero((char *) client_auth_pk,
                               sizeof(*client_auth_pk)));
-  tor_assert(!tor_mem_is_zero((char *) descriptor_cookie,
+  tor_assert(!fast_mem_is_zero((char *) descriptor_cookie,
                               HS_DESC_DESCRIPTOR_COOKIE_LEN));
-  tor_assert(!tor_mem_is_zero((char *) subcredential,
+  tor_assert(!fast_mem_is_zero((char *) subcredential,
                               DIGEST256_LEN));
 
-  /* Calculate x25519(hs_y, client_X) */
-  curve25519_handshake(secret_seed,
-                       auth_ephemeral_sk,
-                       client_auth_pk);
-
-  /* Calculate KEYS = KDF(subcredential | SECRET_SEED, 40) */
-  xof = crypto_xof_new();
-  crypto_xof_add_bytes(xof, subcredential, DIGEST256_LEN);
-  crypto_xof_add_bytes(xof, secret_seed, sizeof(secret_seed));
-  crypto_xof_squeeze_bytes(xof, keystream, sizeof(keystream));
-  crypto_xof_free(xof);
+  /* Get the KEYS part so we can derive the CLIENT-ID and COOKIE-KEY. */
+  keystream_length =
+    build_descriptor_cookie_keys(subcredential,
+                                 auth_ephemeral_sk, client_auth_pk,
+                                 &keystream);
+  tor_assert(keystream_length > 0);
 
+  /* Extract the CLIENT-ID and COOKIE-KEY from the KEYS. */
   memcpy(client_out->client_id, keystream, HS_DESC_CLIENT_ID_LEN);
   cookie_key = keystream + HS_DESC_CLIENT_ID_LEN;
 
@@ -2925,83 +2929,20 @@ hs_desc_build_authorized_client(const uint8_t *subcredential,
                         (const char *) descriptor_cookie,
                         HS_DESC_DESCRIPTOR_COOKIE_LEN);
 
-  memwipe(secret_seed, 0, sizeof(secret_seed));
-  memwipe(keystream, 0, sizeof(keystream));
+  memwipe(keystream, 0, keystream_length);
+  tor_free(keystream);
 
   crypto_cipher_free(cipher);
 }
 
-/* Free an authoriezd client object. */
+/** Free an authoriezd client object. */
 void
 hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client)
 {
   tor_free(client);
 }
 
-/* Free the given descriptor link specifier. */
-void
-hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls)
-{
-  if (ls == NULL) {
-    return;
-  }
-  tor_free(ls);
-}
-
-/* Return a newly allocated descriptor link specifier using the given extend
- * info and requested type. Return NULL on error. */
-hs_desc_link_specifier_t *
-hs_desc_link_specifier_new(const extend_info_t *info, uint8_t type)
-{
-  hs_desc_link_specifier_t *ls = NULL;
-
-  tor_assert(info);
-
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = type;
-  switch (ls->type) {
-  case LS_IPV4:
-    if (info->addr.family != AF_INET) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_IPV6:
-    if (info->addr.family != AF_INET6) {
-      goto err;
-    }
-    tor_addr_copy(&ls->u.ap.addr, &info->addr);
-    ls->u.ap.port = info->port;
-    break;
-  case LS_LEGACY_ID:
-    /* Bug out if the identity digest is not set */
-    if (BUG(tor_mem_is_zero(info->identity_digest,
-                            sizeof(info->identity_digest)))) {
-      goto err;
-    }
-    memcpy(ls->u.legacy_id, info->identity_digest, sizeof(ls->u.legacy_id));
-    break;
-  case LS_ED25519_ID:
-    /* ed25519 keys are optional for intro points */
-    if (ed25519_public_key_is_zero(&info->ed_identity)) {
-      goto err;
-    }
-    memcpy(ls->u.ed25519_id, info->ed_identity.pubkey,
-           sizeof(ls->u.ed25519_id));
-    break;
-  default:
-    /* Unknown type is code flow error. */
-    tor_assert(0);
-  }
-
-  return ls;
- err:
-  tor_free(ls);
-  return NULL;
-}
-
-/* From the given descriptor, remove and free every introduction point. */
+/** From the given descriptor, remove and free every introduction point. */
 void
 hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
 {
@@ -3016,59 +2957,3 @@ hs_descriptor_clear_intro_points(hs_descriptor_t *desc)
     smartlist_clear(ips);
   }
 }
-
-/* From a descriptor link specifier object spec, returned a newly allocated
- * link specifier object that is the encoded representation of spec. Return
- * NULL on error. */
-link_specifier_t *
-hs_desc_lspec_to_trunnel(const hs_desc_link_specifier_t *spec)
-{
-  tor_assert(spec);
-
-  link_specifier_t *ls = link_specifier_new();
-  link_specifier_set_ls_type(ls, spec->type);
-
-  switch (spec->type) {
-  case LS_IPV4:
-    link_specifier_set_un_ipv4_addr(ls,
-                                    tor_addr_to_ipv4h(&spec->u.ap.addr));
-    link_specifier_set_un_ipv4_port(ls, spec->u.ap.port);
-    /* Four bytes IPv4 and two bytes port. */
-    link_specifier_set_ls_len(ls, sizeof(spec->u.ap.addr.addr.in_addr) +
-                                  sizeof(spec->u.ap.port));
-    break;
-  case LS_IPV6:
-  {
-    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
-    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&spec->u.ap.addr);
-    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
-    memcpy(ipv6_array, in6_addr, addr_len);
-    link_specifier_set_un_ipv6_port(ls, spec->u.ap.port);
-    /* Sixteen bytes IPv6 and two bytes port. */
-    link_specifier_set_ls_len(ls, addr_len + sizeof(spec->u.ap.port));
-    break;
-  }
-  case LS_LEGACY_ID:
-  {
-    size_t legacy_id_len = link_specifier_getlen_un_legacy_id(ls);
-    uint8_t *legacy_id_array = link_specifier_getarray_un_legacy_id(ls);
-    memcpy(legacy_id_array, spec->u.legacy_id, legacy_id_len);
-    link_specifier_set_ls_len(ls, legacy_id_len);
-    break;
-  }
-  case LS_ED25519_ID:
-  {
-    size_t ed25519_id_len = link_specifier_getlen_un_ed25519_id(ls);
-    uint8_t *ed25519_id_array = link_specifier_getarray_un_ed25519_id(ls);
-    memcpy(ed25519_id_array, spec->u.ed25519_id, ed25519_id_len);
-    link_specifier_set_ls_len(ls, ed25519_id_len);
-    break;
-  }
-  default:
-    tor_assert_nonfatal_unreached();
-    link_specifier_free(ls);
-    ls = NULL;
-  }
-
-  return ls;
-}
diff --git a/src/feature/hs/hs_descriptor.h b/src/feature/hs/hs_descriptor.h
index 04a8e16d63..08daa904b6 100644
--- a/src/feature/hs/hs_descriptor.h
+++ b/src/feature/hs/hs_descriptor.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,110 +14,118 @@
 #include "core/or/or.h"
 #include "trunnel/ed25519_cert.h" /* needed for trunnel */
 #include "feature/nodelist/torcert.h"
+#include "core/crypto/hs_ntor.h" /* for hs_subcredential_t */
 
 /* Trunnel */
 struct link_specifier_t;
 
-/* The earliest descriptor format version we support. */
+/** The earliest descriptor format version we support. */
 #define HS_DESC_SUPPORTED_FORMAT_VERSION_MIN 3
-/* The latest descriptor format version we support. */
+/** The latest descriptor format version we support. */
 #define HS_DESC_SUPPORTED_FORMAT_VERSION_MAX 3
 
-/* Default lifetime of a descriptor in seconds. The valus is set at 3 hours
+/** Default lifetime of a descriptor in seconds. The valus is set at 3 hours
  * which is 180 minutes or 10800 seconds. */
 #define HS_DESC_DEFAULT_LIFETIME (3 * 60 * 60)
-/* Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
+/** Maximum lifetime of a descriptor in seconds. The value is set at 12 hours
  * which is 720 minutes or 43200 seconds. */
 #define HS_DESC_MAX_LIFETIME (12 * 60 * 60)
-/* Lifetime of certificate in the descriptor. This defines the lifetime of the
+/** Lifetime of certificate in the descriptor. This defines the lifetime of the
  * descriptor signing key and the cross certification cert of that key. It is
  * set to 54 hours because a descriptor can be around for 48 hours and because
  * consensuses are used after the hour, add an extra 6 hours to give some time
  * for the service to stop using it. */
 #define HS_DESC_CERT_LIFETIME (54 * 60 * 60)
-/* Length of the salt needed for the encrypted section of a descriptor. */
+/** Length of the salt needed for the encrypted section of a descriptor. */
 #define HS_DESC_ENCRYPTED_SALT_LEN 16
-/* Length of the KDF output value which is the length of the secret key,
+/** Length of the KDF output value which is the length of the secret key,
  * the secret IV and MAC key length which is the length of H() output. */
 #define HS_DESC_ENCRYPTED_KDF_OUTPUT_LEN \
   CIPHER256_KEY_LEN + CIPHER_IV_LEN + DIGEST256_LEN
-/* Pad plaintext of superencrypted data section before encryption so that its
+/** Pad plaintext of superencrypted data section before encryption so that its
  * length is a multiple of this value. */
 #define HS_DESC_SUPERENC_PLAINTEXT_PAD_MULTIPLE 10000
-/* Maximum length in bytes of a full hidden service descriptor. */
+/** Maximum length in bytes of a full hidden service descriptor. */
 #define HS_DESC_MAX_LEN 50000 /* 50kb max size */
 
-/* Key length for the descriptor symmetric encryption. As specified in the
+/** Key length for the descriptor symmetric encryption. As specified in the
  * protocol, we use AES-256 for the encrypted section of the descriptor. The
  * following is the length in bytes and the bit size. */
 #define HS_DESC_ENCRYPTED_KEY_LEN CIPHER256_KEY_LEN
 #define HS_DESC_ENCRYPTED_BIT_SIZE (HS_DESC_ENCRYPTED_KEY_LEN * 8)
 
-/* Length of each components in the auth client section in the descriptor. */
+/** Length of each components in the auth client section in the descriptor. */
 #define HS_DESC_CLIENT_ID_LEN 8
 #define HS_DESC_DESCRIPTOR_COOKIE_LEN 16
 #define HS_DESC_COOKIE_KEY_LEN 32
 #define HS_DESC_COOKIE_KEY_BIT_SIZE (HS_DESC_COOKIE_KEY_LEN * 8)
 #define HS_DESC_ENCRYPED_COOKIE_LEN HS_DESC_DESCRIPTOR_COOKIE_LEN
 
-/* The number of auth client entries in the descriptor must be the multiple
+/** The number of auth client entries in the descriptor must be the multiple
  * of this constant. */
 #define HS_DESC_AUTH_CLIENT_MULTIPLE 16
 
-/* Type of authentication in the descriptor. */
+/** Type of authentication in the descriptor. */
 typedef enum {
   HS_DESC_AUTH_ED25519 = 1
 } hs_desc_auth_type_t;
 
-/* Link specifier object that contains information on how to extend to the
- * relay that is the address, port and handshake type. */
-typedef struct hs_desc_link_specifier_t {
-  /* Indicate the type of link specifier. See trunnel ed25519_cert
-   * specification. */
-  uint8_t type;
-
-  /* It must be one of these types, can't be more than one. */
-  union {
-    /* IP address and port of the relay use to extend. */
-    tor_addr_port_t ap;
-    /* Legacy identity. A 20-byte SHA1 identity fingerprint. */
-    uint8_t legacy_id[DIGEST_LEN];
-    /* ed25519 identity. A 32-byte key. */
-    uint8_t ed25519_id[ED25519_PUBKEY_LEN];
-  } u;
-} hs_desc_link_specifier_t;
-
-/* Introduction point information located in a descriptor. */
+/** Error code when decoding a descriptor. */
+typedef enum {
+  /* The configured client authorization for the requested .onion address
+   * failed to decode the descriptor. */
+  HS_DESC_DECODE_BAD_CLIENT_AUTH  = -6,
+
+  /* The requested .onion address requires a client authorization. */
+  HS_DESC_DECODE_NEED_CLIENT_AUTH = -5,
+
+  /* Error during decryption of the encrypted layer. */
+  HS_DESC_DECODE_ENCRYPTED_ERROR  = -4,
+
+  /* Error during decryption of the super encrypted layer. */
+  HS_DESC_DECODE_SUPERENC_ERROR   = -3,
+
+  /* Error while decoding the plaintext section. */
+  HS_DESC_DECODE_PLAINTEXT_ERROR  = -2,
+
+  /* Generic error. */
+  HS_DESC_DECODE_GENERIC_ERROR    = -1,
+
+  /* Decoding a descriptor was successful. */
+  HS_DESC_DECODE_OK               =  0,
+} hs_desc_decode_status_t;
+
+/** Introduction point information located in a descriptor. */
 typedef struct hs_desc_intro_point_t {
-  /* Link specifier(s) which details how to extend to the relay. This list
-   * contains hs_desc_link_specifier_t object. It MUST have at least one. */
+  /** Link specifier(s) which details how to extend to the relay. This list
+   * contains link_specifier_t objects. It MUST have at least one. */
   smartlist_t *link_specifiers;
 
-  /* Onion key of the introduction point used to extend to it for the ntor
+  /** Onion key of the introduction point used to extend to it for the ntor
    * handshake. */
   curve25519_public_key_t onion_key;
 
-  /* Authentication key used to establish the introduction point circuit and
+  /** Authentication key used to establish the introduction point circuit and
    * cross-certifies the blinded public key for the replica thus signed by
    * the blinded key and in turn signs it. */
   tor_cert_t *auth_key_cert;
 
-  /* Encryption key for the "ntor" type. */
+  /** Encryption key for the "ntor" type. */
   curve25519_public_key_t enc_key;
 
-  /* Certificate cross certifying the descriptor signing key by the encryption
+  /** Certificate cross certifying the descriptor signing key by the encryption
    * curve25519 key. This certificate contains the signing key and is of type
    * CERT_TYPE_CROSS_HS_IP_KEYS [0B]. */
   tor_cert_t *enc_key_cert;
 
-  /* (Optional): If this introduction point is a legacy one that is version <=
+  /** (Optional): If this introduction point is a legacy one that is version <=
    * 0.2.9.x (HSIntro=3), we use this extra key for the intro point to be able
    * to relay the cells to the service correctly. */
   struct {
-    /* RSA public key. */
+    /** RSA public key. */
     crypto_pk_t *key;
 
-    /* Cross certified cert with the descriptor signing key (RSA->Ed). Because
+    /** Cross certified cert with the descriptor signing key (RSA->Ed). Because
      * of the cross certification API, we need to keep the certificate binary
      * blob and its length in order to properly encode it after. */
     struct {
@@ -126,115 +134,115 @@ typedef struct hs_desc_intro_point_t {
     } cert;
   } legacy;
 
-  /* True iff the introduction point has passed the cross certification. Upon
+  /** True iff the introduction point has passed the cross certification. Upon
    * decoding an intro point, this must be true. */
   unsigned int cross_certified : 1;
 } hs_desc_intro_point_t;
 
-/* Authorized client information located in a descriptor. */
+/** Authorized client information located in a descriptor. */
 typedef struct hs_desc_authorized_client_t {
-  /* An identifier that the client will use to identify which auth client
+  /** An identifier that the client will use to identify which auth client
    * entry it needs to use. */
   uint8_t client_id[HS_DESC_CLIENT_ID_LEN];
 
-  /* An IV that is used to decrypt the encrypted descriptor cookie. */
+  /** An IV that is used to decrypt the encrypted descriptor cookie. */
   uint8_t iv[CIPHER_IV_LEN];
 
-  /* An encrypted descriptor cookie that the client needs to decrypt to use
+  /** An encrypted descriptor cookie that the client needs to decrypt to use
    * it to decrypt the descriptor. */
   uint8_t encrypted_cookie[HS_DESC_ENCRYPED_COOKIE_LEN];
 } hs_desc_authorized_client_t;
 
-/* The encrypted data section of a descriptor. Obviously the data in this is
+/** The encrypted data section of a descriptor. Obviously the data in this is
  * in plaintext but encrypted once encoded. */
 typedef struct hs_desc_encrypted_data_t {
-  /* Bitfield of CREATE2 cell supported formats. The only currently supported
+  /** Bitfield of CREATE2 cell supported formats. The only currently supported
    * format is ntor. */
   unsigned int create2_ntor : 1;
 
-  /* A list of authentication types that a client must at least support one
+  /** A list of authentication types that a client must at least support one
    * in order to contact the service. Contains NULL terminated strings. */
   smartlist_t *intro_auth_types;
 
-  /* Is this descriptor a single onion service? */
+  /** Is this descriptor a single onion service? */
   unsigned int single_onion_service : 1;
 
-  /* A list of intro points. Contains hs_desc_intro_point_t objects. */
+  /** A list of intro points. Contains hs_desc_intro_point_t objects. */
   smartlist_t *intro_points;
 } hs_desc_encrypted_data_t;
 
-/* The superencrypted data section of a descriptor. Obviously the data in
+/** The superencrypted data section of a descriptor. Obviously the data in
  * this is in plaintext but encrypted once encoded. */
 typedef struct hs_desc_superencrypted_data_t {
-  /* This field contains ephemeral x25519 public key which is used by
+  /** This field contains ephemeral x25519 public key which is used by
    * the encryption scheme in the client authorization. */
   curve25519_public_key_t auth_ephemeral_pubkey;
 
-  /* A list of authorized clients. Contains hs_desc_authorized_client_t
+  /** A list of authorized clients. Contains hs_desc_authorized_client_t
    * objects. */
   smartlist_t *clients;
 
-  /* Decoding only: The b64-decoded encrypted blob from the descriptor */
+  /** Decoding only: The b64-decoded encrypted blob from the descriptor */
   uint8_t *encrypted_blob;
 
-  /* Decoding only: Size of the encrypted_blob */
+  /** Decoding only: Size of the encrypted_blob */
   size_t encrypted_blob_size;
 } hs_desc_superencrypted_data_t;
 
-/* Plaintext data that is unencrypted information of the descriptor. */
+/** Plaintext data that is unencrypted information of the descriptor. */
 typedef struct hs_desc_plaintext_data_t {
-  /* Version of the descriptor format. Spec specifies this field as a
+  /** Version of the descriptor format. Spec specifies this field as a
    * positive integer. */
   uint32_t version;
 
-  /* The lifetime of the descriptor in seconds. */
+  /** The lifetime of the descriptor in seconds. */
   uint32_t lifetime_sec;
 
-  /* Certificate with the short-term ed22519 descriptor signing key for the
+  /** Certificate with the short-term ed22519 descriptor signing key for the
    * replica which is signed by the blinded public key for that replica. */
   tor_cert_t *signing_key_cert;
 
-  /* Signing public key which is used to sign the descriptor. Same public key
+  /** Signing public key which is used to sign the descriptor. Same public key
    * as in the signing key certificate. */
   ed25519_public_key_t signing_pubkey;
 
-  /* Blinded public key used for this descriptor derived from the master
+  /** Blinded public key used for this descriptor derived from the master
    * identity key and generated for a specific replica number. */
   ed25519_public_key_t blinded_pubkey;
 
-  /* Revision counter is incremented at each upload, regardless of whether
+  /** Revision counter is incremented at each upload, regardless of whether
    * the descriptor has changed. This avoids leaking whether the descriptor
    * has changed. Spec specifies this as a 8 bytes positive integer. */
   uint64_t revision_counter;
 
-  /* Decoding only: The b64-decoded superencrypted blob from the descriptor */
+  /** Decoding only: The b64-decoded superencrypted blob from the descriptor */
   uint8_t *superencrypted_blob;
 
-  /* Decoding only: Size of the superencrypted_blob */
+  /** Decoding only: Size of the superencrypted_blob */
   size_t superencrypted_blob_size;
 } hs_desc_plaintext_data_t;
 
-/* Service descriptor in its decoded form. */
+/** Service descriptor in its decoded form. */
 typedef struct hs_descriptor_t {
-  /* Contains the plaintext part of the descriptor. */
+  /** Contains the plaintext part of the descriptor. */
   hs_desc_plaintext_data_t plaintext_data;
 
-  /* The following contains what's in the superencrypted part of the
+  /** The following contains what's in the superencrypted part of the
    * descriptor. It's only encrypted in the encoded version of the descriptor
    * thus the data contained in that object is in plaintext. */
   hs_desc_superencrypted_data_t superencrypted_data;
 
-  /* The following contains what's in the encrypted part of the descriptor.
+  /** The following contains what's in the encrypted part of the descriptor.
    * It's only encrypted in the encoded version of the descriptor thus the
    * data contained in that object is in plaintext. */
   hs_desc_encrypted_data_t encrypted_data;
 
-  /* Subcredentials of a service, used by the client and service to decrypt
+  /** Subcredentials of a service, used by the client and service to decrypt
    * the encrypted data. */
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 } hs_descriptor_t;
 
-/* Return true iff the given descriptor format version is supported. */
+/** Return true iff the given descriptor format version is supported. */
 static inline int
 hs_desc_is_supported_version(uint32_t version)
 {
@@ -261,12 +269,6 @@ void hs_desc_encrypted_data_free_(hs_desc_encrypted_data_t *desc);
 #define hs_desc_encrypted_data_free(desc) \
   FREE_AND_NULL(hs_desc_encrypted_data_t, hs_desc_encrypted_data_free_, (desc))
 
-void hs_desc_link_specifier_free_(hs_desc_link_specifier_t *ls);
-#define hs_desc_link_specifier_free(ls) \
-  FREE_AND_NULL(hs_desc_link_specifier_t, hs_desc_link_specifier_free_, (ls))
-
-hs_desc_link_specifier_t *hs_desc_link_specifier_new(
-                                  const extend_info_t *info, uint8_t type);
 void hs_descriptor_clear_intro_points(hs_descriptor_t *desc);
 
 MOCK_DECL(int,
@@ -276,7 +278,7 @@ MOCK_DECL(int,
                                      char **encoded_out));
 
 int hs_desc_decode_descriptor(const char *encoded,
-                              const uint8_t *subcredential,
+                              const hs_subcredential_t *subcredential,
                               const curve25519_secret_key_t *client_auth_sk,
                               hs_descriptor_t **desc_out);
 int hs_desc_decode_plaintext(const char *encoded,
@@ -299,11 +301,9 @@ void hs_desc_authorized_client_free_(hs_desc_authorized_client_t *client);
   FREE_AND_NULL(hs_desc_authorized_client_t, \
                 hs_desc_authorized_client_free_, (client))
 
-link_specifier_t *hs_desc_lspec_to_trunnel(
-                                   const hs_desc_link_specifier_t *spec);
-
 hs_desc_authorized_client_t *hs_desc_build_fake_authorized_client(void);
-void hs_desc_build_authorized_client(const uint8_t *subcredential,
+
+void hs_desc_build_authorized_client(const hs_subcredential_t *subcredential,
                                      const curve25519_public_key_t *
                                      client_auth_pk,
                                      const curve25519_secret_key_t *
@@ -335,10 +335,8 @@ STATIC int desc_sig_is_valid(const char *b64_sig,
                              const char *encoded_desc, size_t encoded_len);
 
 MOCK_DECL(STATIC size_t, decrypt_desc_layer,(const hs_descriptor_t *desc,
-                                             const uint8_t *encrypted_blob,
-                                             size_t encrypted_blob_size,
                                              const uint8_t *descriptor_cookie,
-                                             int is_superencrypted_layer,
+                                             bool is_superencrypted_layer,
                                              char **decrypted_out));
 
 #endif /* defined(HS_DESCRIPTOR_PRIVATE) */
diff --git a/src/feature/hs/hs_dos.c b/src/feature/hs/hs_dos.c
new file mode 100644
index 0000000000..04c2bfbb89
--- /dev/null
+++ b/src/feature/hs/hs_dos.c
@@ -0,0 +1,228 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.c
+ * \brief Implement denial of service mitigation for the onion service
+ *        subsystem.
+ *
+ * This module defenses:
+ *
+ * - Introduction Rate Limiting: If enabled by the consensus, an introduction
+ *   point will rate limit client introduction towards the service (INTRODUCE2
+ *   cells). It uses a token bucket model with a rate and burst per second.
+ *
+ *   Proposal 305 will expand this module by allowing an operator to define
+ *   these values into the ESTABLISH_INTRO cell. Not yet implemented.
+ **/
+
+#define HS_DOS_PRIVATE
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+
+#include "feature/hs/hs_circuitmap.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/relay/routermode.h"
+
+#include "lib/evloop/token_bucket.h"
+
+#include "feature/hs/hs_dos.h"
+
+/** Default value of the allowed INTRODUCE2 cell rate per second. Above that
+ * value per second, the introduction is denied. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC 25
+
+/** Default value of the allowed INTRODUCE2 cell burst per second. This is the
+ * maximum value a token bucket has per second. We thus allow up to this value
+ * of INTRODUCE2 cell per second but the bucket is refilled by the rate value
+ * but never goes above that burst value. */
+#define HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC 200
+
+/** Default value of the consensus parameter enabling or disabling the
+ * introduction DoS defense. Disabled by default. */
+#define HS_DOS_INTRODUCE_ENABLED_DEFAULT 0
+
+/** INTRODUCE2 rejected request counter. */
+static uint64_t intro2_rejected_count = 0;
+
+/* Consensus parameters. The ESTABLISH_INTRO DoS cell extension have higher
+ * priority than these values. If no extension is sent, these are used only by
+ * the introduction point. */
+static uint32_t consensus_param_introduce_rate_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC;
+static uint32_t consensus_param_introduce_burst_per_sec =
+  HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC;
+static uint32_t consensus_param_introduce_defense_enabled =
+  HS_DOS_INTRODUCE_ENABLED_DEFAULT;
+
+STATIC uint32_t
+get_intro2_enable_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSDefense",
+                                 HS_DOS_INTRODUCE_ENABLED_DEFAULT, 0, 1);
+}
+
+/** Return the parameter for the introduction rate per sec. */
+STATIC uint32_t
+get_intro2_rate_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSRatePerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_RATE_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/** Return the parameter for the introduction burst per sec. */
+STATIC uint32_t
+get_intro2_burst_consensus_param(const networkstatus_t *ns)
+{
+  return networkstatus_get_param(ns, "HiddenServiceEnableIntroDoSBurstPerSec",
+                                 HS_DOS_INTRODUCE_DEFAULT_CELL_BURST_PER_SEC,
+                                 0, INT32_MAX);
+}
+
+/** Go over all introduction circuit relay side and adjust their rate/burst
+ * values using the global parameters. This is called right after the
+ * consensus parameters might have changed. */
+static void
+update_intro_circuits(void)
+{
+  /* Returns all HS version intro circuits. */
+  smartlist_t *intro_circs = hs_circuitmap_get_all_intro_circ_relay_side();
+
+  SMARTLIST_FOREACH_BEGIN(intro_circs, circuit_t *, circ) {
+    /* Ignore circuit if the defenses were set explicitly through the
+     * ESTABLISH_INTRO cell DoS extension. */
+    if (TO_OR_CIRCUIT(circ)->introduce2_dos_defense_explicit) {
+      continue;
+    }
+    /* Defenses might have been enabled or disabled. */
+    TO_OR_CIRCUIT(circ)->introduce2_dos_defense_enabled =
+      consensus_param_introduce_defense_enabled;
+    /* Adjust the rate/burst value that might have changed. */
+    token_bucket_ctr_adjust(&TO_OR_CIRCUIT(circ)->introduce2_bucket,
+                            consensus_param_introduce_rate_per_sec,
+                            consensus_param_introduce_burst_per_sec);
+  } SMARTLIST_FOREACH_END(circ);
+
+  smartlist_free(intro_circs);
+}
+
+/** Set consensus parameters. */
+static void
+set_consensus_parameters(const networkstatus_t *ns)
+{
+  consensus_param_introduce_rate_per_sec =
+    get_intro2_rate_consensus_param(ns);
+  consensus_param_introduce_burst_per_sec =
+    get_intro2_burst_consensus_param(ns);
+  consensus_param_introduce_defense_enabled =
+    get_intro2_enable_consensus_param(ns);
+
+  /* The above might have changed which means we need to go through all
+   * introduction circuits (relay side) and update the token buckets. */
+  update_intro_circuits();
+}
+
+/*
+ * Public API.
+ */
+
+/** Initialize the INTRODUCE2 token bucket for the DoS defenses using the
+ * consensus/default values. We might get a cell extension that changes those
+ * later but if we don't, the default or consensus parameters are used. */
+void
+hs_dos_setup_default_intro2_defenses(or_circuit_t *circ)
+{
+  tor_assert(circ);
+
+  circ->introduce2_dos_defense_enabled =
+    consensus_param_introduce_defense_enabled;
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        consensus_param_introduce_rate_per_sec,
+                        consensus_param_introduce_burst_per_sec,
+                        (uint32_t) approx_time());
+}
+
+/** Called when the consensus has changed. We might have new consensus
+ * parameters to look at. */
+void
+hs_dos_consensus_has_changed(const networkstatus_t *ns)
+{
+  /* No point on updating these values if we are not a public relay that can
+   * be picked to be an introduction point. */
+  if (!public_server_mode(get_options())) {
+    return;
+  }
+
+  set_consensus_parameters(ns);
+}
+
+/** Return true iff an INTRODUCE2 cell can be sent on the given service
+ * introduction circuit. */
+bool
+hs_dos_can_send_intro2(or_circuit_t *s_intro_circ)
+{
+  tor_assert(s_intro_circ);
+
+  /* Allow to send the cell if the DoS defenses are disabled on the circuit.
+   * This can be set by the consensus, the ESTABLISH_INTRO cell extension or
+   * the hardcoded values in tor code. */
+  if (!s_intro_circ->introduce2_dos_defense_enabled) {
+    goto allow;
+  }
+
+  /* Should not happen but if so, scream loudly. */
+  if (BUG(TO_CIRCUIT(s_intro_circ)->purpose != CIRCUIT_PURPOSE_INTRO_POINT)) {
+    goto disallow;
+  }
+
+  /* This is called just after we got a valid and parsed INTRODUCE1 cell. The
+   * service has been found and we have its introduction circuit.
+   *
+   * First, the INTRODUCE2 bucket will be refilled (if any). Then, decremented
+   * because we are about to send or not the cell we just got. Finally,
+   * evaluate if we can send it based on our token bucket state. */
+
+  /* Refill INTRODUCE2 bucket. */
+  token_bucket_ctr_refill(&s_intro_circ->introduce2_bucket,
+                          (uint32_t) approx_time());
+
+  /* Decrement the bucket for this valid INTRODUCE1 cell we just got. Don't
+   * underflow else we end up with a too big of a bucket. */
+  if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) {
+    token_bucket_ctr_dec(&s_intro_circ->introduce2_bucket, 1);
+  }
+
+  /* Finally, we can send a new INTRODUCE2 if there are still tokens. */
+  if (token_bucket_ctr_get(&s_intro_circ->introduce2_bucket) > 0) {
+    goto allow;
+  }
+
+  /* If we reach this point, then it means the bucket has reached zero, and
+     we're going to disallow. */
+
+ disallow:
+  /* Increment stats counter, we are rejecting the INTRO2 cell. */
+  intro2_rejected_count++;
+  return false;
+
+ allow:
+  return true;
+}
+
+/** Return rolling count of rejected INTRO2. */
+uint64_t
+hs_dos_get_intro2_rejected_count(void)
+{
+  return intro2_rejected_count;
+}
+
+/** Initialize the onion service Denial of Service subsystem. */
+void
+hs_dos_init(void)
+{
+  set_consensus_parameters(NULL);
+}
diff --git a/src/feature/hs/hs_dos.h b/src/feature/hs/hs_dos.h
new file mode 100644
index 0000000000..8e36ece204
--- /dev/null
+++ b/src/feature/hs/hs_dos.h
@@ -0,0 +1,42 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_dos.h
+ * \brief Header file containing denial of service defenses for the HS
+ *        subsystem for all versions.
+ **/
+
+#ifndef TOR_HS_DOS_H
+#define TOR_HS_DOS_H
+
+#include "core/or/or_circuit_st.h"
+
+#include "feature/nodelist/networkstatus_st.h"
+
+/* Init */
+void hs_dos_init(void);
+
+/* Consensus. */
+void hs_dos_consensus_has_changed(const networkstatus_t *ns);
+
+/* Introduction Point. */
+bool hs_dos_can_send_intro2(or_circuit_t *s_intro_circ);
+void hs_dos_setup_default_intro2_defenses(or_circuit_t *circ);
+
+/* Statistics. */
+uint64_t hs_dos_get_intro2_rejected_count(void);
+
+#ifdef HS_DOS_PRIVATE
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC uint32_t get_intro2_enable_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_rate_consensus_param(const networkstatus_t *ns);
+STATIC uint32_t get_intro2_burst_consensus_param(const networkstatus_t *ns);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(HS_DOS_PRIVATE) */
+
+#endif /* !defined(TOR_HS_DOS_H) */
diff --git a/src/feature/hs/hs_ident.c b/src/feature/hs/hs_ident.c
index 8fd0013941..1d93ff9610 100644
--- a/src/feature/hs/hs_ident.c
+++ b/src/feature/hs/hs_ident.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,21 +10,17 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/hs/hs_ident.h"
 
-/* Return a newly allocated circuit identifier. The given public key is copied
+/** Return a newly allocated circuit identifier. The given public key is copied
  * identity_pk into the identifier. */
 hs_ident_circuit_t *
-hs_ident_circuit_new(const ed25519_public_key_t *identity_pk,
-                     hs_ident_circuit_type_t circuit_type)
+hs_ident_circuit_new(const ed25519_public_key_t *identity_pk)
 {
-  tor_assert(circuit_type == HS_IDENT_CIRCUIT_INTRO ||
-             circuit_type == HS_IDENT_CIRCUIT_RENDEZVOUS);
   hs_ident_circuit_t *ident = tor_malloc_zero(sizeof(*ident));
   ed25519_pubkey_copy(&ident->identity_pk, identity_pk);
-  ident->circuit_type = circuit_type;
   return ident;
 }
 
-/* Free the given circuit identifier. */
+/** Free the given circuit identifier. */
 void
 hs_ident_circuit_free_(hs_ident_circuit_t *ident)
 {
@@ -35,7 +31,7 @@ hs_ident_circuit_free_(hs_ident_circuit_t *ident)
   tor_free(ident);
 }
 
-/* For a given circuit identifier src, return a newly allocated copy of it.
+/** For a given circuit identifier src, return a newly allocated copy of it.
  * This can't fail. */
 hs_ident_circuit_t *
 hs_ident_circuit_dup(const hs_ident_circuit_t *src)
@@ -45,7 +41,7 @@ hs_ident_circuit_dup(const hs_ident_circuit_t *src)
   return ident;
 }
 
-/* For a given directory connection identifier src, return a newly allocated
+/** For a given directory connection identifier src, return a newly allocated
  * copy of it. This can't fail. */
 hs_ident_dir_conn_t *
 hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
@@ -55,7 +51,7 @@ hs_ident_dir_conn_dup(const hs_ident_dir_conn_t *src)
   return ident;
 }
 
-/* Free the given directory connection identifier. */
+/** Free the given directory connection identifier. */
 void
 hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
 {
@@ -66,7 +62,7 @@ hs_ident_dir_conn_free_(hs_ident_dir_conn_t *ident)
   tor_free(ident);
 }
 
-/* Initialized the allocated ident object with identity_pk and blinded_pk.
+/** Initialized the allocated ident object with identity_pk and blinded_pk.
  * None of them can be NULL since a valid directory connection identifier must
  * have all fields set. */
 void
@@ -82,7 +78,7 @@ hs_ident_dir_conn_init(const ed25519_public_key_t *identity_pk,
   ed25519_pubkey_copy(&ident->blinded_pk, blinded_pk);
 }
 
-/* Return a newly allocated edge connection identifier. The given public key
+/** Return a newly allocated edge connection identifier. The given public key
  * identity_pk is copied into the identifier. */
 hs_ident_edge_conn_t *
 hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
@@ -92,7 +88,7 @@ hs_ident_edge_conn_new(const ed25519_public_key_t *identity_pk)
   return ident;
 }
 
-/* Free the given edge connection identifier. */
+/** Free the given edge connection identifier. */
 void
 hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
 {
@@ -103,7 +99,7 @@ hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident)
   tor_free(ident);
 }
 
-/* Return true if the given ident is valid for an introduction circuit. */
+/** Return true if the given ident is valid for an introduction circuit. */
 int
 hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
 {
@@ -124,4 +120,3 @@ hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident)
  invalid:
   return 0;
 }
-
diff --git a/src/feature/hs/hs_ident.h b/src/feature/hs/hs_ident.h
index 8c46936a1e..f4b9b2432d 100644
--- a/src/feature/hs/hs_ident.h
+++ b/src/feature/hs/hs_ident.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,77 +25,71 @@
 
 #include "feature/hs/hs_common.h"
 
-/* Length of the rendezvous cookie that is used to connect circuits at the
+/** Length of the rendezvous cookie that is used to connect circuits at the
  * rendezvous point. */
 #define HS_REND_COOKIE_LEN DIGEST_LEN
 
-/* Type of circuit an hs_ident_t object is associated with. */
+/** Type of circuit an hs_ident_t object is associated with. */
 typedef enum {
   HS_IDENT_CIRCUIT_INTRO      = 1,
   HS_IDENT_CIRCUIT_RENDEZVOUS = 2,
 } hs_ident_circuit_type_t;
 
-/* Client and service side circuit identifier that is used for hidden service
+/** Client and service side circuit identifier that is used for hidden service
  * circuit establishment. Not all fields contain data, it depends on the
  * circuit purpose. This is attached to an origin_circuit_t. All fields are
  * used by both client and service. */
 typedef struct hs_ident_circuit_t {
-  /* (All circuit) The public key used to uniquely identify the service. It is
+  /** (All circuit) The public key used to uniquely identify the service. It is
    * the one found in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* (All circuit) The type of circuit this identifier is attached to.
-   * Accessors of the fields in this object assert non fatal on this circuit
-   * type. In other words, if a rendezvous field is being accessed, the
-   * circuit type MUST BE of type HS_IDENT_CIRCUIT_RENDEZVOUS. This value is
-   * set when an object is initialized in its constructor. */
-  hs_ident_circuit_type_t circuit_type;
-
-  /* (All circuit) Introduction point authentication key. It's also needed on
+  /** (All circuit) Introduction point authentication key. It's also needed on
    * the rendezvous circuit for the ntor handshake. It's used as the unique key
    * of the introduction point so it should not be shared between multiple
    * intro points. */
   ed25519_public_key_t intro_auth_pk;
 
-  /* (Only client rendezvous circuit) Introduction point encryption public
+  /** (Only client rendezvous circuit) Introduction point encryption public
    * key. We keep it in the rendezvous identifier for the ntor handshake. */
   curve25519_public_key_t intro_enc_pk;
 
-  /* (Only rendezvous circuit) Rendezvous cookie sent from the client to the
+  /** (Only rendezvous circuit) Rendezvous cookie sent from the client to the
    * service with an INTRODUCE1 cell and used by the service in an
    * RENDEZVOUS1 cell. */
   uint8_t rendezvous_cookie[HS_REND_COOKIE_LEN];
 
-  /* (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
+  /** (Only service rendezvous circuit) The HANDSHAKE_INFO needed in the
    * RENDEZVOUS1 cell of the service. The construction is as follows:
-   *    SERVER_PK   [32 bytes]
-   *    AUTH_MAC    [32 bytes]
+   *
+   *      SERVER_PK   [32 bytes]
+   *      AUTH_MAC    [32 bytes]
    */
   uint8_t rendezvous_handshake_info[CURVE25519_PUBKEY_LEN + DIGEST256_LEN];
 
-  /* (Only client rendezvous circuit) Client ephemeral keypair needed for the
+  /** (Only client rendezvous circuit) Client ephemeral keypair needed for the
    * e2e encryption with the service. */
   curve25519_keypair_t rendezvous_client_kp;
 
-  /* (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
+  /** (Only rendezvous circuit) The NTOR_KEY_SEED needed for key derivation for
    * the e2e encryption with the client on the circuit. */
   uint8_t rendezvous_ntor_key_seed[DIGEST256_LEN];
 
-  /* (Only rendezvous circuit) Number of streams associated with this
+  /** (Only rendezvous circuit) Number of streams associated with this
    * rendezvous circuit. We track this because there is a check on a maximum
    * value. */
   uint64_t num_rdv_streams;
 } hs_ident_circuit_t;
 
-/* Client and service side directory connection identifier used for a
+/** Client and service side directory connection identifier used for a
  * directory connection to identify which service is being queried. This is
  * attached to a dir_connection_t. */
 typedef struct hs_ident_dir_conn_t {
-  /* The public key used to uniquely identify the service. It is the one found
+  /** The public key used to uniquely identify the service. It is the one found
    * in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* The blinded public key used to uniquely identify the descriptor that this
+  /** The blinded public key used to uniquely identify the descriptor that this
    * directory connection identifier is for. Only used by the service-side code
    * to fine control descriptor uploads. */
   ed25519_public_key_t blinded_pk;
@@ -103,15 +97,15 @@ typedef struct hs_ident_dir_conn_t {
   /* XXX: Client authorization. */
 } hs_ident_dir_conn_t;
 
-/* Client and service side edge connection identifier used for an edge
+/** Client and service side edge connection identifier used for an edge
  * connection to identify which service is being queried. This is attached to
  * a edge_connection_t. */
 typedef struct hs_ident_edge_conn_t {
-  /* The public key used to uniquely identify the service. It is the one found
+  /** The public key used to uniquely identify the service. It is the one found
    * in the onion address. */
   ed25519_public_key_t identity_pk;
 
-  /* The original virtual port that was used by the client to access the onion
+  /** The original virtual port that was used by the client to access the onion
    * service, regardless of the internal port forwarding that might have
    * happened on the service-side. */
   uint16_t orig_virtual_port;
@@ -120,8 +114,7 @@ typedef struct hs_ident_edge_conn_t {
 
 /* Circuit identifier API. */
 hs_ident_circuit_t *hs_ident_circuit_new(
-                             const ed25519_public_key_t *identity_pk,
-                             hs_ident_circuit_type_t circuit_type);
+                             const ed25519_public_key_t *identity_pk);
 void hs_ident_circuit_free_(hs_ident_circuit_t *ident);
 #define hs_ident_circuit_free(id) \
   FREE_AND_NULL(hs_ident_circuit_t, hs_ident_circuit_free_, (id))
@@ -147,4 +140,3 @@ void hs_ident_edge_conn_free_(hs_ident_edge_conn_t *ident);
 int hs_ident_intro_circ_is_valid(const hs_ident_circuit_t *ident);
 
 #endif /* !defined(TOR_HS_IDENT_H) */
-
diff --git a/src/feature/hs/hs_intropoint.c b/src/feature/hs/hs_intropoint.c
index 7717ed53d4..69d60f21c3 100644
--- a/src/feature/hs/hs_intropoint.c
+++ b/src/feature/hs/hs_intropoint.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,6 +10,7 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "core/or/relay.h"
@@ -24,9 +25,11 @@
 #include "trunnel/hs/cell_introduce1.h"
 
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
 #include "feature/hs/hs_descriptor.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
-#include "feature/hs/hs_common.h"
 
 #include "core/or/or_circuit_st.h"
 
@@ -144,7 +147,7 @@ verify_establish_intro_cell(const trn_cell_establish_intro_t *cell,
   return 0;
 }
 
-/* Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
+/** Send an INTRO_ESTABLISHED cell to <b>circ</b>. */
 MOCK_IMPL(int,
 hs_intro_send_intro_established_cell,(or_circuit_t *circ))
 {
@@ -179,6 +182,190 @@ hs_intro_send_intro_established_cell,(or_circuit_t *circ))
   return ret;
 }
 
+/** Validate the cell DoS extension parameters. Return true iff they've been
+ * bound check and can be used. Else return false. See proposal 305 for
+ * details and reasons about this validation. */
+STATIC bool
+cell_dos_extension_parameters_are_valid(uint64_t intro2_rate_per_sec,
+                                        uint64_t intro2_burst_per_sec)
+{
+  bool ret = false;
+
+  /* Check that received value is not below the minimum. Don't check if minimum
+     is set to 0, since the param is a positive value and gcc will complain. */
+#if HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0
+  if (intro2_rate_per_sec < HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too small. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_rate_per_sec > HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses rate per second is "
+           "too big. Received value: %" PRIu64, intro2_rate_per_sec);
+    goto end;
+  }
+
+  /* Check that received value is not below the minimum */
+#if HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0
+  if (intro2_burst_per_sec < HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too small. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+#endif /* HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MIN > 0 */
+
+  /* Check that received value is not above maximum */
+  if (intro2_burst_per_sec > HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_MAX) {
+    log_fn(LOG_PROTOCOL_WARN, LD_REND,
+           "Intro point DoS defenses burst per second is "
+           "too big. Received value: %" PRIu64, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* In a rate limiting scenario, burst can never be smaller than the rate. At
+   * best it can be equal. */
+  if (intro2_burst_per_sec < intro2_rate_per_sec) {
+    log_info(LD_REND, "Intro point DoS defenses burst is smaller than rate. "
+                      "Rate: %" PRIu64 " vs Burst: %" PRIu64,
+             intro2_rate_per_sec, intro2_burst_per_sec);
+    goto end;
+  }
+
+  /* Passing validation. */
+  ret = true;
+
+ end:
+  return ret;
+}
+
+/** Parse the cell DoS extension and apply defenses on the given circuit if
+ * validation passes. If the cell extension is malformed or contains unusable
+ * values, the DoS defenses is disabled on the circuit. */
+static void
+handle_establish_intro_cell_dos_extension(
+                                const trn_cell_extension_field_t *field,
+                                or_circuit_t *circ)
+{
+  ssize_t ret;
+  uint64_t intro2_rate_per_sec = 0, intro2_burst_per_sec = 0;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  tor_assert(field);
+  tor_assert(circ);
+
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  if (ret < 0) {
+    goto end;
+  }
+
+  for (size_t i = 0; i < trn_cell_extension_dos_get_n_params(dos); i++) {
+    const trn_cell_extension_dos_param_t *param =
+      trn_cell_extension_dos_getconst_params(dos, i);
+    if (BUG(param == NULL)) {
+      goto end;
+    }
+
+    switch (trn_cell_extension_dos_param_get_type(param)) {
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC:
+      intro2_rate_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    case TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC:
+      intro2_burst_per_sec = trn_cell_extension_dos_param_get_value(param);
+      break;
+    default:
+      goto end;
+    }
+  }
+
+  /* At this point, the extension is valid so any values out of it implies
+   * that it was set explicitly and thus flag the circuit that it should not
+   * look at the consensus for that reason for the defenses' values. */
+  circ->introduce2_dos_defense_explicit = 1;
+
+  /* A value of 0 is valid in the sense that we accept it but we still disable
+   * the defenses so return false. */
+  if (intro2_rate_per_sec == 0 || intro2_burst_per_sec == 0) {
+    log_info(LD_REND, "Intro point DoS defenses parameter set to 0. "
+                      "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    circ->introduce2_dos_defense_enabled = 0;
+    goto end;
+  }
+
+  /* If invalid, we disable the defense on the circuit. */
+  if (!cell_dos_extension_parameters_are_valid(intro2_rate_per_sec,
+                                               intro2_burst_per_sec)) {
+    circ->introduce2_dos_defense_enabled = 0;
+    log_info(LD_REND, "Disabling INTRO2 DoS defenses on circuit id %u",
+             circ->p_circ_id);
+    goto end;
+  }
+
+  /* We passed validation, enable defenses and apply rate/burst. */
+  circ->introduce2_dos_defense_enabled = 1;
+
+  /* Initialize the INTRODUCE2 token bucket for the rate limiting. */
+  token_bucket_ctr_init(&circ->introduce2_bucket,
+                        (uint32_t) intro2_rate_per_sec,
+                        (uint32_t) intro2_burst_per_sec,
+                        (uint32_t) approx_time());
+  log_info(LD_REND, "Intro point DoS defenses enabled. Rate is %" PRIu64
+                    " and Burst is %" PRIu64,
+           intro2_rate_per_sec, intro2_burst_per_sec);
+
+ end:
+  trn_cell_extension_dos_free(dos);
+  return;
+}
+
+/** Parse every cell extension in the given ESTABLISH_INTRO cell. */
+static void
+handle_establish_intro_cell_extensions(
+                            const trn_cell_establish_intro_t *parsed_cell,
+                            or_circuit_t *circ)
+{
+  const trn_cell_extension_t *extensions;
+
+  tor_assert(parsed_cell);
+  tor_assert(circ);
+
+  extensions = trn_cell_establish_intro_getconst_extensions(parsed_cell);
+  if (extensions == NULL) {
+    goto end;
+  }
+
+  /* Go over all extensions. */
+  for (size_t idx = 0; idx < trn_cell_extension_get_num(extensions); idx++) {
+    const trn_cell_extension_field_t *field =
+      trn_cell_extension_getconst_fields(extensions, idx);
+    if (BUG(field == NULL)) {
+      /* The number of extensions should match the number of fields. */
+      break;
+    }
+
+    switch (trn_cell_extension_field_get_field_type(field)) {
+    case TRUNNEL_CELL_EXTENSION_TYPE_DOS:
+      /* After this, the circuit should be set for DoS defenses. */
+      handle_establish_intro_cell_dos_extension(field, circ);
+      break;
+    default:
+      /* Unknown extension. Skip over. */
+      break;
+    }
+  }
+
+ end:
+  return;
+}
+
 /** We received an ESTABLISH_INTRO <b>parsed_cell</b> on <b>circ</b>. It's
  *  well-formed and passed our verifications. Perform appropriate actions to
  *  establish an intro point. */
@@ -191,6 +378,13 @@ handle_verified_establish_intro_cell(or_circuit_t *circ,
   get_auth_key_from_cell(&auth_key, RELAY_COMMAND_ESTABLISH_INTRO,
                          parsed_cell);
 
+  /* Setup INTRODUCE2 defenses on the circuit. Must be done before parsing the
+   * cell extension that can possibly change the defenses' values. */
+  hs_dos_setup_default_intro2_defenses(circ);
+
+  /* Handle cell extension if any. */
+  handle_establish_intro_cell_extensions(parsed_cell, circ);
+
   /* Then notify the hidden service that the intro point is established by
      sending an INTRO_ESTABLISHED cell */
   if (hs_intro_send_intro_established_cell(circ)) {
@@ -268,7 +462,7 @@ handle_establish_intro(or_circuit_t *circ, const uint8_t *request,
   return retval;
 }
 
-/* Return True if circuit is suitable for being an intro circuit. */
+/** Return True if circuit is suitable for being an intro circuit. */
 static int
 circuit_is_suitable_intro_point(const or_circuit_t *circ,
                                 const char *log_cell_type_str)
@@ -293,14 +487,14 @@ circuit_is_suitable_intro_point(const or_circuit_t *circ,
   return 1;
 }
 
-/* Return True if circuit is suitable for being service-side intro circuit. */
+/** Return True if circuit is suitable for being service-side intro circuit. */
 int
 hs_intro_circuit_is_suitable_for_establish_intro(const or_circuit_t *circ)
 {
   return circuit_is_suitable_intro_point(circ, "ESTABLISH_INTRO");
 }
 
-/* We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
+/** We just received an ESTABLISH_INTRO cell in <b>circ</b>. Figure out of it's
  * a legacy or a next gen cell, and pass it to the appropriate handler. */
 int
 hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
@@ -334,7 +528,7 @@ hs_intro_received_establish_intro(or_circuit_t *circ, const uint8_t *request,
   return -1;
 }
 
-/* Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
+/** Send an INTRODUCE_ACK cell onto the circuit <b>circ</b> with the status
  * value in <b>status</b>. Depending on the status, it can be ACK or a NACK.
  * Return 0 on success else a negative value on error which will close the
  * circuit. */
@@ -378,7 +572,7 @@ send_introduce_ack_cell(or_circuit_t *circ, uint16_t status)
   return ret;
 }
 
-/* Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
+/** Validate a parsed INTRODUCE1 <b>cell</b>. Return 0 if valid or else a
  * negative value for an invalid cell that should be NACKed. */
 STATIC int
 validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
@@ -392,7 +586,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
    * safety net here. The legacy ID must be zeroes in this case. */
   legacy_key_id_len = trn_cell_introduce1_getlen_legacy_key_id(cell);
   legacy_key_id = trn_cell_introduce1_getconstarray_legacy_key_id(cell);
-  if (BUG(!tor_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
+  if (BUG(!fast_mem_is_zero((char *) legacy_key_id, legacy_key_id_len))) {
     goto invalid;
   }
 
@@ -424,7 +618,7 @@ validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell)
   return -1;
 }
 
-/* We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
+/** We just received a non legacy INTRODUCE1 cell on <b>client_circ</b> with
  * the payload in <b>request</b> of size <b>request_len</b>. Return 0 if
  * everything went well, or -1 if an error occurred. This function is in charge
  * of sending back an INTRODUCE_ACK cell and will close client_circ on error.
@@ -480,6 +674,20 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
     }
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(service_circ)) {
+    char *msg;
+    static ratelim_t rlimit = RATELIM_INIT(5 * 60);
+    if ((msg = rate_limit_log(&rlimit, approx_time()))) {
+      log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v3 cell due to DoS "
+                            "limitations. Sending NACK to client.");
+      tor_free(msg);
+    }
+    status = TRUNNEL_HS_INTRO_ACK_STATUS_UNKNOWN_ID;
+    goto send_ack;
+  }
+
   /* Relay the cell to the service on its intro circuit with an INTRODUCE2
    * cell which is the same exact payload. */
   if (relay_send_command_from_edge(CONTROL_CELL_ID, TO_CIRCUIT(service_circ),
@@ -509,7 +717,7 @@ handle_introduce1(or_circuit_t *client_circ, const uint8_t *request,
   return ret;
 }
 
-/* Identify if the encoded cell we just received is a legacy one or not. The
+/** Identify if the encoded cell we just received is a legacy one or not. The
  * <b>request</b> should be at least DIGEST_LEN bytes long. */
 STATIC int
 introduce1_cell_is_legacy(const uint8_t *request)
@@ -518,7 +726,7 @@ introduce1_cell_is_legacy(const uint8_t *request)
 
   /* If the first 20 bytes of the cell (DIGEST_LEN) are NOT zeroes, it
    * indicates a legacy cell (v2). */
-  if (!tor_mem_is_zero((const char *) request, DIGEST_LEN)) {
+  if (!fast_mem_is_zero((const char *) request, DIGEST_LEN)) {
     /* Legacy cell. */
     return 1;
   }
@@ -526,7 +734,7 @@ introduce1_cell_is_legacy(const uint8_t *request)
   return 0;
 }
 
-/* Return true iff the circuit <b>circ</b> is suitable for receiving an
+/** Return true iff the circuit <b>circ</b> is suitable for receiving an
  * INTRODUCE1 cell. */
 STATIC int
 circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
@@ -546,10 +754,18 @@ circuit_is_suitable_for_introduce1(const or_circuit_t *circ)
     return 0;
   }
 
+  /* Disallow single hop client circuit. */
+  if (circ->p_chan && channel_is_client(circ->p_chan)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Single hop client was rejected while trying to introduce. "
+           "Closing circuit.");
+    return 0;
+  }
+
   return 1;
 }
 
-/* We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
+/** We just received an INTRODUCE1 cell on <b>circ</b>. Figure out which type
  * it is and pass it to the appropriate handler. Return 0 on success else a
  * negative value and the circuit is closed. */
 int
@@ -593,8 +809,8 @@ hs_intro_received_introduce1(or_circuit_t *circ, const uint8_t *request,
   return -1;
 }
 
-/* Clear memory allocated by the given intropoint object ip (but don't free the
- * object itself). */
+/** Clear memory allocated by the given intropoint object ip (but don't free
+ * the object itself). */
 void
 hs_intropoint_clear(hs_intropoint_t *ip)
 {
@@ -602,8 +818,8 @@ hs_intropoint_clear(hs_intropoint_t *ip)
     return;
   }
   tor_cert_free(ip->auth_key_cert);
-  SMARTLIST_FOREACH(ip->link_specifiers, hs_desc_link_specifier_t *, ls,
-                    hs_desc_link_specifier_free(ls));
+  SMARTLIST_FOREACH(ip->link_specifiers, link_specifier_t *, ls,
+                    link_specifier_free(ls));
   smartlist_free(ip->link_specifiers);
   memset(ip, 0, sizeof(hs_intropoint_t));
 }
diff --git a/src/feature/hs/hs_intropoint.h b/src/feature/hs/hs_intropoint.h
index e82575f052..8b2b9892b3 100644
--- a/src/feature/hs/hs_intropoint.h
+++ b/src/feature/hs/hs_intropoint.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,15 +12,15 @@
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "feature/nodelist/torcert.h"
 
-/* Object containing introduction point common data between the service and
+/** Object containing introduction point common data between the service and
  * the client side. */
 typedef struct hs_intropoint_t {
-  /* Does this intro point only supports legacy ID ?. */
+  /** Does this intro point only supports legacy ID ?. */
   unsigned int is_only_legacy : 1;
 
-  /* Authentication key certificate from the descriptor. */
+  /** Authentication key certificate from the descriptor. */
   tor_cert_t *auth_key_cert;
-  /* A list of link specifier. */
+  /** A list of link specifier. */
   smartlist_t *link_specifiers;
 } hs_intropoint_t;
 
@@ -57,8 +57,10 @@ STATIC int handle_introduce1(or_circuit_t *client_circ,
                              const uint8_t *request, size_t request_len);
 STATIC int validate_introduce1_parsed_cell(const trn_cell_introduce1_t *cell);
 STATIC int circuit_is_suitable_for_introduce1(const or_circuit_t *circ);
+STATIC bool cell_dos_extension_parameters_are_valid(
+                                          uint64_t intro2_rate_per_sec,
+                                          uint64_t intro2_burst_per_sec);
 
 #endif /* defined(HS_INTROPOINT_PRIVATE) */
 
 #endif /* !defined(TOR_HS_INTRO_H) */
-
diff --git a/src/feature/hs/hs_ob.c b/src/feature/hs/hs_ob.c
new file mode 100644
index 0000000000..9499c28d20
--- /dev/null
+++ b/src/feature/hs/hs_ob.c
@@ -0,0 +1,408 @@
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ob.c
+ * \brief Implement Onion Balance specific code.
+ **/
+
+#define HS_OB_PRIVATE
+
+#include "feature/hs/hs_service.h"
+
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "lib/confmgt/confmgt.h"
+#include "lib/encoding/confline.h"
+
+#include "feature/hs/hs_ob.h"
+
+/* Options config magic number. */
+#define OB_OPTIONS_MAGIC 0x631DE7EA
+
+/* Helper macros. */
+#define VAR(varname, conftype, member, initvalue)                          \
+  CONFIG_VAR_ETYPE(ob_options_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)        \
+  VAR(#member, conftype, member, initvalue)
+
+/* Dummy instance of ob_options_t, used for type-checking its members with
+ * CONF_CHECK_VAR_TYPE. */
+DUMMY_TYPECHECK_INSTANCE(ob_options_t);
+
+/* Array of variables for the config file options. */
+static const config_var_t config_vars[] = {
+  V(MasterOnionAddress, LINELIST, NULL),
+
+  END_OF_CONFIG_VARS
+};
+
+/* "Extra" variable in the state that receives lines we can't parse. This
+ * lets us preserve options from versions of Tor newer than us. */
+static const struct_member_t config_extra_vars = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(ob_options_t, ExtraLines),
+};
+
+/* Configuration format of ob_options_t. */
+static const config_format_t config_format = {
+  .size = sizeof(ob_options_t),
+  .magic = {
+     "ob_options_t",
+     OB_OPTIONS_MAGIC,
+     offsetof(ob_options_t, magic_),
+  },
+  .vars = config_vars,
+  .extra = &config_extra_vars,
+};
+
+/* Global configuration manager for the config file. */
+static config_mgr_t *config_options_mgr = NULL;
+
+/* Return the configuration manager for the config file. */
+static const config_mgr_t *
+get_config_options_mgr(void)
+{
+  if (PREDICT_UNLIKELY(config_options_mgr == NULL)) {
+    config_options_mgr = config_mgr_new(&config_format);
+    config_mgr_freeze(config_options_mgr);
+  }
+  return config_options_mgr;
+}
+
+#define ob_option_free(val) \
+  FREE_AND_NULL(ob_options_t, ob_option_free_, (val))
+
+/** Helper: Free a config options object. */
+static void
+ob_option_free_(ob_options_t *opts)
+{
+  if (opts == NULL) {
+    return;
+  }
+  config_free(get_config_options_mgr(), opts);
+}
+
+/** Return an allocated config options object. */
+static ob_options_t *
+ob_option_new(void)
+{
+  ob_options_t *opts = config_new(get_config_options_mgr());
+  config_init(get_config_options_mgr(), opts);
+  return opts;
+}
+
+/** Helper function: From the configuration line value which is an onion
+ * address with the ".onion" extension, find the public key and put it in
+ * pkey_out.
+ *
+ * On success, true is returned. Else, false and pkey is untouched. */
+static bool
+get_onion_public_key(const char *value, ed25519_public_key_t *pkey_out)
+{
+  char address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+  tor_assert(value);
+  tor_assert(pkey_out);
+
+  if (strcmpend(value, ".onion")) {
+    /* Not a .onion extension, bad format. */
+    return false;
+  }
+
+  /* Length validation. The -1 is because sizeof() counts the NUL byte. */
+  if (strlen(value) >
+      (HS_SERVICE_ADDR_LEN_BASE32 + sizeof(".onion") - 1)) {
+    /* Too long, bad format. */
+    return false;
+  }
+
+  /* We don't want the .onion so we add 2 because size - 1 is copied with
+   * strlcpy() in order to accomodate the NUL byte and sizeof() counts the NUL
+   * byte so we need to remove them from the equation. */
+  strlcpy(address, value, strlen(value) - sizeof(".onion") + 2);
+
+  if (hs_parse_address_no_log(address, pkey_out, NULL, NULL, NULL) < 0) {
+    return false;
+  }
+
+  /* Success. */
+  return true;
+}
+
+/** Parse the given ob options in opts and set the service config object
+ * accordingly.
+ *
+ * Return 1 on success else 0. */
+static int
+ob_option_parse(hs_service_config_t *config, const ob_options_t *opts)
+{
+  int ret = 0;
+  config_line_t *line;
+
+  tor_assert(config);
+  tor_assert(opts);
+
+  for (line = opts->MasterOnionAddress; line; line = line->next) {
+    /* Allocate config list if need be. */
+    if (!config->ob_master_pubkeys) {
+      config->ob_master_pubkeys = smartlist_new();
+    }
+    ed25519_public_key_t *pubkey = tor_malloc_zero(sizeof(*pubkey));
+
+    if (!get_onion_public_key(line->value, pubkey)) {
+      log_warn(LD_REND, "OnionBalance: MasterOnionAddress %s is invalid",
+               line->value);
+      tor_free(pubkey);
+      goto end;
+    }
+    smartlist_add(config->ob_master_pubkeys, pubkey);
+    log_notice(LD_REND, "OnionBalance: MasterOnionAddress %s registered",
+               line->value);
+  }
+  /* Success. */
+  ret = 1;
+
+ end:
+  /* No keys added, we free the list since no list means no onion balance
+   * support for this tor instance. */
+  if (smartlist_len(config->ob_master_pubkeys) == 0) {
+    smartlist_free(config->ob_master_pubkeys);
+  }
+  return ret;
+}
+
+/** For the given master public key and time period, compute the subcredential
+ * and put them into subcredential. The subcredential parameter needs to be at
+ * least DIGEST256_LEN in size. */
+static void
+build_subcredential(const ed25519_public_key_t *pkey, uint64_t tp,
+                    hs_subcredential_t *subcredential)
+{
+  ed25519_public_key_t blinded_pubkey;
+
+  tor_assert(pkey);
+  tor_assert(subcredential);
+
+  hs_build_blinded_pubkey(pkey, NULL, 0, tp, &blinded_pubkey);
+  hs_get_subcredential(pkey, &blinded_pubkey, subcredential);
+}
+
+/*
+ * Public API.
+ */
+
+/** Return true iff the given service is configured as an onion balance
+ * instance. To satisfy that condition, there must at least be one master
+ * ed25519 public key configured. */
+bool
+hs_ob_service_is_instance(const hs_service_t *service)
+{
+  if (BUG(service == NULL)) {
+    return false;
+  }
+
+  /* No list, we are not an instance. */
+  if (!service->config.ob_master_pubkeys) {
+    return false;
+  }
+
+  return smartlist_len(service->config.ob_master_pubkeys) > 0;
+}
+
+/** Read and parse the config file at fname on disk. The service config object
+ * is populated with the options if any.
+ *
+ * Return 1 on success else 0. This is to follow the "ok" convention in
+ * hs_config.c. */
+int
+hs_ob_parse_config_file(hs_service_config_t *config)
+{
+  static const char *fname = "ob_config";
+  int ret = 0;
+  char *content = NULL, *errmsg = NULL, *config_file_path = NULL;
+  ob_options_t *options = NULL;
+  config_line_t *lines = NULL;
+
+  tor_assert(config);
+
+  /* Read file from disk. */
+  config_file_path = hs_path_from_filename(config->directory_path, fname);
+  content = read_file_to_str(config_file_path, 0, NULL);
+  if (!content) {
+    log_warn(LD_FS, "OnionBalance: Unable to read config file %s",
+             escaped(config_file_path));
+    goto end;
+  }
+
+  /* Parse lines. */
+  if (config_get_lines(content, &lines, 0) < 0) {
+    goto end;
+  }
+
+  options = ob_option_new();
+  config_assign(get_config_options_mgr(), options, lines, 0, &errmsg);
+  if (errmsg) {
+    log_warn(LD_REND, "OnionBalance: Unable to parse config file: %s",
+             errmsg);
+    tor_free(errmsg);
+    goto end;
+  }
+
+  /* Parse the options and set the service config object with the details. */
+  ret = ob_option_parse(config, options);
+
+ end:
+  config_free_lines(lines);
+  ob_option_free(options);
+  tor_free(content);
+  tor_free(config_file_path);
+  return ret;
+}
+
+/** Compute all possible subcredentials for every onion master key in the given
+ * service config object. subcredentials_out is allocated and set as an
+ * continous array containing all possible values.
+ *
+ * On success, return the number of subcredential put in the array which will
+ * correspond to an arry of size: n * DIGEST256_LEN where DIGEST256_LEN is the
+ * length of a single subcredential.
+ *
+ * If the given configuration object has no OB master keys configured, 0 is
+ * returned and subcredentials_out is set to NULL.
+ *
+ * Otherwise, this can't fail. */
+STATIC size_t
+compute_subcredentials(const hs_service_t *service,
+                       hs_subcredential_t **subcredentials_out)
+{
+  unsigned int num_pkeys, idx = 0;
+  hs_subcredential_t *subcreds = NULL;
+  const int steps[3] = {0, -1, 1};
+  const unsigned int num_steps = ARRAY_LENGTH(steps);
+  const uint64_t tp = hs_get_time_period_num(0);
+
+  tor_assert(service);
+  tor_assert(subcredentials_out);
+  /* Our caller has checked these too */
+  tor_assert(service->desc_current);
+  tor_assert(service->desc_next);
+
+  /* Make sure we are an OB instance, or bail out. */
+  num_pkeys = smartlist_len(service->config.ob_master_pubkeys);
+  if (!num_pkeys) {
+    *subcredentials_out = NULL;
+    return 0;
+  }
+
+  /* Time to build all the subcredentials for each time period: two for each
+   * instance descriptor plus three for the onionbalance frontend service: the
+   * previous one (-1), the current one (0) and the next one (1) for each
+   * configured key in order to accomodate client and service consensus skew.
+   *
+   * If the client consensus after_time is at 23:00 but the service one is at
+   * 01:00, the client will be using the previous time period where the
+   * service will think it is the client next time period. Thus why we have
+   * to try them all.
+   *
+   * The normal use case works because the service gets the descriptor object
+   * that corresponds to the intro point's request, and because each
+   * descriptor corresponds to a specific subcredential, we get the right
+   * subcredential out of it, and use that to do the decryption.
+   *
+   * As a slight optimization, statistically, the current time period (0) will
+   * be the one to work first so we'll put them first in the array to maximize
+   * our chance of success. */
+
+  /* We use a flat array, not a smartlist_t, in order to minimize memory
+   * allocation.
+   *
+   * Size of array is: length of a single subcredential multiplied by the
+   * number of time period we need to compute and finally multiplied by the
+   * total number of keys we are about to process. In other words, for each
+   * key, we allocate 3 subcredential slots. Then in the end we also add two
+   * subcredentials for this instance's active descriptors. */
+  subcreds =
+    tor_calloc((num_steps * num_pkeys) + 2, sizeof(hs_subcredential_t));
+
+  /* For each master pubkey we add 3 subcredentials: */
+  for (unsigned int i = 0; i < num_steps; i++) {
+    SMARTLIST_FOREACH_BEGIN(service->config.ob_master_pubkeys,
+                            const ed25519_public_key_t *, pkey) {
+      build_subcredential(pkey, tp + steps[i], &subcreds[idx]);
+      idx++;
+    } SMARTLIST_FOREACH_END(pkey);
+  }
+
+  /* And then in the end we add the two subcredentials of the current active
+   * instance descriptors */
+  memcpy(&subcreds[idx++], &service->desc_current->desc->subcredential,
+         sizeof(hs_subcredential_t));
+  memcpy(&subcreds[idx++], &service->desc_next->desc->subcredential,
+         sizeof(hs_subcredential_t));
+
+  log_info(LD_REND, "Refreshing %u onionbalance keys (TP #%d).",
+             idx, (int)tp);
+
+  *subcredentials_out = subcreds;
+  return idx;
+}
+
+/**
+ *  If we are an Onionbalance instance, refresh our keys.
+ *
+ *  If we are not an Onionbalance instance or we are not ready to do so, this
+ *  is a NOP.
+ *
+ *  This function is called everytime we build a new descriptor. That's because
+ *  we want our Onionbalance keys to always use up-to-date subcredentials both
+ *  for the instance (ourselves) and for the onionbalance frontend.
+ */
+void
+hs_ob_refresh_keys(hs_service_t *service)
+{
+  hs_subcredential_t *ob_subcreds = NULL;
+  size_t num_subcreds;
+
+  tor_assert(service);
+
+  /* Don't do any of this if we are not configured as an OB instance */
+  if (!hs_ob_service_is_instance(service)) {
+    return;
+  }
+
+  /* We need both service descriptors created to make onionbalance keys.
+   *
+   * That's because we fetch our own (the instance's) subcredentials from our
+   * own descriptors which should always include the latest subcredentials that
+   * clients would use.
+   *
+   * This function is called with each descriptor build, so we will be
+   * eventually be called when both descriptors are created. */
+  if (!service->desc_current || !service->desc_next) {
+    return;
+  }
+
+  /* Get a new set of subcreds */
+  num_subcreds = compute_subcredentials(service, &ob_subcreds);
+  if (BUG(!num_subcreds)) {
+    return;
+  }
+
+  /* Delete old subcredentials if any */
+  if (service->state.ob_subcreds) {
+    tor_free(service->state.ob_subcreds);
+  }
+
+  service->state.ob_subcreds = ob_subcreds;
+  service->state.n_ob_subcreds = num_subcreds;
+}
+
+/** Free any memory allocated by the onionblance subsystem. */
+void
+hs_ob_free_all(void)
+{
+  config_mgr_free(config_options_mgr);
+}
diff --git a/src/feature/hs/hs_ob.h b/src/feature/hs/hs_ob.h
new file mode 100644
index 0000000000..d6e6e73a84
--- /dev/null
+++ b/src/feature/hs/hs_ob.h
@@ -0,0 +1,40 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file hs_ob.h
+ * \brief Header file for the specific code for onion balance.
+ **/
+
+#ifndef TOR_HS_OB_H
+#define TOR_HS_OB_H
+
+#include "feature/hs/hs_service.h"
+
+bool hs_ob_service_is_instance(const hs_service_t *service);
+
+int hs_ob_parse_config_file(hs_service_config_t *config);
+
+struct hs_subcredential_t;
+
+void hs_ob_free_all(void);
+
+void hs_ob_refresh_keys(hs_service_t *service);
+
+#ifdef HS_OB_PRIVATE
+
+STATIC size_t compute_subcredentials(const hs_service_t *service,
+                                   struct hs_subcredential_t **subcredentials);
+
+typedef struct ob_options_t {
+  /** Magic number to identify the structure in memory. */
+  uint32_t magic_;
+  /** Master Onion Address(es). */
+  struct config_line_t *MasterOnionAddress;
+  /** Extra Lines for configuration we might not know. */
+  struct config_line_t *ExtraLines;
+} ob_options_t;
+
+#endif /* defined(HS_OB_PRIVATE) */
+
+#endif /* !defined(TOR_HS_OB_H) */
diff --git a/src/feature/hs/hs_options.inc b/src/feature/hs/hs_options.inc
new file mode 100644
index 0000000000..1a1444fd05
--- /dev/null
+++ b/src/feature/hs/hs_options.inc
@@ -0,0 +1,36 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options.inc
+ * @brief Declare configuration options for a single hidden service.
+ *
+ * Note that this options file behaves differently from most, since it
+ * is not used directly by the options manager.  Instead, it is applied to
+ * a group of hidden service options starting with a HiddenServiceDir and
+ * extending up to the next HiddenServiceDir.
+ **/
+
+/** Holds configuration for a single hidden service. */
+BEGIN_CONF_STRUCT(hs_opts_t)
+
+CONF_VAR(HiddenServiceDir, FILENAME, 0, NULL)
+CONF_VAR(HiddenServiceDirGroupReadable, BOOL, 0, "0")
+CONF_VAR(HiddenServicePort, LINELIST, 0, NULL)
+// "-1" means "auto" here.
+CONF_VAR(HiddenServiceVersion, INT, 0, "-1")
+CONF_VAR(HiddenServiceAuthorizeClient, STRING, 0, NULL)
+CONF_VAR(HiddenServiceAllowUnknownPorts, BOOL, 0, "0")
+CONF_VAR(HiddenServiceMaxStreams, POSINT, 0, "0")
+CONF_VAR(HiddenServiceMaxStreamsCloseCircuit, BOOL, 0, "0")
+CONF_VAR(HiddenServiceNumIntroductionPoints, POSINT, 0, "3")
+CONF_VAR(HiddenServiceExportCircuitID, STRING, 0, NULL)
+CONF_VAR(HiddenServiceEnableIntroDoSDefense, BOOL, 0, "0")
+CONF_VAR(HiddenServiceEnableIntroDoSRatePerSec, POSINT, 0, "25")
+CONF_VAR(HiddenServiceEnableIntroDoSBurstPerSec, POSINT, 0, "200")
+CONF_VAR(HiddenServiceOnionBalanceInstance, BOOL, 0, "0")
+
+END_CONF_STRUCT(hs_opts_t)
diff --git a/src/feature/hs/hs_opts_st.h b/src/feature/hs/hs_opts_st.h
new file mode 100644
index 0000000000..279f0d6da6
--- /dev/null
+++ b/src/feature/hs/hs_opts_st.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dirauth_options_st.h
+ * @brief Structure hs_opts_t to hold options for a single hidden service.
+ **/
+
+#ifndef TOR_FEATURE_HS_HS_OPTS_ST_H
+#define TOR_FEATURE_HS_HS_OPTS_ST_H
+
+#include "lib/conf/confdecl.h"
+#define CONF_CONTEXT STRUCT
+#include "feature/hs/hs_options.inc"
+#undef CONF_CONTEXT
+
+/**
+ * An hs_opts_t holds the parsed options for a single HS configuration
+ * section.
+ *
+ * This name ends with 'opts' instead of 'options' to signal that it is not
+ * handled directly by the or_options_t configuration manager, but that
+ * first we partition the "HiddenService*" options by section.
+ **/
+typedef struct hs_opts_t hs_opts_t;
+
+#endif /* !defined(TOR_FEATURE_HS_HS_OPTS_ST_H) */
diff --git a/src/feature/hs/hs_service.c b/src/feature/hs/hs_service.c
index e820ce9d0b..c29f39c6b4 100644
--- a/src/feature/hs/hs_service.c
+++ b/src/feature/hs/hs_service.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,7 +31,6 @@
 #include "feature/rend/rendservice.h"
 #include "lib/crypt_ops/crypto_ope.h"
 #include "lib/crypt_ops/crypto_rand.h"
-#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "feature/hs/hs_circuit.h"
@@ -43,6 +42,7 @@
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 #include "feature/hs/hs_stats.h"
+#include "feature/hs/hs_ob.h"
 
 #include "feature/dircommon/dir_connection_st.h"
 #include "core/or/edge_connection_st.h"
@@ -68,7 +68,8 @@
 #include <unistd.h>
 #endif
 
-/* Helper macro. Iterate over every service in the global map. The var is the
+#ifndef COCCI
+/** Helper macro. Iterate over every service in the global map. The var is the
  * name of the service pointer. */
 #define FOR_EACH_SERVICE_BEGIN(var)                          \
     STMT_BEGIN                                               \
@@ -77,7 +78,7 @@
       var = *var##_iter;
 #define FOR_EACH_SERVICE_END } STMT_END ;
 
-/* Helper macro. Iterate over both current and previous descriptor of a
+/** Helper macro. Iterate over both current and previous descriptor of a
  * service. The var is the name of the descriptor pointer. This macro skips
  * any descriptor object of the service that is NULL. */
 #define FOR_EACH_DESCRIPTOR_BEGIN(service, var)                  \
@@ -89,6 +90,7 @@
                                 (var = service->desc_next);      \
       if (var == NULL) continue;
 #define FOR_EACH_DESCRIPTOR_END } STMT_END ;
+#endif /* !defined(COCCI) */
 
 /* Onion service directory file names. */
 static const char fname_keyfile_prefix[] = "hs_ed25519";
@@ -96,7 +98,7 @@ static const char dname_client_pubkeys[] = "authorized_clients";
 static const char fname_hostname[] = "hostname";
 static const char address_tld[] = "onion";
 
-/* Staging list of service object. When configuring service, we add them to
+/** Staging list of service object. When configuring service, we add them to
  * this list considered a staging area and they will get added to our global
  * map once the keys have been loaded. These two steps are separated because
  * loading keys requires that we are an actual running tor process. */
@@ -119,7 +121,7 @@ static int service_encode_descriptor(const hs_service_t *service,
                                      const ed25519_keypair_t *signing_kp,
                                      char **encoded_out);
 
-/* Helper: Function to compare two objects in the service map. Return 1 if the
+/** Helper: Function to compare two objects in the service map. Return 1 if the
  * two service have the same master public identity key. */
 static inline int
 hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
@@ -131,7 +133,7 @@ hs_service_ht_eq(const hs_service_t *first, const hs_service_t *second)
                            &second->keys.identity_pk);
 }
 
-/* Helper: Function for the service hash table code below. The key used is the
+/** Helper: Function for the service hash table code below. The key used is the
  * master public identity key which is ultimately the onion address. */
 static inline unsigned int
 hs_service_ht_hash(const hs_service_t *service)
@@ -141,7 +143,7 @@ hs_service_ht_hash(const hs_service_t *service)
                                    sizeof(service->keys.identity_pk.pubkey));
 }
 
-/* This is _the_ global hash map of hidden services which indexed the service
+/** This is _the_ global hash map of hidden services which indexed the service
  * contained in it by master public identity key which is roughly the onion
  * address of the service. */
 static struct hs_service_ht *hs_service_map;
@@ -151,13 +153,13 @@ HT_PROTOTYPE(hs_service_ht,      /* Name of hashtable. */
              hs_service_t,       /* Object contained in the map. */
              hs_service_node,    /* The name of the HT_ENTRY member. */
              hs_service_ht_hash, /* Hashing function. */
-             hs_service_ht_eq)   /* Compare function for objects. */
+             hs_service_ht_eq);  /* Compare function for objects. */
 
 HT_GENERATE2(hs_service_ht, hs_service_t, hs_service_node,
              hs_service_ht_hash, hs_service_ht_eq,
-             0.6, tor_reallocarray, tor_free_)
+             0.6, tor_reallocarray, tor_free_);
 
-/* Query the given service map with a public key and return a service object
+/** Query the given service map with a public key and return a service object
  * if found else NULL. It is also possible to set a directory path in the
  * search query. If pk is NULL, then it will be set to zero indicating the
  * hash table to compare the directory path instead. */
@@ -172,7 +174,7 @@ find_service(hs_service_ht *map, const ed25519_public_key_t *pk)
   return HT_FIND(hs_service_ht, map, &dummy_service);
 }
 
-/* Register the given service in the given map. If the service already exists
+/** Register the given service in the given map. If the service already exists
  * in the map, -1 is returned. On success, 0 is returned and the service
  * ownership has been transferred to the global map. */
 STATIC int
@@ -197,7 +199,7 @@ register_service(hs_service_ht *map, hs_service_t *service)
   return 0;
 }
 
-/* Remove a given service from the given map. If service is NULL or the
+/** Remove a given service from the given map. If service is NULL or the
  * service key is unset, return gracefully. */
 STATIC void
 remove_service(hs_service_ht *map, hs_service_t *service)
@@ -227,7 +229,7 @@ remove_service(hs_service_ht *map, hs_service_t *service)
   }
 }
 
-/* Set the default values for a service configuration object <b>c</b>. */
+/** Set the default values for a service configuration object <b>c</b>. */
 static void
 set_service_default_config(hs_service_config_t *c,
                            const or_options_t *options)
@@ -243,9 +245,12 @@ set_service_default_config(hs_service_config_t *c,
   c->is_single_onion = 0;
   c->dir_group_readable = 0;
   c->is_ephemeral = 0;
+  c->has_dos_defense_enabled = HS_CONFIG_V3_DOS_DEFENSE_DEFAULT;
+  c->intro_dos_rate_per_sec = HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT;
+  c->intro_dos_burst_per_sec = HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT;
 }
 
-/* From a service configuration object config, clear everything from it
+/** From a service configuration object config, clear everything from it
  * meaning free allocated pointers and reset the values. */
 STATIC void
 service_clear_config(hs_service_config_t *config)
@@ -264,10 +269,15 @@ service_clear_config(hs_service_config_t *config)
                       service_authorized_client_free(p));
     smartlist_free(config->clients);
   }
+  if (config->ob_master_pubkeys) {
+    SMARTLIST_FOREACH(config->ob_master_pubkeys, ed25519_public_key_t *, k,
+                      tor_free(k));
+    smartlist_free(config->ob_master_pubkeys);
+  }
   memset(config, 0, sizeof(*config));
 }
 
-/* Helper function to return a human readable description of the given intro
+/** Helper function to return a human readable description of the given intro
  * point object.
  *
  * This function is not thread-safe. Each call to this invalidates the
@@ -281,9 +291,10 @@ describe_intro_point(const hs_service_intro_point_t *ip)
   const char *legacy_id = NULL;
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, lspec) {
-    if (lspec->type == LS_LEGACY_ID) {
-      legacy_id = (const char *) lspec->u.legacy_id;
+                          const link_specifier_t *, lspec) {
+    if (link_specifier_get_ls_type(lspec) == LS_LEGACY_ID) {
+      legacy_id = (const char *)
+        link_specifier_getconstarray_un_legacy_id(lspec);
       break;
     }
   } SMARTLIST_FOREACH_END(lspec);
@@ -298,7 +309,7 @@ describe_intro_point(const hs_service_intro_point_t *ip)
   return buf;
 }
 
-/* Return the lower bound of maximum INTRODUCE2 cells per circuit before we
+/** Return the lower bound of maximum INTRODUCE2 cells per circuit before we
  * rotate intro point (defined by a consensus parameter or the default
  * value). */
 static int32_t
@@ -311,7 +322,7 @@ get_intro_point_min_introduce2(void)
                                  0, INT32_MAX);
 }
 
-/* Return the upper bound of maximum INTRODUCE2 cells per circuit before we
+/** Return the upper bound of maximum INTRODUCE2 cells per circuit before we
  * rotate intro point (defined by a consensus parameter or the default
  * value). */
 static int32_t
@@ -324,8 +335,8 @@ get_intro_point_max_introduce2(void)
                                  0, INT32_MAX);
 }
 
-/* Return the minimum lifetime in seconds of an introduction point defined by a
- * consensus parameter or the default value. */
+/** Return the minimum lifetime in seconds of an introduction point defined by
+ * a consensus parameter or the default value. */
 static int32_t
 get_intro_point_min_lifetime(void)
 {
@@ -341,8 +352,8 @@ get_intro_point_min_lifetime(void)
                                  0, INT32_MAX);
 }
 
-/* Return the maximum lifetime in seconds of an introduction point defined by a
- * consensus parameter or the default value. */
+/** Return the maximum lifetime in seconds of an introduction point defined by
+ * a consensus parameter or the default value. */
 static int32_t
 get_intro_point_max_lifetime(void)
 {
@@ -358,7 +369,7 @@ get_intro_point_max_lifetime(void)
                                  0, INT32_MAX);
 }
 
-/* Return the number of extra introduction point defined by a consensus
+/** Return the number of extra introduction point defined by a consensus
  * parameter or the default value. */
 static int32_t
 get_intro_point_num_extra(void)
@@ -369,7 +380,7 @@ get_intro_point_num_extra(void)
                                  NUM_INTRO_POINTS_EXTRA, 0, 128);
 }
 
-/* Helper: Function that needs to return 1 for the HT for each loop which
+/** Helper: Function that needs to return 1 for the HT for each loop which
  * frees every service in an hash map. */
 static int
 ht_free_service_(struct hs_service_t *service, void *data)
@@ -381,7 +392,7 @@ ht_free_service_(struct hs_service_t *service, void *data)
   return 1;
 }
 
-/* Free every service that can be found in the global map. Once done, clear
+/** Free every service that can be found in the global map. Once done, clear
  * and free the global map. */
 static void
 service_free_all(void)
@@ -403,7 +414,7 @@ service_free_all(void)
   }
 }
 
-/* Free a given service intro point object. */
+/** Free a given service intro point object. */
 STATIC void
 service_intro_point_free_(hs_service_intro_point_t *ip)
 {
@@ -418,7 +429,7 @@ service_intro_point_free_(hs_service_intro_point_t *ip)
   tor_free(ip);
 }
 
-/* Helper: free an hs_service_intro_point_t object. This function is used by
+/** Helper: free an hs_service_intro_point_t object. This function is used by
  * digest256map_free() which requires a void * pointer. */
 static void
 service_intro_point_free_void(void *obj)
@@ -426,24 +437,17 @@ service_intro_point_free_void(void *obj)
   service_intro_point_free_(obj);
 }
 
-/* Return a newly allocated service intro point and fully initialized from the
- * given extend_info_t ei if non NULL.
- * If is_legacy is true, we also generate the legacy key.
- * If supports_ed25519_link_handshake_any is true, we add the relay's ed25519
- * key to the link specifiers.
+/** Return a newly allocated service intro point and fully initialized from the
+ * given node_t node, if non NULL.
  *
- * If ei is NULL, returns a hs_service_intro_point_t with an empty link
+ * If node is NULL, returns a hs_service_intro_point_t with an empty link
  * specifier list and no onion key. (This is used for testing.)
  * On any other error, NULL is returned.
  *
- * ei must be an extend_info_t containing an IPv4 address. (We will add supoort
- * for IPv6 in a later release.) When calling extend_info_from_node(), pass
- * 0 in for_direct_connection to make sure ei always has an IPv4 address. */
+ * node must be an node_t with an IPv4 address. */
 STATIC hs_service_intro_point_t *
-service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
-                        unsigned int supports_ed25519_link_handshake_any)
+service_intro_point_new(const node_t *node)
 {
-  hs_desc_link_specifier_t *ls;
   hs_service_intro_point_t *ip;
 
   ip = tor_malloc_zero(sizeof(*ip));
@@ -473,12 +477,17 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
   ip->replay_cache = replaycache_new(0, 0);
 
   /* Initialize the base object. We don't need the certificate object. */
-  ip->base.link_specifiers = smartlist_new();
+  ip->base.link_specifiers = node_get_link_specifier_smartlist(node, 0);
+
+  if (node == NULL) {
+    goto done;
+  }
 
   /* Generate the encryption key for this intro point. */
   curve25519_keypair_generate(&ip->enc_key_kp, 0);
-  /* Figure out if this chosen node supports v3 or is legacy only. */
-  if (is_legacy) {
+  /* Figure out if this chosen node supports v3 or is legacy only.
+   * NULL nodes are used in the unit tests. */
+  if (!node_supports_ed25519_hs_intro(node)) {
     ip->base.is_only_legacy = 1;
     /* Legacy mode that is doesn't support v3+ with ed25519 auth key. */
     ip->legacy_key = crypto_pk_new();
@@ -491,40 +500,13 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
     }
   }
 
-  if (ei == NULL) {
-    goto done;
-  }
-
-  /* We'll try to add all link specifiers. Legacy is mandatory.
-   * IPv4 or IPv6 is required, and we always send IPv4. */
-  ls = hs_desc_link_specifier_new(ei, LS_IPV4);
-  /* It is impossible to have an extend info object without a v4. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  ls = hs_desc_link_specifier_new(ei, LS_LEGACY_ID);
-  /* It is impossible to have an extend info object without an identity
-   * digest. */
-  if (BUG(!ls)) {
-    goto err;
-  }
-  smartlist_add(ip->base.link_specifiers, ls);
-
-  /* ed25519 identity key is optional for intro points. If the node supports
-   * ed25519 link authentication, we include it. */
-  if (supports_ed25519_link_handshake_any) {
-    ls = hs_desc_link_specifier_new(ei, LS_ED25519_ID);
-    if (ls) {
-      smartlist_add(ip->base.link_specifiers, ls);
-    }
-  }
-
-  /* IPv6 is not supported in this release. */
+  /* Flag if this intro point supports the INTRO2 dos defenses. */
+  ip->support_intro2_dos_defense =
+    node_supports_establish_intro_dos_extension(node);
 
-  /* Finally, copy onion key from the extend_info_t object. */
-  memcpy(&ip->onion_key, &ei->curve25519_onion_key, sizeof(ip->onion_key));
+  /* Finally, copy onion key from the node. */
+  memcpy(&ip->onion_key, node_get_curve25519_onion_key(node),
+         sizeof(ip->onion_key));
 
  done:
   return ip;
@@ -533,7 +515,7 @@ service_intro_point_new(const extend_info_t *ei, unsigned int is_legacy,
   return NULL;
 }
 
-/* Add the given intro point object to the given intro point map. The intro
+/** Add the given intro point object to the given intro point map. The intro
  * point MUST have its RSA encryption key set if this is a legacy type or the
  * authentication key set otherwise. */
 STATIC void
@@ -549,7 +531,7 @@ service_intro_point_add(digest256map_t *map, hs_service_intro_point_t *ip)
   tor_assert_nonfatal(!old_ip_entry);
 }
 
-/* For a given service, remove the intro point from that service's descriptors
+/** For a given service, remove the intro point from that service's descriptors
  * (check both current and next descriptor) */
 STATIC void
 service_intro_point_remove(const hs_service_t *service,
@@ -567,7 +549,7 @@ service_intro_point_remove(const hs_service_t *service,
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* For a given service and authentication key, return the intro point or NULL
+/** For a given service and authentication key, return the intro point or NULL
  * if not found. This will check both descriptors in the service. */
 STATIC hs_service_intro_point_t *
 service_intro_point_find(const hs_service_t *service,
@@ -598,7 +580,7 @@ service_intro_point_find(const hs_service_t *service,
   return ip;
 }
 
-/* For a given service and intro point, return the descriptor for which the
+/** For a given service and intro point, return the descriptor for which the
  * intro point is assigned to. NULL is returned if not found. */
 STATIC hs_service_descriptor_t *
 service_desc_find_by_intro(const hs_service_t *service,
@@ -620,7 +602,7 @@ service_desc_find_by_intro(const hs_service_t *service,
   return descp;
 }
 
-/* From a circuit identifier, get all the possible objects associated with the
+/** From a circuit identifier, get all the possible objects associated with the
  * ident. If not NULL, service, ip or desc are set if the object can be found.
  * They are untouched if they can't be found.
  *
@@ -653,20 +635,20 @@ get_objects_from_ident(const hs_ident_circuit_t *ident,
   }
 }
 
-/* From a given intro point, return the first link specifier of type
+/** From a given intro point, return the first link specifier of type
  * encountered in the link specifier list. Return NULL if it can't be found.
  *
  * The caller does NOT have ownership of the object, the intro point does. */
-static hs_desc_link_specifier_t *
+static link_specifier_t *
 get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
 {
-  hs_desc_link_specifier_t *lnk_spec = NULL;
+  link_specifier_t *lnk_spec = NULL;
 
   tor_assert(ip);
 
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          hs_desc_link_specifier_t *, ls) {
-    if (ls->type == type) {
+                          link_specifier_t *, ls) {
+    if (link_specifier_get_ls_type(ls) == type) {
       lnk_spec = ls;
       goto end;
     }
@@ -676,13 +658,13 @@ get_link_spec_by_type(const hs_service_intro_point_t *ip, uint8_t type)
   return lnk_spec;
 }
 
-/* Given a service intro point, return the node_t associated to it. This can
+/** Given a service intro point, return the node_t associated to it. This can
  * return NULL if the given intro point has no legacy ID or if the node can't
  * be found in the consensus. */
 STATIC const node_t *
 get_node_from_intro_point(const hs_service_intro_point_t *ip)
 {
-  const hs_desc_link_specifier_t *ls;
+  const link_specifier_t *ls;
 
   tor_assert(ip);
 
@@ -691,10 +673,11 @@ get_node_from_intro_point(const hs_service_intro_point_t *ip)
     return NULL;
   }
   /* XXX In the future, we want to only use the ed25519 ID (#22173). */
-  return node_get_by_id((const char *) ls->u.legacy_id);
+  return node_get_by_id(
+    (const char *) link_specifier_getconstarray_un_legacy_id(ls));
 }
 
-/* Given a service intro point, return the extend_info_t for it. This can
+/** Given a service intro point, return the extend_info_t for it. This can
  * return NULL if the node can't be found for the intro point or the extend
  * info can't be created for the found node. If direct_conn is set, the extend
  * info is validated on if we can connect directly. */
@@ -723,10 +706,10 @@ get_extend_info_from_intro_point(const hs_service_intro_point_t *ip,
   return info;
 }
 
-/* Return the number of introduction points that are established for the
+/** Return the number of introduction points that are established for the
  * given descriptor. */
-static unsigned int
-count_desc_circuit_established(const hs_service_descriptor_t *desc)
+MOCK_IMPL(STATIC unsigned int,
+count_desc_circuit_established, (const hs_service_descriptor_t *desc))
 {
   unsigned int count = 0;
 
@@ -734,13 +717,13 @@ count_desc_circuit_established(const hs_service_descriptor_t *desc)
 
   DIGEST256MAP_FOREACH(desc->intro_points.map, key,
                        const hs_service_intro_point_t *, ip) {
-    count += ip->circuit_established;
+    count += !!hs_circ_service_get_established_intro_circ(ip);
   } DIGEST256MAP_FOREACH_END;
 
   return count;
 }
 
-/* For a given service and descriptor of that service, close all active
+/** For a given service and descriptor of that service, close all active
  * directory connections. */
 static void
 close_directory_connections(const hs_service_t *service,
@@ -775,7 +758,7 @@ close_directory_connections(const hs_service_t *service,
   smartlist_free(dir_conns);
 }
 
-/* Close all rendezvous circuits for the given service. */
+/** Close all rendezvous circuits for the given service. */
 static void
 close_service_rp_circuits(hs_service_t *service)
 {
@@ -805,7 +788,7 @@ close_service_rp_circuits(hs_service_t *service)
   }
 }
 
-/* Close the circuit(s) for the given map of introduction points. */
+/** Close the circuit(s) for the given map of introduction points. */
 static void
 close_intro_circuits(hs_service_intropoints_t *intro_points)
 {
@@ -823,7 +806,7 @@ close_intro_circuits(hs_service_intropoints_t *intro_points)
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Close all introduction circuits for the given service. */
+/** Close all introduction circuits for the given service. */
 static void
 close_service_intro_circuits(hs_service_t *service)
 {
@@ -834,7 +817,7 @@ close_service_intro_circuits(hs_service_t *service)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Close any circuits related to the given service. */
+/** Close any circuits related to the given service. */
 static void
 close_service_circuits(hs_service_t *service)
 {
@@ -850,7 +833,7 @@ close_service_circuits(hs_service_t *service)
   close_service_rp_circuits(service);
 }
 
-/* Move every ephemeral services from the src service map to the dst service
+/** Move every ephemeral services from the src service map to the dst service
  * map. It is possible that a service can't be register to the dst map which
  * won't stop the process of moving them all but will trigger a log warn. */
 static void
@@ -880,7 +863,7 @@ move_ephemeral_services(hs_service_ht *src, hs_service_ht *dst)
   }
 }
 
-/* Return a const string of the directory path escaped. If this is an
+/** Return a const string of the directory path escaped. If this is an
  * ephemeral service, it returns "[EPHEMERAL]". This can only be called from
  * the main thread because escaped() uses a static variable. */
 static const char *
@@ -908,13 +891,21 @@ move_hs_state(hs_service_t *src_service, hs_service_t *dst_service)
   if (dst->replay_cache_rend_cookie != NULL) {
     replaycache_free(dst->replay_cache_rend_cookie);
   }
+
   dst->replay_cache_rend_cookie = src->replay_cache_rend_cookie;
+  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+
   dst->next_rotation_time = src->next_rotation_time;
 
-  src->replay_cache_rend_cookie = NULL; /* steal pointer reference */
+  if (src->ob_subcreds) {
+    dst->ob_subcreds = src->ob_subcreds;
+    dst->n_ob_subcreds =  src->n_ob_subcreds;
+
+    src->ob_subcreds = NULL; /* steal pointer reference */
+  }
 }
 
-/* Register services that are in the staging list. Once this function returns,
+/** Register services that are in the staging list. Once this function returns,
  * the global service map will be set with the right content and all non
  * surviving services will be cleaned up. */
 static void
@@ -982,7 +973,7 @@ register_all_services(void)
   hs_service_map_has_changed();
 }
 
-/* Write the onion address of a given service to the given filename fname_ in
+/** Write the onion address of a given service to the given filename fname_ in
  * the service directory. Return 0 on success else -1 on error. */
 STATIC int
 write_address_to_file(const hs_service_t *service, const char *fname_)
@@ -1023,7 +1014,7 @@ write_address_to_file(const hs_service_t *service, const char *fname_)
   return ret;
 }
 
-/* Load and/or generate private keys for the given service. On success, the
+/** Load and/or generate private keys for the given service. On success, the
  * hostname file will be written to disk along with the master private key iff
  * the service is not configured for offline keys. Return 0 on success else -1
  * on failure. */
@@ -1099,7 +1090,7 @@ load_service_keys(hs_service_t *service)
   return ret;
 }
 
-/* Check if the client file name is valid or not. Return 1 if valid,
+/** Check if the client file name is valid or not. Return 1 if valid,
  * otherwise return 0. */
 STATIC int
 client_filename_is_valid(const char *filename)
@@ -1121,7 +1112,7 @@ client_filename_is_valid(const char *filename)
   return ret;
 }
 
-/* Parse an authorized client from a string. The format of a client string
+/** Parse an authorized client from a string. The format of a client string
  * looks like (see rend-spec-v3.txt):
  *
  *  <auth-type>:<key-type>:<base32-encoded-public-key>
@@ -1180,7 +1171,8 @@ parse_authorized_client(const char *client_key_str)
   client = tor_malloc_zero(sizeof(hs_service_authorized_client_t));
   if (base32_decode((char *) client->client_pk.public_key,
                     sizeof(client->client_pk.public_key),
-                    pubkey_b32, strlen(pubkey_b32)) < 0) {
+                    pubkey_b32, strlen(pubkey_b32)) !=
+      sizeof(client->client_pk.public_key)) {
     log_warn(LD_REND, "Client authorization public key cannot be decoded: %s",
              pubkey_b32);
     goto err;
@@ -1202,7 +1194,7 @@ parse_authorized_client(const char *client_key_str)
   return client;
 }
 
-/* Load all the client public keys for the given service. Return 0 on
+/** Load all the client public keys for the given service. Return 0 on
  * success else -1 on failure. */
 static int
 load_client_keys(hs_service_t *service)
@@ -1262,7 +1254,7 @@ load_client_keys(hs_service_t *service)
     client_key_str = read_file_to_str(client_key_file_path, 0, NULL);
 
     /* If we cannot read the file, continue with the next file. */
-    if (!client_key_str)  {
+    if (!client_key_str) {
       log_warn(LD_REND, "Client authorization file %s can't be read. "
                         "Corrupted or verify permission? Ignoring.",
                client_key_file_path);
@@ -1305,6 +1297,7 @@ load_client_keys(hs_service_t *service)
   return ret;
 }
 
+/** Release all storage held in <b>client</b>. */
 STATIC void
 service_authorized_client_free_(hs_service_authorized_client_t *client)
 {
@@ -1315,7 +1308,7 @@ service_authorized_client_free_(hs_service_authorized_client_t *client)
   tor_free(client);
 }
 
-/* Free a given service descriptor object and all key material is wiped. */
+/** Free a given service descriptor object and all key material is wiped. */
 STATIC void
 service_descriptor_free_(hs_service_descriptor_t *desc)
 {
@@ -1336,7 +1329,7 @@ service_descriptor_free_(hs_service_descriptor_t *desc)
   tor_free(desc);
 }
 
-/* Return a newly allocated service descriptor object. */
+/** Return a newly allocated service descriptor object. */
 STATIC hs_service_descriptor_t *
 service_descriptor_new(void)
 {
@@ -1349,7 +1342,7 @@ service_descriptor_new(void)
   return sdesc;
 }
 
-/* Allocate and return a deep copy of client. */
+/** Allocate and return a deep copy of client. */
 static hs_service_authorized_client_t *
 service_authorized_client_dup(const hs_service_authorized_client_t *client)
 {
@@ -1367,7 +1360,7 @@ service_authorized_client_dup(const hs_service_authorized_client_t *client)
   return client_dup;
 }
 
-/* If two authorized clients are equal, return 0. If the first one should come
+/** If two authorized clients are equal, return 0. If the first one should come
  * before the second, return less than zero. If the first should come after
  * the second, return greater than zero. */
 static int
@@ -1384,7 +1377,7 @@ service_authorized_client_cmp(const hs_service_authorized_client_t *client1,
                     CURVE25519_PUBKEY_LEN);
 }
 
-/* Helper for sorting authorized clients. */
+/** Helper for sorting authorized clients. */
 static int
 compare_service_authorzized_client_(const void **_a, const void **_b)
 {
@@ -1392,7 +1385,7 @@ compare_service_authorzized_client_(const void **_a, const void **_b)
   return service_authorized_client_cmp(a, b);
 }
 
-/* If the list of hs_service_authorized_client_t's is different between
+/** If the list of hs_service_authorized_client_t's is different between
  * src and dst, return 1. Otherwise, return 0. */
 STATIC int
 service_authorized_client_config_equal(const hs_service_config_t *config1,
@@ -1453,7 +1446,7 @@ service_authorized_client_config_equal(const hs_service_config_t *config1,
   return ret;
 }
 
-/* Move descriptor(s) from the src service to the dst service and modify their
+/** Move descriptor(s) from the src service to the dst service and modify their
  * content if necessary. We do this during SIGHUP when we re-create our
  * hidden services. */
 static void
@@ -1512,7 +1505,7 @@ move_descriptors(hs_service_t *src, hs_service_t *dst)
   service_descriptor_free(dst->desc_next);
 }
 
-/* From the given service, remove all expired failing intro points for each
+/** From the given service, remove all expired failing intro points for each
  * descriptor. */
 static void
 remove_expired_failing_intro(hs_service_t *service, time_t now)
@@ -1531,7 +1524,7 @@ remove_expired_failing_intro(hs_service_t *service, time_t now)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* For the given descriptor desc, put all node_t object found from its failing
+/** For the given descriptor desc, put all node_t object found from its failing
  * intro point list and put them in the given node_list. */
 static void
 setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
@@ -1549,7 +1542,7 @@ setup_intro_point_exclude_list(const hs_service_descriptor_t *desc,
   } DIGESTMAP_FOREACH_END;
 }
 
-/* For the given failing intro point ip, we add its time of failure to the
+/** For the given failing intro point ip, we add its time of failure to the
  * failed map and index it by identity digest (legacy ID) in the descriptor
  * desc failed id map. */
 static void
@@ -1557,7 +1550,7 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
                              hs_service_descriptor_t *desc, time_t now)
 {
   time_t *time_of_failure, *prev_ptr;
-  const hs_desc_link_specifier_t *legacy_ls;
+  const link_specifier_t *legacy_ls;
 
   tor_assert(ip);
   tor_assert(desc);
@@ -1566,23 +1559,14 @@ remember_failing_intro_point(const hs_service_intro_point_t *ip,
   *time_of_failure = now;
   legacy_ls = get_link_spec_by_type(ip, LS_LEGACY_ID);
   tor_assert(legacy_ls);
-  prev_ptr = digestmap_set(desc->intro_points.failed_id,
-                           (const char *) legacy_ls->u.legacy_id,
-                           time_of_failure);
+  prev_ptr = digestmap_set(
+    desc->intro_points.failed_id,
+    (const char *) link_specifier_getconstarray_un_legacy_id(legacy_ls),
+    time_of_failure);
   tor_free(prev_ptr);
 }
 
-/* Copy the descriptor link specifier object from src to dst. */
-static void
-link_specifier_copy(hs_desc_link_specifier_t *dst,
-                    const hs_desc_link_specifier_t *src)
-{
-  tor_assert(dst);
-  tor_assert(src);
-  memcpy(dst, src, sizeof(hs_desc_link_specifier_t));
-}
-
-/* Using a given descriptor signing keypair signing_kp, a service intro point
+/** Using a given descriptor signing keypair signing_kp, a service intro point
  * object ip and the time now, setup the content of an already allocated
  * descriptor intro desc_ip.
  *
@@ -1616,9 +1600,14 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
 
   /* Copy link specifier(s). */
   SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                          const hs_desc_link_specifier_t *, ls) {
-    hs_desc_link_specifier_t *copy = tor_malloc_zero(sizeof(*copy));
-    link_specifier_copy(copy, ls);
+                          const link_specifier_t *, ls) {
+    if (BUG(!ls)) {
+      goto done;
+    }
+    link_specifier_t *copy = link_specifier_dup(ls);
+    if (BUG(!copy)) {
+      goto done;
+    }
     smartlist_add(desc_ip->link_specifiers, copy);
   } SMARTLIST_FOREACH_END(ls);
 
@@ -1666,7 +1655,7 @@ setup_desc_intro_point(const ed25519_keypair_t *signing_kp,
   return ret;
 }
 
-/* Using the given descriptor from the given service, build the descriptor
+/** Using the given descriptor from the given service, build the descriptor
  * intro point list so we can then encode the descriptor for publication. This
  * function does not pick intro points, they have to be in the descriptor
  * current map. Cryptographic material (keys) must be initialized in the
@@ -1687,7 +1676,7 @@ build_desc_intro_points(const hs_service_t *service,
 
   DIGEST256MAP_FOREACH(desc->intro_points.map, key,
                        const hs_service_intro_point_t *, ip) {
-    if (!ip->circuit_established) {
+    if (!hs_circ_service_get_established_intro_circ(ip)) {
       /* Ignore un-established intro points. They can linger in that list
        * because their circuit has not opened and they haven't been removed
        * yet even though we have enough intro circuits.
@@ -1706,7 +1695,7 @@ build_desc_intro_points(const hs_service_t *service,
   } DIGEST256MAP_FOREACH_END;
 }
 
-/* Build the descriptor signing key certificate. */
+/** Build the descriptor signing key certificate. */
 static void
 build_desc_signing_key_cert(hs_service_descriptor_t *desc, time_t now)
 {
@@ -1732,7 +1721,7 @@ build_desc_signing_key_cert(hs_service_descriptor_t *desc, time_t now)
   tor_assert_nonfatal(plaintext->signing_key_cert);
 }
 
-/* Populate the descriptor encrypted section from the given service object.
+/** Populate the descriptor encrypted section from the given service object.
  * This will generate a valid list of introduction points that can be used
  * after for circuit creation. Return 0 on success else -1 on error. */
 static int
@@ -1762,7 +1751,7 @@ build_service_desc_encrypted(const hs_service_t *service,
   return 0;
 }
 
-/* Populate the descriptor superencrypted section from the given service
+/** Populate the descriptor superencrypted section from the given service
  * object. This will generate a valid list of hs_desc_authorized_client_t
  * of clients that are authorized to use the service. Return 0 on success
  * else -1 on error. */
@@ -1790,7 +1779,8 @@ build_service_desc_superencrypted(const hs_service_t *service,
          sizeof(curve25519_public_key_t));
 
   /* Test that subcred is not zero because we might use it below */
-  if (BUG(tor_mem_is_zero((char*)desc->desc->subcredential, DIGEST256_LEN))) {
+  if (BUG(fast_mem_is_zero((char*)desc->desc->subcredential.subcred,
+                           DIGEST256_LEN))) {
     return -1;
   }
 
@@ -1807,7 +1797,7 @@ build_service_desc_superencrypted(const hs_service_t *service,
 
       /* Prepare the client for descriptor and then add to the list in the
        * superencrypted part of the descriptor */
-      hs_desc_build_authorized_client(desc->desc->subcredential,
+      hs_desc_build_authorized_client(&desc->desc->subcredential,
                                       &client->client_pk,
                                       &desc->auth_ephemeral_kp.seckey,
                                       desc->descriptor_cookie, desc_client);
@@ -1845,7 +1835,7 @@ build_service_desc_superencrypted(const hs_service_t *service,
   return 0;
 }
 
-/* Populate the descriptor plaintext section from the given service object.
+/** Populate the descriptor plaintext section from the given service object.
  * The caller must make sure that the keys in the descriptors are valid that
  * is are non-zero. This can't fail. */
 static void
@@ -1856,14 +1846,14 @@ build_service_desc_plaintext(const hs_service_t *service,
 
   tor_assert(service);
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &desc->blinded_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->blinded_kp,
                               sizeof(desc->blinded_kp)));
-  tor_assert(!tor_mem_is_zero((char *) &desc->signing_kp,
+  tor_assert(!fast_mem_is_zero((char *) &desc->signing_kp,
                               sizeof(desc->signing_kp)));
 
   /* Set the subcredential. */
   hs_get_subcredential(&service->keys.identity_pk, &desc->blinded_kp.pubkey,
-                       desc->desc->subcredential);
+                       &desc->desc->subcredential);
 
   plaintext = &desc->desc->plaintext_data;
 
@@ -1896,7 +1886,7 @@ generate_ope_cipher_for_desc(const hs_service_descriptor_t *hs_desc)
   return crypto_ope_new(key);
 }
 
-/* For the given service and descriptor object, create the key material which
+/** For the given service and descriptor object, create the key material which
  * is the blinded keypair, the descriptor signing keypair, the ephemeral
  * keypair, and the descriptor cookie. Return 0 on success else -1 on error
  * where the generated keys MUST be ignored. */
@@ -1908,7 +1898,7 @@ build_service_desc_keys(const hs_service_t *service,
   ed25519_keypair_t kp;
 
   tor_assert(desc);
-  tor_assert(!tor_mem_is_zero((char *) &service->keys.identity_pk,
+  tor_assert(!fast_mem_is_zero((char *) &service->keys.identity_pk,
              ED25519_PUBKEY_LEN));
 
   /* XXX: Support offline key feature (#18098). */
@@ -1958,7 +1948,7 @@ build_service_desc_keys(const hs_service_t *service,
   return ret;
 }
 
-/* Given a service and the current time, build a descriptor for the service.
+/** Given a service and the current time, build a descriptor for the service.
  * This function does not pick introduction point, this needs to be done by
  * the update function. On success, desc_out will point to the newly allocated
  * descriptor object.
@@ -2006,16 +1996,22 @@ build_service_descriptor(hs_service_t *service, uint64_t time_period_num,
 
   /* Assign newly built descriptor to the next slot. */
   *desc_out = desc;
+
   /* Fire a CREATED control port event. */
   hs_control_desc_event_created(service->onion_address,
                                 &desc->blinded_kp.pubkey);
+
+  /* If we are an onionbalance instance, we refresh our keys when we rotate
+   * descriptors. */
+  hs_ob_refresh_keys(service);
+
   return;
 
  err:
   service_descriptor_free(desc);
 }
 
-/* Build both descriptors for the given service that has just booted up.
+/** Build both descriptors for the given service that has just booted up.
  * Because it's a special case, it deserves its special function ;). */
 static void
 build_descriptors_for_new_service(hs_service_t *service, time_t now)
@@ -2065,7 +2061,7 @@ build_descriptors_for_new_service(hs_service_t *service, time_t now)
            safe_str_client(service->onion_address));
 }
 
-/* Build descriptors for each service if needed. There are conditions to build
+/** Build descriptors for each service if needed. There are conditions to build
  * a descriptor which are details in the function. */
 STATIC void
 build_all_descriptors(time_t now)
@@ -2098,7 +2094,7 @@ build_all_descriptors(time_t now)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Randomly pick a node to become an introduction point but not present in the
+/** Randomly pick a node to become an introduction point but not present in the
  * given exclude_nodes list. The chosen node is put in the exclude list
  * regardless of success or not because in case of failure, the node is simply
  * unsusable from that point on.
@@ -2117,7 +2113,6 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
 {
   const or_options_t *options = get_options();
   const node_t *node;
-  extend_info_t *info = NULL;
   hs_service_intro_point_t *ip = NULL;
   /* Normal 3-hop introduction point flags. */
   router_crn_flags_t flags = CRN_NEED_UPTIME | CRN_NEED_DESC;
@@ -2146,47 +2141,21 @@ pick_intro_point(unsigned int direct_conn, smartlist_t *exclude_nodes)
    * we don't want to use that node anymore. */
   smartlist_add(exclude_nodes, (void *) node);
 
-  /* We do this to ease our life but also this call makes appropriate checks
-   * of the node object such as validating ntor support for instance.
-   *
-   * We must provide an extend_info for clients to connect over a 3-hop path,
-   * so we don't pass direct_conn here. */
-  info = extend_info_from_node(node, 0);
-  if (BUG(info == NULL)) {
-    goto err;
-  }
-
-  /* Let's do a basic sanity check here so that we don't end up advertising the
-   * ed25519 identity key of relays that don't actually support the link
-   * protocol */
-  if (!node_supports_ed25519_link_authentication(node, 0)) {
-    tor_assert_nonfatal(ed25519_public_key_is_zero(&info->ed_identity));
-  } else {
-    /* Make sure we *do* have an ed key if we support the link authentication.
-     * Sending an empty key would result in a failure to extend. */
-    tor_assert_nonfatal(!ed25519_public_key_is_zero(&info->ed_identity));
-  }
+  /* Create our objects and populate them with the node information. */
+  ip = service_intro_point_new(node);
 
-  /* Create our objects and populate them with the node information.
-   * We don't care if the intro's link auth is compatible with us, because
-   * we are sending the ed25519 key to a remote client via the descriptor. */
-  ip = service_intro_point_new(info, !node_supports_ed25519_hs_intro(node),
-                               node_supports_ed25519_link_authentication(node,
-                                                                         0));
   if (ip == NULL) {
     goto err;
   }
 
-  log_info(LD_REND, "Picked intro point: %s", extend_info_describe(info));
-  extend_info_free(info);
+  log_info(LD_REND, "Picked intro point: %s", node_describe(node));
   return ip;
  err:
   service_intro_point_free(ip);
-  extend_info_free(info);
   return NULL;
 }
 
-/* For a given descriptor from the given service, pick any needed intro points
+/** For a given descriptor from the given service, pick any needed intro points
  * and update the current map with those newly picked intro points. Return the
  * number node that might have been added to the descriptor current map. */
 static unsigned int
@@ -2310,7 +2279,7 @@ service_desc_schedule_upload(hs_service_descriptor_t *desc,
   }
 }
 
-/* Pick missing intro points for this descriptor if needed. */
+/** Pick missing intro points for this descriptor if needed. */
 static void
 update_service_descriptor_intro_points(hs_service_t *service,
                           hs_service_descriptor_t *desc, time_t now)
@@ -2351,7 +2320,7 @@ update_service_descriptor_intro_points(hs_service_t *service,
   }
 }
 
-/* Update descriptor intro points for each service if needed. We do this as
+/** Update descriptor intro points for each service if needed. We do this as
  * part of the periodic event because we need to establish intro point circuits
  * before we publish descriptors. */
 STATIC void
@@ -2366,7 +2335,7 @@ update_all_descriptors_intro_points(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Return true iff the given intro point has expired that is it has been used
+/** Return true iff the given intro point has expired that is it has been used
  * for too long or we've reached our max seen INTRODUCE2 cell. */
 STATIC int
 intro_point_should_expire(const hs_service_intro_point_t *ip,
@@ -2388,15 +2357,66 @@ intro_point_should_expire(const hs_service_intro_point_t *ip,
   return 1;
 }
 
-/* Go over the given set of intro points for each service and remove any
- * invalid ones. The conditions for removal are:
+/** Return true iff we should remove the intro point ip from its service.
+ *
+ * We remove an intro point from the service descriptor list if one of
+ * these criteria is met:
+ *    - It has expired (either in INTRO2 count or in time).
+ *    - No node was found (fell off the consensus).
+ *    - We are over the maximum amount of retries.
  *
- *    - The node doesn't exists anymore (not in consensus)
- *                          OR
- *    - The intro point maximum circuit retry count has been reached and no
- *      circuit can be found associated with it.
- *                          OR
- *    - The intro point has expired and we should pick a new one.
+ * If an established or pending circuit is found for the given ip object, this
+ * return false indicating it should not be removed. */
+static bool
+should_remove_intro_point(hs_service_intro_point_t *ip, time_t now)
+{
+  bool ret = false;
+
+  tor_assert(ip);
+
+  /* Any one of the following needs to be True to furfill the criteria to
+   * remove an intro point. */
+  bool has_no_retries = (ip->circuit_retries >
+                         MAX_INTRO_POINT_CIRCUIT_RETRIES);
+  bool has_no_node = (get_node_from_intro_point(ip) == NULL);
+  bool has_expired = intro_point_should_expire(ip, now);
+
+  /* If the node fell off the consensus or the IP has expired, we have to
+   * remove it now. */
+  if (has_no_node || has_expired) {
+    ret = true;
+    goto end;
+  }
+
+  /* Pass this point, even though we might be over the retry limit, we check
+   * if a circuit (established or pending) exists. In that case, we should not
+   * remove it because it might simply be valid and opened at the previous
+   * scheduled event for the last retry. */
+
+  /* Do we simply have an existing circuit regardless of its state? */
+  if (hs_circ_service_get_intro_circ(ip)) {
+    goto end;
+  }
+
+  /* Getting here means we have _no_ circuits so then return if we have any
+   * remaining retries. */
+  ret = has_no_retries;
+
+ end:
+  /* Meaningful log in case we are about to remove the IP. */
+  if (ret) {
+    log_info(LD_REND, "Intro point %s%s (retried: %u times). "
+                      "Removing it.",
+             describe_intro_point(ip),
+             has_expired ? " has expired" :
+               (has_no_node) ?  " fell off the consensus" : "",
+             ip->circuit_retries);
+  }
+  return ret;
+}
+
+/** Go over the given set of intro points for each service and remove any
+ * invalid ones.
  *
  * If an intro point is removed, the circuit (if any) is immediately close.
  * If a circuit can't be found, the intro point is kept if it hasn't reached
@@ -2405,12 +2425,10 @@ static void
 cleanup_intro_points(hs_service_t *service, time_t now)
 {
   /* List of intro points to close. We can't mark the intro circuits for close
-   * in the modify loop because doing so calls
-   * hs_service_intro_circ_has_closed() which does a digest256map_get() on the
-   * intro points map (that we are iterating over). This can't be done in a
-   * single iteration after a MAP_DEL_CURRENT, the object will still be
-   * returned leading to a use-after-free. So, we close the circuits and free
-   * the intro points after the loop if any. */
+   * in the modify loop because doing so calls back into the HS subsystem and
+   * we need to keep that code path outside of the service/desc loop so those
+   * maps don't get modified during the close making us in a possible
+   * use-after-free situation. */
   smartlist_t *ips_to_free = smartlist_new();
 
   tor_assert(service);
@@ -2421,21 +2439,7 @@ cleanup_intro_points(hs_service_t *service, time_t now)
      * valid and remove any of them that aren't. */
     DIGEST256MAP_FOREACH_MODIFY(desc->intro_points.map, key,
                                 hs_service_intro_point_t *, ip) {
-      const node_t *node = get_node_from_intro_point(ip);
-      int has_expired = intro_point_should_expire(ip, now);
-
-      /* We cleanup an intro point if it has expired or if we do not know the
-       * node_t anymore (removed from our latest consensus) or if we've
-       * reached the maximum number of retry with a non existing circuit. */
-      if (has_expired || node == NULL ||
-          ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
-        log_info(LD_REND, "Intro point %s%s (retried: %u times). "
-                          "Removing it.",
-                 describe_intro_point(ip),
-                 has_expired ? " has expired" :
-                    (node == NULL) ?  " fell off the consensus" : "",
-                 ip->circuit_retries);
-
+      if (should_remove_intro_point(ip, now)) {
         /* We've retried too many times, remember it as a failed intro point
          * so we don't pick it up again for INTRO_CIRC_RETRY_PERIOD sec. */
         if (ip->circuit_retries > MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -2472,7 +2476,7 @@ cleanup_intro_points(hs_service_t *service, time_t now)
   smartlist_free(ips_to_free);
 }
 
-/* Set the next rotation time of the descriptors for the given service for the
+/** Set the next rotation time of the descriptors for the given service for the
  * time now. */
 static void
 set_rotation_time(hs_service_t *service)
@@ -2491,7 +2495,7 @@ set_rotation_time(hs_service_t *service)
   }
 }
 
-/* Return true iff the service should rotate its descriptor. The time now is
+/** Return true iff the service should rotate its descriptor. The time now is
  * only used to fetch the live consensus and if none can be found, this
  * returns false. */
 static unsigned int
@@ -2544,7 +2548,7 @@ should_rotate_descriptors(hs_service_t *service, time_t now)
   return 1;
 }
 
-/* Rotate the service descriptors of the given service. The current descriptor
+/** Rotate the service descriptors of the given service. The current descriptor
  * will be freed, the next one put in as the current and finally the next
  * descriptor pointer is NULLified. */
 static void
@@ -2566,7 +2570,7 @@ rotate_service_descriptors(hs_service_t *service)
   set_rotation_time(service);
 }
 
-/* Rotate descriptors for each service if needed. A non existing current
+/** Rotate descriptors for each service if needed. A non existing current
  * descriptor will trigger a descriptor build for the next time period. */
 STATIC void
 rotate_all_descriptors(time_t now)
@@ -2595,7 +2599,7 @@ rotate_all_descriptors(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Scheduled event run from the main loop. Make sure all our services are up
+/** Scheduled event run from the main loop. Make sure all our services are up
  * to date and ready for the other scheduled events. This includes looking at
  * the introduction points status and descriptor rotation time. */
 STATIC void
@@ -2630,7 +2634,7 @@ run_housekeeping_event(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Scheduled event run from the main loop. Make sure all descriptors are up to
+/** Scheduled event run from the main loop. Make sure all descriptors are up to
  * date. Once this returns, each service descriptor needs to be considered for
  * new introduction circuits and then for upload. */
 static void
@@ -2653,7 +2657,7 @@ run_build_descriptor_event(time_t now)
   update_all_descriptors_intro_points(now);
 }
 
-/* For the given service, launch any intro point circuits that could be
+/** For the given service, launch any intro point circuits that could be
  * needed. This considers every descriptor of the service. */
 static void
 launch_intro_point_circuits(hs_service_t *service)
@@ -2707,7 +2711,7 @@ launch_intro_point_circuits(hs_service_t *service)
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Don't try to build more than this many circuits before giving up for a
+/** Don't try to build more than this many circuits before giving up for a
  * while. Dynamically calculated based on the configured number of intro
  * points for the given service and how many descriptor exists. The default
  * use case of 3 introduction points and two descriptors will allow 28
@@ -2723,7 +2727,7 @@ get_max_intro_circ_per_period(const hs_service_t *service)
   tor_assert(service->config.num_intro_points <=
              HS_CONFIG_V3_MAX_INTRO_POINTS);
 
-/* For a testing network, allow to do it for the maximum amount so circuit
+/** For a testing network, allow to do it for the maximum amount so circuit
  * creation and rotation and so on can actually be tested without limit. */
 #define MAX_INTRO_POINT_CIRCUIT_RETRIES_TESTING -1
   if (get_options()->TestingTorNetwork) {
@@ -2752,7 +2756,7 @@ get_max_intro_circ_per_period(const hs_service_t *service)
   return (count * multiplier);
 }
 
-/* For the given service, return 1 if the service is allowed to launch more
+/** For the given service, return 1 if the service is allowed to launch more
  * introduction circuits else 0 if the maximum has been reached for the retry
  * period of INTRO_CIRC_RETRY_PERIOD. */
 STATIC int
@@ -2798,7 +2802,7 @@ can_service_launch_intro_circuit(hs_service_t *service, time_t now)
   return 1;
 }
 
-/* Scheduled event run from the main loop. Make sure we have all the circuits
+/** Scheduled event run from the main loop. Make sure we have all the circuits
  * we need for each service. */
 static void
 run_build_circuit_event(time_t now)
@@ -2828,7 +2832,7 @@ run_build_circuit_event(time_t now)
   } FOR_EACH_SERVICE_END;
 }
 
-/* Encode and sign the service descriptor desc and upload it to the given
+/** Encode and sign the service descriptor desc and upload it to the given
  * hidden service directory.  This does nothing if PublishHidServDescriptors
  * is false. */
 static void
@@ -2844,7 +2848,7 @@ upload_descriptor_to_hsdir(const hs_service_t *service,
   /* Let's avoid doing that if tor is configured to not publish. */
   if (!get_options()->PublishHidServDescriptors) {
     log_info(LD_REND, "Service %s not publishing descriptor. "
-                      "PublishHidServDescriptors is set to 1.",
+                      "PublishHidServDescriptors is set to 0.",
              safe_str_client(service->onion_address));
     goto end;
   }
@@ -2958,13 +2962,13 @@ set_descriptor_revision_counter(hs_service_descriptor_t *hs_desc, time_t now,
   /* The OPE module returns CRYPTO_OPE_ERROR in case of errors. */
   tor_assert_nonfatal(rev_counter < CRYPTO_OPE_ERROR);
 
-  log_info(LD_REND, "Encrypted revision counter %d to %ld",
-           (int) seconds_since_start_of_srv, (long int) rev_counter);
+  log_info(LD_REND, "Encrypted revision counter %d to %" PRIu64,
+           (int) seconds_since_start_of_srv, rev_counter);
 
   hs_desc->desc->plaintext_data.revision_counter = rev_counter;
 }
 
-/* Encode and sign the service descriptor desc and upload it to the
+/** Encode and sign the service descriptor desc and upload it to the
  * responsible hidden service directories. If for_next_period is true, the set
  * of directories are selected using the next hsdir_index. This does nothing
  * if PublishHidServDescriptors is false. */
@@ -3061,13 +3065,85 @@ service_desc_hsdirs_changed(const hs_service_t *service,
   return should_reupload;
 }
 
-/* Return 1 if the given descriptor from the given service can be uploaded
+/** These are all the reasons why a descriptor upload can't occur. We use
+ * those to log the reason properly with the right rate limiting and for the
+ * right descriptor. */
+typedef enum {
+  LOG_DESC_UPLOAD_REASON_MISSING_IPS = 0,
+  LOG_DESC_UPLOAD_REASON_IP_NOT_ESTABLISHED = 1,
+  LOG_DESC_UPLOAD_REASON_NOT_TIME = 2,
+  LOG_DESC_UPLOAD_REASON_NO_LIVE_CONSENSUS = 3,
+  LOG_DESC_UPLOAD_REASON_NO_DIRINFO = 4,
+} log_desc_upload_reason_t;
+
+/** Maximum number of reasons. This is used to allocate the static array of
+ * all rate limiting objects. */
+#define LOG_DESC_UPLOAD_REASON_MAX LOG_DESC_UPLOAD_REASON_NO_DIRINFO
+
+/** Log the reason why we can't upload the given descriptor for the given
+ * service. This takes a message string (allocated by the caller) and a
+ * reason.
+ *
+ * Depending on the reason and descriptor, different rate limit applies. This
+ * is done because this function will basically be called every second. Each
+ * descriptor for each reason uses its own log rate limit object in order to
+ * avoid message suppression for different reasons and descriptors. */
+static void
+log_cant_upload_desc(const hs_service_t *service,
+                     const hs_service_descriptor_t *desc, const char *msg,
+                     const log_desc_upload_reason_t reason)
+{
+  /* Writing the log every minute shouldn't be too annoying for log rate limit
+   * since this can be emitted every second for each descriptor.
+   *
+   * However, for one specific case, we increase it to 10 minutes because it
+   * is hit constantly, as an expected behavior, which is the reason
+   * indicating that it is not the time to upload. */
+  static ratelim_t limits[2][LOG_DESC_UPLOAD_REASON_MAX + 1] =
+    { { RATELIM_INIT(60), RATELIM_INIT(60), RATELIM_INIT(60 * 10),
+        RATELIM_INIT(60), RATELIM_INIT(60) },
+      { RATELIM_INIT(60), RATELIM_INIT(60), RATELIM_INIT(60 * 10),
+        RATELIM_INIT(60), RATELIM_INIT(60) },
+    };
+  bool is_next_desc = false;
+  unsigned int rlim_pos = 0;
+  ratelim_t *rlim = NULL;
+
+  tor_assert(service);
+  tor_assert(desc);
+  tor_assert(msg);
+
+  /* Make sure the reason value is valid. It should never happen because we
+   * control that value in the code flow but will be apparent during
+   * development if a reason is added but LOG_DESC_UPLOAD_REASON_NUM_ is not
+   * updated. */
+  if (BUG(reason > LOG_DESC_UPLOAD_REASON_MAX)) {
+    return;
+  }
+
+  /* Ease our life. Flag that tells us if the descriptor is the next one. */
+  is_next_desc = (service->desc_next == desc);
+
+  /* Current descriptor is the first element in the ratelimit object array.
+   * The next descriptor is the second element. */
+  rlim_pos = (is_next_desc ? 1 : 0);
+  /* Get the ratelimit object for the reason _and_ right descriptor. */
+  rlim = &limits[rlim_pos][reason];
+
+  log_fn_ratelim(rlim, LOG_INFO, LD_REND,
+                 "Service %s can't upload its %s descriptor: %s",
+                 safe_str_client(service->onion_address),
+                 (is_next_desc) ? "next" : "current", msg);
+}
+
+/** Return 1 if the given descriptor from the given service can be uploaded
  * else return 0 if it can not. */
 static int
 should_service_upload_descriptor(const hs_service_t *service,
                               const hs_service_descriptor_t *desc, time_t now)
 {
-  unsigned int num_intro_points;
+  char *msg = NULL;
+  unsigned int num_intro_points, count_ip_established;
 
   tor_assert(service);
   tor_assert(desc);
@@ -3087,39 +3163,59 @@ should_service_upload_descriptor(const hs_service_t *service,
    * upload descriptor in this case. We need at least one for the service to
    * be reachable. */
   if (desc->missing_intro_points && num_intro_points == 0) {
+    msg = tor_strdup("Missing intro points");
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_MISSING_IPS);
     goto cannot;
   }
 
   /* Check if all our introduction circuit have been established for all the
    * intro points we have selected. */
-  if (count_desc_circuit_established(desc) != num_intro_points) {
+  count_ip_established = count_desc_circuit_established(desc);
+  if (count_ip_established != num_intro_points) {
+    tor_asprintf(&msg, "Intro circuits aren't yet all established (%d/%d).",
+                 count_ip_established, num_intro_points);
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_IP_NOT_ESTABLISHED);
     goto cannot;
   }
 
   /* Is it the right time to upload? */
   if (desc->next_upload_time > now) {
+    tor_asprintf(&msg, "Next upload time is %ld, it is now %ld.",
+                 (long int) desc->next_upload_time, (long int) now);
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_NOT_TIME);
     goto cannot;
   }
 
   /* Don't upload desc if we don't have a live consensus */
   if (!networkstatus_get_reasonably_live_consensus(now,
                                             usable_consensus_flavor())) {
+    msg = tor_strdup("No reasonably live consensus");
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_NO_LIVE_CONSENSUS);
     goto cannot;
   }
 
   /* Do we know enough router descriptors to have adequate vision of the HSDir
      hash ring? */
   if (!router_have_minimum_dir_info()) {
+    msg = tor_strdup("Not enough directory information");
+    log_cant_upload_desc(service, desc, msg,
+                         LOG_DESC_UPLOAD_REASON_NO_DIRINFO);
     goto cannot;
   }
 
   /* Can upload! */
   return 1;
+
  cannot:
+  tor_free(msg);
   return 0;
 }
 
-/* Refresh the given service descriptor meaning this will update every mutable
+/** Refresh the given service descriptor meaning this will update every mutable
  * field that needs to be updated before we upload.
  *
  * This should ONLY be called before uploading a descriptor. It assumes that
@@ -3150,7 +3246,7 @@ refresh_service_descriptor(const hs_service_t *service,
   set_descriptor_revision_counter(desc, now, service->desc_current == desc);
 }
 
-/* Scheduled event run from the main loop. Try to upload the descriptor for
+/** Scheduled event run from the main loop. Try to upload the descriptor for
  * each service. */
 STATIC void
 run_upload_descriptor_event(time_t now)
@@ -3199,7 +3295,7 @@ run_upload_descriptor_event(time_t now)
   consider_republishing_hs_descriptors = 0;
 }
 
-/* Called when the introduction point circuit is done building and ready to be
+/** Called when the introduction point circuit is done building and ready to be
  * used. */
 static void
 service_intro_circ_has_opened(origin_circuit_t *circ)
@@ -3257,7 +3353,7 @@ service_intro_circ_has_opened(origin_circuit_t *circ)
   return;
 }
 
-/* Called when a rendezvous circuit is done building and ready to be used. */
+/** Called when a rendezvous circuit is done building and ready to be used. */
 static void
 service_rendezvous_circ_has_opened(origin_circuit_t *circ)
 {
@@ -3298,7 +3394,7 @@ service_rendezvous_circ_has_opened(origin_circuit_t *circ)
   return;
 }
 
-/* We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
+/** We've been expecting an INTRO_ESTABLISHED cell on this circuit and it just
  * arrived. Handle the INTRO_ESTABLISHED cell arriving on the given
  * introduction circuit. Return 0 on success else a negative value. */
 static int
@@ -3341,11 +3437,6 @@ service_handle_intro_established(origin_circuit_t *circ,
     goto err;
   }
 
-  /* Flag that we have an established circuit for this intro point. This value
-   * is what indicates the upload scheduled event if we are ready to build the
-   * intro point into the descriptor and upload. */
-  ip->circuit_established = 1;
-
   log_info(LD_REND, "Successfully received an INTRO_ESTABLISHED cell "
                     "on circuit %u for service %s",
            TO_CIRCUIT(circ)->n_circ_id,
@@ -3356,7 +3447,7 @@ service_handle_intro_established(origin_circuit_t *circ,
   return -1;
 }
 
-/* We just received an INTRODUCE2 cell on the established introduction circuit
+/** We just received an INTRODUCE2 cell on the established introduction circuit
  * circ. Handle the cell and return 0 on success else a negative value. */
 static int
 service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
@@ -3394,7 +3485,7 @@ service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
 
   /* The following will parse, decode and launch the rendezvous point circuit.
    * Both current and legacy cells are handled. */
-  if (hs_circ_handle_introduce2(service, circ, ip, desc->desc->subcredential,
+  if (hs_circ_handle_introduce2(service, circ, ip, &desc->desc->subcredential,
                                 payload, payload_len) < 0) {
     goto err;
   }
@@ -3404,7 +3495,7 @@ service_handle_introduce2(origin_circuit_t *circ, const uint8_t *payload,
   return -1;
 }
 
-/* Add to list every filename used by service. This is used by the sandbox
+/** Add to list every filename used by service. This is used by the sandbox
  * subsystem. */
 static void
 service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
@@ -3426,7 +3517,7 @@ service_add_fnames_to_list(const hs_service_t *service, smartlist_t *list)
   smartlist_add(list, hs_path_from_filename(s_dir, fname));
 }
 
-/* Return true iff the given service identity key is present on disk. */
+/** Return true iff the given service identity key is present on disk. */
 static int
 service_key_on_disk(const char *directory_path)
 {
@@ -3450,7 +3541,7 @@ service_key_on_disk(const char *directory_path)
   return ret;
 }
 
-/* This is a proxy function before actually calling hs_desc_encode_descriptor
+/** This is a proxy function before actually calling hs_desc_encode_descriptor
  * because we need some preprocessing here */
 static int
 service_encode_descriptor(const hs_service_t *service,
@@ -3481,7 +3572,7 @@ service_encode_descriptor(const hs_service_t *service,
 /* Public API */
 /* ========== */
 
-/* This is called everytime the service map (v2 or v3) changes that is if an
+/** This is called everytime the service map (v2 or v3) changes that is if an
  * element is added or removed. */
 void
 hs_service_map_has_changed(void)
@@ -3492,7 +3583,7 @@ hs_service_map_has_changed(void)
   rescan_periodic_events(get_options());
 }
 
-/* Upload an encoded descriptor in encoded_desc of the given version. This
+/** Upload an encoded descriptor in encoded_desc of the given version. This
  * descriptor is for the service identity_pk and blinded_pk used to setup the
  * directory connection identifier. It is uploaded to the directory hsdir_rs
  * routerstatus_t object.
@@ -3540,7 +3631,7 @@ hs_service_upload_desc_to_dir(const char *encoded_desc,
   directory_request_free(dir_req);
 }
 
-/* Add the ephemeral service using the secret key sk and ports. Both max
+/** Add the ephemeral service using the secret key sk and ports. Both max
  * streams parameter will be set in the newly created service.
  *
  * Ownership of sk and ports is passed to this routine.  Regardless of
@@ -3626,7 +3717,7 @@ hs_service_add_ephemeral(ed25519_secret_key_t *sk, smartlist_t *ports,
   return ret;
 }
 
-/* For the given onion address, delete the ephemeral service. Return 0 on
+/** For the given onion address, delete the ephemeral service. Return 0 on
  * success else -1 on error. */
 int
 hs_service_del_ephemeral(const char *address)
@@ -3676,7 +3767,7 @@ hs_service_del_ephemeral(const char *address)
   return -1;
 }
 
-/* Using the ed25519 public key pk, find a service for that key and return the
+/** Using the ed25519 public key pk, find a service for that key and return the
  * current encoded descriptor as a newly allocated string or NULL if not
  * found. This is used by the control port subsystem. */
 char *
@@ -3702,9 +3793,9 @@ hs_service_lookup_current_desc(const ed25519_public_key_t *pk)
   return NULL;
 }
 
-/* Return the number of service we have configured and usable. */
-unsigned int
-hs_service_get_num_services(void)
+/** Return the number of service we have configured and usable. */
+MOCK_IMPL(unsigned int,
+hs_service_get_num_services,(void))
 {
   if (hs_service_map == NULL) {
     return 0;
@@ -3712,49 +3803,7 @@ hs_service_get_num_services(void)
   return HT_SIZE(hs_service_map);
 }
 
-/* Called once an introduction circuit is closed. If the circuit doesn't have
- * a v3 identifier, it is ignored. */
-void
-hs_service_intro_circ_has_closed(origin_circuit_t *circ)
-{
-  hs_service_t *service = NULL;
-  hs_service_intro_point_t *ip = NULL;
-  hs_service_descriptor_t *desc = NULL;
-
-  tor_assert(circ);
-
-  if (circ->hs_ident == NULL) {
-    /* This is not a v3 circuit, ignore. */
-    goto end;
-  }
-
-  get_objects_from_ident(circ->hs_ident, &service, &ip, &desc);
-  if (service == NULL) {
-    /* This is possible if the circuits are closed and the service is
-     * immediately deleted. */
-    log_info(LD_REND, "Unable to find any hidden service associated "
-                      "identity key %s on intro circuit %u.",
-             ed25519_fmt(&circ->hs_ident->identity_pk),
-             TO_CIRCUIT(circ)->n_circ_id);
-    goto end;
-  }
-  if (ip == NULL) {
-    /* The introduction point object has already been removed probably by our
-     * cleanup process so ignore. */
-    goto end;
-  }
-  /* Can't have an intro point object without a descriptor. */
-  tor_assert(desc);
-
-  /* Circuit disappeared so make sure the intro point is updated. By
-   * keeping the object in the descriptor, we'll be able to retry. */
-  ip->circuit_established = 0;
-
- end:
-  return;
-}
-
-/* Given conn, a rendezvous edge connection acting as an exit stream, look up
+/** Given conn, a rendezvous edge connection acting as an exit stream, look up
  * the hidden service for the circuit circ, and look up the port and address
  * based on the connection port. Assign the actual connection address.
  *
@@ -3852,7 +3901,7 @@ hs_service_exports_circuit_id(const ed25519_public_key_t *pk)
   return service->config.circuit_id_protocol;
 }
 
-/* Add to file_list every filename used by a configured hidden service, and to
+/** Add to file_list every filename used by a configured hidden service, and to
  * dir_list every directory path used by a configured hidden service. This is
  * used by the sandbox subsystem to whitelist those. */
 void
@@ -3877,7 +3926,7 @@ hs_service_lists_fnames_for_sandbox(smartlist_t *file_list,
   } FOR_EACH_DESCRIPTOR_END;
 }
 
-/* Called when our internal view of the directory has changed. We might have
+/** Called when our internal view of the directory has changed. We might have
  * received a new batch of descriptors which might affect the shape of the
  * HSDir hash ring. Signal that we should reexamine the hash ring and
  * re-upload our HS descriptors if needed. */
@@ -3894,7 +3943,7 @@ hs_service_dir_info_changed(void)
   }
 }
 
-/* Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
+/** Called when we get an INTRODUCE2 cell on the circ. Respond to the cell and
  * launch a circuit to the rendezvous point. */
 int
 hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
@@ -3925,7 +3974,7 @@ hs_service_receive_introduce2(origin_circuit_t *circ, const uint8_t *payload,
   return ret;
 }
 
-/* Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
+/** Called when we get an INTRO_ESTABLISHED cell. Mark the circuit as an
  * established introduction point. Return 0 on success else a negative value
  * and the circuit is closed. */
 int
@@ -3962,7 +4011,7 @@ hs_service_receive_intro_established(origin_circuit_t *circ,
   return -1;
 }
 
-/* Called when any kind of hidden service circuit is done building thus
+/** Called when any kind of hidden service circuit is done building thus
  * opened. This is the entry point from the circuit subsystem. */
 void
 hs_service_circuit_has_opened(origin_circuit_t *circ)
@@ -3991,7 +4040,7 @@ hs_service_circuit_has_opened(origin_circuit_t *circ)
   }
 }
 
-/* Return the service version by looking at the key in the service directory.
+/** Return the service version by looking at the key in the service directory.
  * If the key is not found or unrecognized, -1 is returned. Else, the service
  * version is returned. */
 int
@@ -4021,7 +4070,7 @@ hs_service_get_version_from_key(const hs_service_t *service)
   return version;
 }
 
-/* Load and/or generate keys for all onion services including the client
+/** Load and/or generate keys for all onion services including the client
  * authorization if any. Return 0 on success, -1 on failure. */
 int
 hs_service_load_all_keys(void)
@@ -4057,7 +4106,51 @@ hs_service_load_all_keys(void)
   return -1;
 }
 
-/* Put all service object in the given service list. After this, the caller
+/** Log the status of introduction points for all version 3 onion services
+ * at log severity <b>severity</b>.
+ */
+void
+hs_service_dump_stats(int severity)
+{
+  origin_circuit_t *circ;
+
+  FOR_EACH_SERVICE_BEGIN(hs) {
+
+    tor_log(severity, LD_GENERAL, "Service configured in %s:",
+            service_escaped_dir(hs));
+    FOR_EACH_DESCRIPTOR_BEGIN(hs, desc) {
+
+      DIGEST256MAP_FOREACH(desc->intro_points.map, key,
+                           hs_service_intro_point_t *, ip) {
+        const node_t *intro_node;
+        const char *nickname;
+
+        intro_node = get_node_from_intro_point(ip);
+        if (!intro_node) {
+          tor_log(severity, LD_GENERAL, "  Couldn't find intro point, "
+                  "skipping");
+          continue;
+        }
+        nickname = node_get_nickname(intro_node);
+        if (!nickname) {
+          continue;
+        }
+
+        circ = hs_circ_service_get_intro_circ(ip);
+        if (!circ) {
+          tor_log(severity, LD_GENERAL, "  Intro point at %s: no circuit",
+                  nickname);
+          continue;
+        }
+        tor_log(severity, LD_GENERAL, "  Intro point %s: circuit is %s",
+                nickname, circuit_state_to_string(circ->base_.state));
+      } DIGEST256MAP_FOREACH_END;
+
+    } FOR_EACH_DESCRIPTOR_END;
+  } FOR_EACH_SERVICE_END;
+}
+
+/** Put all service object in the given service list. After this, the caller
  * looses ownership of every elements in the list and responsible to free the
  * list pointer. */
 void
@@ -4074,7 +4167,7 @@ hs_service_stage_services(const smartlist_t *service_list)
   smartlist_add_all(hs_service_staging_list, service_list);
 }
 
-/* Allocate and initilize a service object. The service configuration will
+/** Allocate and initilize a service object. The service configuration will
  * contain the default values. Return the newly allocated object pointer. This
  * function can't fail. */
 hs_service_t *
@@ -4092,7 +4185,7 @@ hs_service_new(const or_options_t *options)
   return service;
 }
 
-/* Free the given <b>service</b> object and all its content. This function
+/** Free the given <b>service</b> object and all its content. This function
  * also takes care of wiping service keys from memory. It is safe to pass a
  * NULL pointer. */
 void
@@ -4115,13 +4208,18 @@ hs_service_free_(hs_service_t *service)
     replaycache_free(service->state.replay_cache_rend_cookie);
   }
 
+  /* Free onionbalance subcredentials (if any) */
+  if (service->state.ob_subcreds) {
+    tor_free(service->state.ob_subcreds);
+  }
+
   /* Wipe service keys. */
   memwipe(&service->keys.identity_sk, 0, sizeof(service->keys.identity_sk));
 
   tor_free(service);
 }
 
-/* Periodic callback. Entry point from the main loop to the HS service
+/** Periodic callback. Entry point from the main loop to the HS service
  * subsystem. This is call every second. This is skipped if tor can't build a
  * circuit or the network is disabled. */
 void
@@ -4144,7 +4242,7 @@ hs_service_run_scheduled_events(time_t now)
   run_upload_descriptor_event(now);
 }
 
-/* Initialize the service HS subsystem. */
+/** Initialize the service HS subsystem. */
 void
 hs_service_init(void)
 {
@@ -4161,24 +4259,25 @@ hs_service_init(void)
   hs_service_staging_list = smartlist_new();
 }
 
-/* Release all global storage of the hidden service subsystem. */
+/** Release all global storage of the hidden service subsystem. */
 void
 hs_service_free_all(void)
 {
   rend_service_free_all();
   service_free_all();
+  hs_config_free_all();
 }
 
 #ifdef TOR_UNIT_TESTS
 
-/* Return the global service map size. Only used by unit test. */
+/** Return the global service map size. Only used by unit test. */
 STATIC unsigned int
 get_hs_service_map_size(void)
 {
   return HT_SIZE(hs_service_map);
 }
 
-/* Return the staging list size. Only used by unit test. */
+/** Return the staging list size. Only used by unit test. */
 STATIC int
 get_hs_service_staging_list_size(void)
 {
diff --git a/src/feature/hs/hs_service.h b/src/feature/hs/hs_service.h
index 5f43233ea1..b5bff5bee5 100644
--- a/src/feature/hs/hs_service.h
+++ b/src/feature/hs/hs_service.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,88 +21,92 @@
 /* Trunnel */
 #include "trunnel/hs/cell_establish_intro.h"
 
-/* When loading and configuring a service, this is the default version it will
+#include "ext/ht.h"
+
+/** When loading and configuring a service, this is the default version it will
  * be configured for as it is possible that no HiddenServiceVersion is
  * present. */
 #define HS_SERVICE_DEFAULT_VERSION HS_VERSION_THREE
 
-/* As described in the specification, service publishes their next descriptor
+/** As described in the specification, service publishes their next descriptor
  * at a random time between those two values (in seconds). */
 #define HS_SERVICE_NEXT_UPLOAD_TIME_MIN (60 * 60)
+/** Maximum interval for uploading next descriptor (in seconds). */
 #define HS_SERVICE_NEXT_UPLOAD_TIME_MAX (120 * 60)
 
-/* Service side introduction point. */
+/** Service side introduction point. */
 typedef struct hs_service_intro_point_t {
-  /* Top level intropoint "shared" data between client/service. */
+  /** Top level intropoint "shared" data between client/service. */
   hs_intropoint_t base;
 
-  /* Onion key of the introduction point used to extend to it for the ntor
+  /** Onion key of the introduction point used to extend to it for the ntor
    * handshake. */
   curve25519_public_key_t onion_key;
 
-  /* Authentication keypair used to create the authentication certificate
+  /** Authentication keypair used to create the authentication certificate
    * which is published in the descriptor. */
   ed25519_keypair_t auth_key_kp;
 
-  /* Encryption keypair for the "ntor" type. */
+  /** Encryption keypair for the "ntor" type. */
   curve25519_keypair_t enc_key_kp;
 
-  /* Legacy key if that intro point doesn't support v3. This should be used if
+  /** Legacy key if that intro point doesn't support v3. This should be used if
    * the base object legacy flag is set. */
   crypto_pk_t *legacy_key;
-  /* Legacy key SHA1 public key digest. This should be used only if the base
+  /** Legacy key SHA1 public key digest. This should be used only if the base
    * object legacy flag is set. */
   uint8_t legacy_key_digest[DIGEST_LEN];
 
-  /* Amount of INTRODUCE2 cell accepted from this intro point. */
+  /** Amount of INTRODUCE2 cell accepted from this intro point. */
   uint64_t introduce2_count;
 
-  /* Maximum number of INTRODUCE2 cell this intro point should accept. */
+  /** Maximum number of INTRODUCE2 cell this intro point should accept. */
   uint64_t introduce2_max;
 
-  /* The time at which this intro point should expire and stop being used. */
+  /** The time at which this intro point should expire and stop being used. */
   time_t time_to_expire;
 
-  /* The amount of circuit creation we've made to this intro point. This is
+  /** The amount of circuit creation we've made to this intro point. This is
    * incremented every time we do a circuit relaunch on this intro point which
    * is triggered when the circuit dies but the node is still in the
    * consensus. After MAX_INTRO_POINT_CIRCUIT_RETRIES, we give up on it. */
   uint32_t circuit_retries;
 
-  /* Set if this intro point has an established circuit. */
-  unsigned int circuit_established : 1;
-
-  /* Replay cache recording the encrypted part of an INTRODUCE2 cell that the
+  /** Replay cache recording the encrypted part of an INTRODUCE2 cell that the
    * circuit associated with this intro point has received. This is used to
    * prevent replay attacks. */
   replaycache_t *replay_cache;
+
+  /** Support the INTRO2 DoS defense. If set, the DoS extension described by
+   * proposal 305 is sent. */
+  unsigned int support_intro2_dos_defense : 1;
 } hs_service_intro_point_t;
 
-/* Object handling introduction points of a service. */
+/** Object handling introduction points of a service. */
 typedef struct hs_service_intropoints_t {
-  /* The time at which we've started our retry period to build circuits. We
+  /** The time at which we've started our retry period to build circuits. We
    * don't want to stress circuit creation so we can only retry for a certain
    * time and then after we stop and wait. */
   time_t retry_period_started;
 
-  /* Number of circuit we've launched during a single retry period. */
+  /** Number of circuit we've launched during a single retry period. */
   unsigned int num_circuits_launched;
 
-  /* Contains the current hs_service_intro_point_t objects indexed by
+  /** Contains the current hs_service_intro_point_t objects indexed by
    * authentication public key. */
   digest256map_t *map;
 
-  /* Contains node's identity key digest that were introduction point for this
+  /** Contains node's identity key digest that were introduction point for this
    * descriptor but were retried to many times. We keep those so we avoid
    * re-picking them over and over for a circuit retry period.
    * XXX: Once we have #22173, change this to only use ed25519 identity. */
   digestmap_t *failed_id;
 } hs_service_intropoints_t;
 
-/* Representation of a service descriptor.
+/** Representation of a service descriptor.
  *
  * Some elements of the descriptor are mutable whereas others are immutable:
-
+ *
  * Immutable elements are initialized once when the descriptor is built (when
  * service descriptors gets rotated). This means that these elements are
  * initialized once and then they don't change for the lifetime of the
@@ -117,40 +121,42 @@ typedef struct hs_service_intropoints_t {
  * update_service_descriptor_intro_points().
  */
 typedef struct hs_service_descriptor_t {
-  /* Immutable: Client authorization ephemeral keypair. */
+  /** Immutable: Client authorization ephemeral keypair. */
   curve25519_keypair_t auth_ephemeral_kp;
 
-  /* Immutable: Descriptor cookie used to encrypt the descriptor, when the
+  /** Immutable: Descriptor cookie used to encrypt the descriptor, when the
    * client authorization is enabled */
   uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
 
-  /* Immutable: Descriptor signing keypair. */
+  /** Immutable: Descriptor signing keypair. */
   ed25519_keypair_t signing_kp;
 
-  /* Immutable: Blinded keypair derived from the master identity public key. */
+  /** Immutable: Blinded keypair derived from the master identity public
+   * key. */
   ed25519_keypair_t blinded_kp;
 
-  /* Immutable: The time period number this descriptor has been created for. */
+  /** Immutable: The time period number this descriptor has been created
+   * for. */
   uint64_t time_period_num;
 
   /** Immutable: The OPE cipher for encrypting revision counters for this
    *  descriptor.  Tied to the descriptor blinded key. */
   struct crypto_ope_t *ope_cipher;
 
-  /* Mutable: Decoded descriptor. This object is used for encoding when the
+  /** Mutable: Decoded descriptor. This object is used for encoding when the
    * service publishes the descriptor. */
   hs_descriptor_t *desc;
 
-  /* Mutable: When is the next time when we should upload the descriptor. */
+  /** Mutable: When is the next time when we should upload the descriptor. */
   time_t next_upload_time;
 
-  /* Mutable: Introduction points assign to this descriptor which contains
+  /** Mutable: Introduction points assign to this descriptor which contains
    * hs_service_intropoints_t object indexed by authentication key (the RSA key
    * if the node is legacy). */
   hs_service_intropoints_t intro_points;
 
-  /* Mutable: True iff we have missing intro points for this descriptor because
-   * we couldn't pick any nodes. */
+  /** Mutable: True iff we have missing intro points for this descriptor
+   * because we couldn't pick any nodes. */
   unsigned int missing_intro_points : 1;
 
   /** Mutable: List of the responsible HSDirs (their b64ed identity digest)
@@ -160,20 +166,20 @@ typedef struct hs_service_descriptor_t {
   smartlist_t *previous_hsdirs;
 } hs_service_descriptor_t;
 
-/* Service key material. */
+/** Service key material. */
 typedef struct hs_service_keys_t {
-  /* Master identify public key. */
+  /** Master identify public key. */
   ed25519_public_key_t identity_pk;
-  /* Master identity private key. */
+  /** Master identity private key. */
   ed25519_secret_key_t identity_sk;
-  /* True iff the key is kept offline which means the identity_sk MUST not be
+  /** True iff the key is kept offline which means the identity_sk MUST not be
    * used in that case. */
   unsigned int is_identify_key_offline : 1;
 } hs_service_keys_t;
 
 /** Service side configuration of client authorization. */
 typedef struct hs_service_authorized_client_t {
-  /* The client auth public key used to encrypt the descriptor cookie. */
+  /** The client auth public key used to encrypt the descriptor cookie. */
   curve25519_public_key_t client_pk;
 } hs_service_authorized_client_t;
 
@@ -186,115 +192,129 @@ typedef enum {
   HS_CIRCUIT_ID_PROTOCOL_HAPROXY
 } hs_circuit_id_protocol_t;
 
-/* Service configuration. The following are set from the torrc options either
+/** Service configuration. The following are set from the torrc options either
  * set by the configuration file or by the control port. Nothing else should
  * change those values. */
 typedef struct hs_service_config_t {
-  /* Protocol version of the service. Specified by HiddenServiceVersion
+  /** Protocol version of the service. Specified by HiddenServiceVersion
    * option. */
   uint32_t version;
 
-  /* Have we explicitly set HiddenServiceVersion? */
+  /** Have we explicitly set HiddenServiceVersion? */
   unsigned int hs_version_explicitly_set : 1;
 
-  /* List of rend_service_port_config_t */
+  /** List of rend_service_port_config_t */
   smartlist_t *ports;
 
-  /* Path on the filesystem where the service persistent data is stored. NULL
+  /** Path on the filesystem where the service persistent data is stored. NULL
    * if the service is ephemeral. Specified by HiddenServiceDir option. */
   char *directory_path;
 
-  /* The maximum number of simultaneous streams per rendezvous circuit that
+  /** The maximum number of simultaneous streams per rendezvous circuit that
    * are allowed to be created. No limit if 0. Specified by
    * HiddenServiceMaxStreams option. */
   uint64_t max_streams_per_rdv_circuit;
 
-  /* If true, we close circuits that exceed the max_streams_per_rdv_circuit
+  /** If true, we close circuits that exceed the max_streams_per_rdv_circuit
    * limit. Specified by HiddenServiceMaxStreamsCloseCircuit option. */
   unsigned int max_streams_close_circuit : 1;
 
-  /* How many introduction points this service has. Specified by
+  /** How many introduction points this service has. Specified by
    * HiddenServiceNumIntroductionPoints option. */
   unsigned int num_intro_points;
 
-  /* True iff the client auth is enabled. */
+  /** True iff the client auth is enabled. */
   unsigned int is_client_auth_enabled : 1;
 
-  /* List of hs_service_authorized_client_t's of clients that may access this
+  /** List of hs_service_authorized_client_t's of clients that may access this
    * service. Specified by HiddenServiceAuthorizeClient option. */
   smartlist_t *clients;
 
-  /* True iff we allow request made on unknown ports. Specified by
+  /** True iff we allow request made on unknown ports. Specified by
    * HiddenServiceAllowUnknownPorts option. */
   unsigned int allow_unknown_ports : 1;
 
-  /* If true, this service is a Single Onion Service. Specified by
+  /** If true, this service is a Single Onion Service. Specified by
    * HiddenServiceSingleHopMode and HiddenServiceNonAnonymousMode options. */
   unsigned int is_single_onion : 1;
 
-  /* If true, allow group read permissions on the directory_path. Specified by
+  /** If true, allow group read permissions on the directory_path. Specified by
    * HiddenServiceDirGroupReadable option. */
   unsigned int dir_group_readable : 1;
 
-  /* Is this service ephemeral? */
+  /** Is this service ephemeral? */
   unsigned int is_ephemeral : 1;
 
-  /* Does this service export the circuit ID of its clients? */
+  /** Does this service export the circuit ID of its clients? */
   hs_circuit_id_protocol_t circuit_id_protocol;
+
+  /** DoS defenses. For the ESTABLISH_INTRO cell extension. */
+  unsigned int has_dos_defense_enabled : 1;
+  uint32_t intro_dos_rate_per_sec;
+  uint32_t intro_dos_burst_per_sec;
+
+  /** If set, contains the Onion Balance master ed25519 public key (taken from
+   * an .onion addresses) that this tor instance serves as backend. */
+  smartlist_t *ob_master_pubkeys;
 } hs_service_config_t;
 
-/* Service state. */
+/** Service state. */
 typedef struct hs_service_state_t {
-  /* The time at which we've started our retry period to build circuits. We
+  /** The time at which we've started our retry period to build circuits. We
    * don't want to stress circuit creation so we can only retry for a certain
    * time and then after we stop and wait. */
   time_t intro_circ_retry_started_time;
 
-  /* Number of circuit we've launched during a single retry period. This
+  /** Number of circuit we've launched during a single retry period. This
    * should never go over MAX_INTRO_CIRCS_PER_PERIOD. */
   unsigned int num_intro_circ_launched;
 
-  /* Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
+  /** Replay cache tracking the REND_COOKIE found in INTRODUCE2 cell to detect
    * repeats. Clients may send INTRODUCE1 cells for the same rendezvous point
    * through two or more different introduction points; when they do, this
    * keeps us from launching multiple simultaneous attempts to connect to the
    * same rend point. */
   replaycache_t *replay_cache_rend_cookie;
 
-  /* When is the next time we should rotate our descriptors. This is has to be
+  /** When is the next time we should rotate our descriptors. This is has to be
    * done at the start time of the next SRV protocol run. */
   time_t next_rotation_time;
+
+  /* If this is an onionbalance instance, this is an array of subcredentials
+   * that should be used when decrypting an INTRO2 cell. If this is not an
+   * onionbalance instance, this is NULL.
+   * See [ONIONBALANCE] section in rend-spec-v3.txt for more details . */
+  hs_subcredential_t *ob_subcreds;
+  /* Number of OB subcredentials */
+  size_t n_ob_subcreds;
 } hs_service_state_t;
 
-/* Representation of a service running on this tor instance. */
+/** Representation of a service running on this tor instance. */
 typedef struct hs_service_t {
-  /* Onion address base32 encoded and NUL terminated. We keep it for logging
+  /** Onion address base32 encoded and NUL terminated. We keep it for logging
    * purposes so we don't have to build it everytime. */
   char onion_address[HS_SERVICE_ADDR_LEN_BASE32 + 1];
 
-  /* Hashtable node: use to look up the service by its master public identity
+  /** Hashtable node: use to look up the service by its master public identity
    * key in the service global map. */
   HT_ENTRY(hs_service_t) hs_service_node;
 
-  /* Service state which contains various flags and counters. */
+  /** Service state which contains various flags and counters. */
   hs_service_state_t state;
 
-  /* Key material of the service. */
+  /** Key material of the service. */
   hs_service_keys_t keys;
 
-  /* Configuration of the service. */
+  /** Configuration of the service. */
   hs_service_config_t config;
 
-  /* Current descriptor. */
+  /** Current descriptor. */
   hs_service_descriptor_t *desc_current;
-  /* Next descriptor. */
+  /** Next descriptor. */
   hs_service_descriptor_t *desc_next;
-
-  /* XXX: Credential (client auth.) #20700. */
-
 } hs_service_t;
 
-/* For the service global hash map, we define a specific type for it which
+/** For the service global hash map, we define a specific type for it which
  * will make it safe to use and specific to some controlled parameters such as
  * the hashing function and how to compare services. */
 typedef HT_HEAD(hs_service_ht, hs_service_t) hs_service_ht;
@@ -308,9 +328,14 @@ void hs_service_free_all(void);
 /* Service new/free functions. */
 hs_service_t *hs_service_new(const or_options_t *options);
 void hs_service_free_(hs_service_t *service);
+/**
+ * @copydoc hs_service_free_
+ *
+ * Additionally, set the pointer <b>s</b> to NULL.
+ **/
 #define hs_service_free(s) FREE_AND_NULL(hs_service_t, hs_service_free_, (s))
 
-unsigned int hs_service_get_num_services(void);
+MOCK_DECL(unsigned int, hs_service_get_num_services,(void));
 void hs_service_stage_services(const smartlist_t *service_list);
 int hs_service_load_all_keys(void);
 int hs_service_get_version_from_key(const hs_service_t *service);
@@ -330,8 +355,6 @@ int hs_service_receive_introduce2(origin_circuit_t *circ,
                                   const uint8_t *payload,
                                   size_t payload_len);
 
-void hs_service_intro_circ_has_closed(origin_circuit_t *circ);
-
 char *hs_service_lookup_current_desc(const ed25519_public_key_t *pk);
 
 hs_service_add_ephemeral_status_t
@@ -350,6 +373,8 @@ void hs_service_upload_desc_to_dir(const char *encoded_desc,
 hs_circuit_id_protocol_t
 hs_service_exports_circuit_id(const ed25519_public_key_t *pk);
 
+void hs_service_dump_stats(int severity);
+
 #ifdef HS_SERVICE_PRIVATE
 
 #ifdef TOR_UNIT_TESTS
@@ -361,7 +386,10 @@ STATIC hs_service_t *get_first_service(void);
 STATIC hs_service_intro_point_t *service_intro_point_find_by_ident(
                                          const hs_service_t *service,
                                          const hs_ident_circuit_t *ident);
-#endif
+
+MOCK_DECL(STATIC unsigned int, count_desc_circuit_established,
+          (const hs_service_descriptor_t *desc));
+#endif /* defined(TOR_UNIT_TESTS) */
 
 /* Service accessors. */
 STATIC hs_service_t *find_service(hs_service_ht *map,
@@ -369,10 +397,7 @@ STATIC hs_service_t *find_service(hs_service_ht *map,
 STATIC void remove_service(hs_service_ht *map, hs_service_t *service);
 STATIC int register_service(hs_service_ht *map, hs_service_t *service);
 /* Service introduction point functions. */
-STATIC hs_service_intro_point_t *service_intro_point_new(
-                            const extend_info_t *ei,
-                            unsigned int is_legacy,
-                            unsigned int supports_ed25519_link_handshake_any);
+STATIC hs_service_intro_point_t *service_intro_point_new(const node_t *node);
 STATIC void service_intro_point_free_(hs_service_intro_point_t *ip);
 #define service_intro_point_free(ip)                            \
   FREE_AND_NULL(hs_service_intro_point_t,             \
diff --git a/src/feature/hs/hs_stats.c b/src/feature/hs/hs_stats.c
index f24b731328..f9d458d630 100644
--- a/src/feature/hs/hs_stats.c
+++ b/src/feature/hs/hs_stats.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/hs/hs_stats.h b/src/feature/hs/hs_stats.h
index d89440faca..aea2ccf5c2 100644
--- a/src/feature/hs/hs_stats.h
+++ b/src/feature/hs/hs_stats.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,9 +6,13 @@
  * \brief Header file for hs_stats.c
  **/
 
+#ifndef TOR_HS_STATS_H
+#define TOR_HS_STATS_H
+
 void hs_stats_note_introduce2_cell(int is_hsv3);
 uint32_t hs_stats_get_n_introduce2_v3_cells(void);
 uint32_t hs_stats_get_n_introduce2_v2_cells(void);
 void hs_stats_note_service_rendezvous_launch(void);
 uint32_t hs_stats_get_n_rendezvous_launches(void);
 
+#endif /* !defined(TOR_HS_STATS_H) */
diff --git a/src/feature/hs/hsdir_index_st.h b/src/feature/hs/hsdir_index_st.h
index 7d4116d8bb..6ce0bf5c69 100644
--- a/src/feature/hs/hsdir_index_st.h
+++ b/src/feature/hs/hsdir_index_st.h
@@ -1,24 +1,29 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file hsdir_index_st.h
+ * @brief HS directory index structure
+ **/
+
 #ifndef HSDIR_INDEX_ST_H
 #define HSDIR_INDEX_ST_H
 
-/* Hidden service directory index used in a node_t which is set once we set
+/** Hidden service directory index used in a node_t which is set once we set
  * the consensus. */
 struct hsdir_index_t {
-  /* HSDir index to use when fetching a descriptor. */
+  /** HSDir index to use when fetching a descriptor. */
   uint8_t fetch[DIGEST256_LEN];
 
-  /* HSDir index used by services to store their first and second
+  /** HSDir index used by services to store their first and second
    * descriptor. The first descriptor is chronologically older than the second
    * one and uses older TP and SRV values. */
   uint8_t store_first[DIGEST256_LEN];
+  /** Newer index, for second descriptor. */
   uint8_t store_second[DIGEST256_LEN];
 };
 
-#endif
-
+#endif /* !defined(HSDIR_INDEX_ST_H) */
diff --git a/src/feature/hs/include.am b/src/feature/hs/include.am
new file mode 100644
index 0000000000..af1dc65585
--- /dev/null
+++ b/src/feature/hs/include.am
@@ -0,0 +1,39 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/hs/hs_cache.c		\
+	src/feature/hs/hs_cell.c		\
+	src/feature/hs/hs_circuit.c		\
+	src/feature/hs/hs_circuitmap.c		\
+	src/feature/hs/hs_client.c		\
+	src/feature/hs/hs_common.c		\
+	src/feature/hs/hs_config.c		\
+	src/feature/hs/hs_control.c		\
+	src/feature/hs/hs_descriptor.c		\
+	src/feature/hs/hs_dos.c			\
+	src/feature/hs/hs_ident.c		\
+	src/feature/hs/hs_intropoint.c		\
+	src/feature/hs/hs_ob.c			\
+	src/feature/hs/hs_service.c		\
+	src/feature/hs/hs_stats.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/hs/hs_cache.h			\
+	src/feature/hs/hs_cell.h			\
+	src/feature/hs/hs_circuit.h			\
+	src/feature/hs/hs_circuitmap.h			\
+	src/feature/hs/hs_client.h			\
+	src/feature/hs/hs_common.h			\
+	src/feature/hs/hs_config.h			\
+	src/feature/hs/hs_control.h			\
+	src/feature/hs/hs_descriptor.h			\
+	src/feature/hs/hs_dos.h				\
+	src/feature/hs/hs_ident.h			\
+	src/feature/hs/hs_intropoint.h			\
+	src/feature/hs/hs_ob.h				\
+	src/feature/hs/hs_opts_st.h			\
+	src/feature/hs/hs_options.inc			\
+	src/feature/hs/hs_service.h			\
+	src/feature/hs/hs_stats.h			\
+	src/feature/hs/hsdir_index_st.h
diff --git a/src/feature/hs_common/.may_include b/src/feature/hs_common/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/hs_common/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/hs_common/feature_hs_common.md b/src/feature/hs_common/feature_hs_common.md
new file mode 100644
index 0000000000..3a5e351a0a
--- /dev/null
+++ b/src/feature/hs_common/feature_hs_common.md
@@ -0,0 +1,3 @@
+@dir /feature/hs_common
+@brief feature/hs_common: Common to v2 (old) and v3 (current) onion services
+
diff --git a/src/feature/hs_common/include.am b/src/feature/hs_common/include.am
new file mode 100644
index 0000000000..3bb9225c12
--- /dev/null
+++ b/src/feature/hs_common/include.am
@@ -0,0 +1,10 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/hs_common/replaycache.c	        \
+	src/feature/hs_common/shared_random_client.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/hs_common/replaycache.h		\
+	src/feature/hs_common/shared_random_client.h
diff --git a/src/feature/hs_common/replaycache.c b/src/feature/hs_common/replaycache.c
index 9e8c13b1c5..ab058ce759 100644
--- a/src/feature/hs_common/replaycache.c
+++ b/src/feature/hs_common/replaycache.c
@@ -1,4 +1,4 @@
- /* Copyright (c) 2012-2019, The Tor Project, Inc. */
+ /* Copyright (c) 2012-2020, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/hs_common/replaycache.h b/src/feature/hs_common/replaycache.h
index 01f5e600c2..3a3eed29c0 100644
--- a/src/feature/hs_common/replaycache.h
+++ b/src/feature/hs_common/replaycache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,16 +14,16 @@ typedef struct replaycache_t replaycache_t;
 #ifdef REPLAYCACHE_PRIVATE
 
 struct replaycache_t {
-  /* Scrub interval */
+  /** Scrub interval */
   time_t scrub_interval;
-  /* Last scrubbed */
+  /** Last scrubbed */
   time_t scrubbed;
-  /*
+  /**
    * Horizon
    * (don't return true on digests in the cache but older than this)
    */
   time_t horizon;
-  /*
+  /**
    * Digest map: keys are digests, values are times the digest was last seen
    */
   digest256map_t *digests_seen;
@@ -34,6 +34,11 @@ struct replaycache_t {
 /* replaycache_t free/new */
 
 void replaycache_free_(replaycache_t *r);
+/**
+ * @copydoc replaycache_free_
+ *
+ * Additionally, set the pointer <b>r</b> to NULL.
+ **/
 #define replaycache_free(r) \
   FREE_AND_NULL(replaycache_t, replaycache_free_, (r))
 replaycache_t * replaycache_new(time_t horizon, time_t interval);
diff --git a/src/feature/hs_common/shared_random_client.c b/src/feature/hs_common/shared_random_client.c
index ead5d681a9..4e8a2942fc 100644
--- a/src/feature/hs_common/shared_random_client.c
+++ b/src/feature/hs_common/shared_random_client.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,18 +8,18 @@
  *        as part of the dirauth module.
  **/
 
-#define SHARED_RANDOM_CLIENT_PRIVATE
 #include "feature/hs_common/shared_random_client.h"
 
 #include "app/config/config.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/authmode.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "lib/encoding/binascii.h"
 
 #include "feature/nodelist/networkstatus_st.h"
 
-/* Convert a given srv object to a string for the control port. This doesn't
+/** Convert a given srv object to a string for the control port. This doesn't
  * fail and the srv object MUST be valid. */
 static char *
 srv_to_control_string(const sr_srv_t *srv)
@@ -33,7 +33,25 @@ srv_to_control_string(const sr_srv_t *srv)
   return srv_str;
 }
 
-/* Return the voting interval of the tor vote subsystem. */
+/**
+ * If we have no consensus and we are not an authority, assume that this is
+ * the voting interval.  We should never actually use this: only authorities
+ * should be trying to figure out the schedule when they don't have a
+ * consensus.
+ **/
+#define DEFAULT_NETWORK_VOTING_INTERVAL (3600)
+
+/* This is an unpleasing workaround for tests.  Our unit tests assume that we
+ * are scheduling all of our shared random stuff as if we were a directory
+ * authority, but they do not always set V3AuthoritativeDir.
+ */
+#ifdef TOR_UNIT_TESTS
+#define ASSUME_AUTHORITY_SCHEDULING 1
+#else
+#define ASSUME_AUTHORITY_SCHEDULING 0
+#endif
+
+/** Return the voting interval of the tor vote subsystem. */
 int
 get_voting_interval(void)
 {
@@ -43,45 +61,32 @@ get_voting_interval(void)
                                                 usable_consensus_flavor());
 
   if (consensus) {
+    /* Ideally we have a live consensus and we can just use that. */
+    interval = (int)(consensus->fresh_until - consensus->valid_after);
+  } else if (authdir_mode(get_options()) || ASSUME_AUTHORITY_SCHEDULING) {
+    /* If we don't have a live consensus and we're an authority,
+     * we should believe our own view of what the schedule ought to be. */
+    interval = dirauth_sched_get_configured_interval();
+  } else if ((consensus = networkstatus_get_latest_consensus())) {
+    /* If we're a client, then maybe a latest consensus is good enough?
+     * It's better than falling back to the non-consensus case. */
     interval = (int)(consensus->fresh_until - consensus->valid_after);
   } else {
-    /* Same for both a testing and real network. We voluntarily ignore the
-     * InitialVotingInterval since it complexifies things and it doesn't
-     * affect the SR protocol. */
-    interval = get_options()->V3AuthVotingInterval;
+    /* We should never be reaching this point, since a client should never
+     * call this code unless they have some kind of a consensus. All we can
+     * do is hope that this network is using the default voting interval. */
+    tor_assert_nonfatal_unreached_once();
+    interval = DEFAULT_NETWORK_VOTING_INTERVAL;
   }
   tor_assert(interval > 0);
   return interval;
 }
 
-/* Given the current consensus, return the start time of the current round of
- * the SR protocol. For example, if it's 23:47:08, the current round thus
- * started at 23:47:00 for a voting interval of 10 seconds.
- *
- * This function uses the consensus voting schedule to derive its results,
- * instead of the actual consensus we are currently using, so it should be used
- * for voting purposes. */
-time_t
-get_start_time_of_current_round(void)
-{
-  const or_options_t *options = get_options();
-  int voting_interval = get_voting_interval();
-  /* First, get the start time of the next round */
-  time_t next_start = voting_schedule_get_next_valid_after_time();
-  /* Now roll back next_start by a voting interval to find the start time of
-     the current round. */
-  time_t curr_start = voting_schedule_get_start_of_next_interval(
-                                     next_start - voting_interval - 1,
-                                     voting_interval,
-                                     options->TestingV3AuthVotingStartOffset);
-  return curr_start;
-}
-
 /*
  * Public API
  */
 
-/* Encode the given shared random value and put it in dst. Destination
+/** Encode the given shared random value and put it in dst. Destination
  * buffer must be at least SR_SRV_VALUE_BASE64_LEN plus the NULL byte. */
 void
 sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
@@ -102,7 +107,7 @@ sr_srv_encode(char *dst, size_t dst_len, const sr_srv_t *srv)
   strlcpy(dst, buf, dst_len);
 }
 
-/* Return the current SRV string representation for the control port. Return a
+/** Return the current SRV string representation for the control port. Return a
  * newly allocated string on success containing the value else "" if not found
  * or if we don't have a valid consensus yet. */
 char *
@@ -118,7 +123,7 @@ sr_get_current_for_control(void)
   return srv_str;
 }
 
-/* Return the previous SRV string representation for the control port. Return
+/** Return the previous SRV string representation for the control port. Return
  * a newly allocated string on success containing the value else "" if not
  * found or if we don't have a valid consensus yet. */
 char *
@@ -134,7 +139,7 @@ sr_get_previous_for_control(void)
   return srv_str;
 }
 
-/* Return current shared random value from the latest consensus. Caller can
+/** Return current shared random value from the latest consensus. Caller can
  * NOT keep a reference to the returned pointer. Return NULL if none. */
 const sr_srv_t *
 sr_get_current(const networkstatus_t *ns)
@@ -158,7 +163,7 @@ sr_get_current(const networkstatus_t *ns)
   return NULL;
 }
 
-/* Return previous shared random value from the latest consensus. Caller can
+/** Return previous shared random value from the latest consensus. Caller can
  * NOT keep a reference to the returned pointer. Return NULL if none. */
 const sr_srv_t *
 sr_get_previous(const networkstatus_t *ns)
@@ -182,7 +187,7 @@ sr_get_previous(const networkstatus_t *ns)
   return NULL;
 }
 
-/* Parse a list of arguments from a SRV value either from a vote, consensus
+/** Parse a list of arguments from a SRV value either from a vote, consensus
  * or from our disk state and return a newly allocated srv object. NULL is
  * returned on error.
  *
@@ -252,8 +257,19 @@ sr_state_get_start_time_of_current_protocol_run(void)
                                                 usable_consensus_flavor());
   if (ns) {
     beginning_of_curr_round = ns->valid_after;
+  } else if (authdir_mode(get_options()) || ASSUME_AUTHORITY_SCHEDULING) {
+    beginning_of_curr_round = dirauth_sched_get_cur_valid_after_time();
   } else {
-    beginning_of_curr_round = get_start_time_of_current_round();
+    /* voting_interval comes from get_voting_interval(), so if we're in
+     * this case as a client, we already tried to get the voting interval
+     * from the latest_consensus and gave a bug warning if we couldn't.
+     *
+     * We wouldn't want to look at the latest consensus's valid_after time,
+     * since that would be out of date. */
+    beginning_of_curr_round = voting_sched_get_start_of_interval_after(
+                                             approx_time() - voting_interval,
+                                             voting_interval,
+                                             0);
   }
 
   /* Get current SR protocol round */
@@ -295,4 +311,3 @@ sr_state_get_protocol_run_duration(void)
   int total_protocol_rounds = SHARED_RANDOM_N_ROUNDS * SHARED_RANDOM_N_PHASES;
   return total_protocol_rounds * get_voting_interval();
 }
-
diff --git a/src/feature/hs_common/shared_random_client.h b/src/feature/hs_common/shared_random_client.h
index 95fe2c65ab..37a086d590 100644
--- a/src/feature/hs_common/shared_random_client.h
+++ b/src/feature/hs_common/shared_random_client.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,11 +38,9 @@ time_t sr_state_get_start_time_of_current_protocol_run(void);
 time_t sr_state_get_start_time_of_previous_protocol_run(void);
 unsigned int sr_state_get_phase_duration(void);
 unsigned int sr_state_get_protocol_run_duration(void);
-time_t get_start_time_of_current_round(void);
 
 #ifdef TOR_UNIT_TESTS
 
 #endif /* TOR_UNIT_TESTS */
 
-#endif /* TOR_SHARED_RANDOM_CLIENT_H */
-
+#endif /* !defined(TOR_SHARED_RANDOM_CLIENT_H) */
diff --git a/src/feature/keymgt/.may_include b/src/feature/keymgt/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/keymgt/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/keymgt/feature_keymgt.md b/src/feature/keymgt/feature_keymgt.md
new file mode 100644
index 0000000000..1eac7cca50
--- /dev/null
+++ b/src/feature/keymgt/feature_keymgt.md
@@ -0,0 +1,3 @@
+@dir /feature/keymgt
+@brief feature/keymgt: Store keys for relays, authorities, etc.
+
diff --git a/src/feature/keymgt/include.am b/src/feature/keymgt/include.am
new file mode 100644
index 0000000000..bc9beaa523
--- /dev/null
+++ b/src/feature/keymgt/include.am
@@ -0,0 +1,8 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/keymgt/loadkey.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/keymgt/loadkey.h
diff --git a/src/feature/keymgt/loadkey.c b/src/feature/keymgt/loadkey.c
index a8cbf0e582..7958bd964f 100644
--- a/src/feature/keymgt/loadkey.c
+++ b/src/feature/keymgt/loadkey.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,7 +33,7 @@
 /** Try to read an RSA key from <b>fname</b>.  If <b>fname</b> doesn't exist
  * and <b>generate</b> is true, create a new RSA key and save it in
  * <b>fname</b>.  Return the read/created key, or NULL on error.  Log all
- * errors at level <b>severity</b>. If <b>created_out/b> is non-NULL and a
+ * errors at level <b>severity</b>. If <b>created_out</b> is non-NULL and a
  * new key was created, set *<b>created_out</b> to true.
  */
 crypto_pk_t *
diff --git a/src/feature/keymgt/loadkey.h b/src/feature/keymgt/loadkey.h
index 8beee57a20..5a8ca32dea 100644
--- a/src/feature/keymgt/loadkey.h
+++ b/src/feature/keymgt/loadkey.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -52,4 +52,4 @@ int read_encrypted_secret_key(ed25519_secret_key_t *out,
 int write_encrypted_secret_key(const ed25519_secret_key_t *out,
                                const char *fname);
 
-#endif
+#endif /* !defined(TOR_LOADKEY_H) */
diff --git a/src/feature/nodelist/.may_include b/src/feature/nodelist/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/nodelist/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/nodelist/authcert.c b/src/feature/nodelist/authcert.c
index 7a065662a7..97e44d53e3 100644
--- a/src/feature/nodelist/authcert.c
+++ b/src/feature/nodelist/authcert.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -46,7 +46,7 @@
 #include "feature/nodelist/networkstatus_voter_info_st.h"
 #include "feature/nodelist/node_st.h"
 
-DECLARE_TYPED_DIGESTMAP_FNS(dsmap_, digest_ds_map_t, download_status_t)
+DECLARE_TYPED_DIGESTMAP_FNS(dsmap, digest_ds_map_t, download_status_t)
 #define DSMAP_FOREACH(map, keyvar, valvar) \
   DIGESTMAP_FOREACH(dsmap_to_digestmap(map), keyvar, download_status_t *, \
                     valvar)
@@ -380,7 +380,8 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
   int added_trusted_cert = 0;
 
   for (s = contents; *s; s = eos) {
-    authority_cert_t *cert = authority_cert_parse_from_string(s, &eos);
+    authority_cert_t *cert = authority_cert_parse_from_string(s, strlen(s),
+                                                              &eos);
     cert_list_t *cl;
     if (!cert) {
       failure_code = -1;
@@ -463,11 +464,13 @@ trusted_dirs_load_certs_from_string(const char *contents, int source,
           (ds->addr != cert->addr ||
            ds->dir_port != cert->dir_port)) {
         char *a = tor_dup_ip(cert->addr);
-        log_notice(LD_DIR, "Updating address for directory authority %s "
-                   "from %s:%d to %s:%d based on certificate.",
-                   ds->nickname, ds->address, (int)ds->dir_port,
-                   a, cert->dir_port);
-        tor_free(a);
+        if (a) {
+          log_notice(LD_DIR, "Updating address for directory authority %s "
+                     "from %s:%d to %s:%d based on certificate.",
+                     ds->nickname, ds->address, (int)ds->dir_port,
+                     a, cert->dir_port);
+          tor_free(a);
+        }
         ds->addr = cert->addr;
         ds->dir_port = cert->dir_port;
       }
diff --git a/src/feature/nodelist/authcert.h b/src/feature/nodelist/authcert.h
index 2effdb06e6..33065589ba 100644
--- a/src/feature/nodelist/authcert.h
+++ b/src/feature/nodelist/authcert.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -57,4 +57,4 @@ MOCK_DECL(download_status_t *, download_status_for_authority_id_and_sk,
 
 void authcert_free_all(void);
 
-#endif
+#endif /* !defined(TOR_AUTHCERT_H) */
diff --git a/src/feature/nodelist/authority_cert_st.h b/src/feature/nodelist/authority_cert_st.h
index 68a84bc452..9145b12bbf 100644
--- a/src/feature/nodelist/authority_cert_st.h
+++ b/src/feature/nodelist/authority_cert_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file authority_cert_st.h
+ * @brief Authority certificate structure.
+ **/
+
 #ifndef AUTHORITY_CERT_ST_H
 #define AUTHORITY_CERT_ST_H
 
@@ -28,5 +33,4 @@ struct authority_cert_t {
   uint16_t dir_port;
 };
 
-#endif
-
+#endif /* !defined(AUTHORITY_CERT_ST_H) */
diff --git a/src/feature/nodelist/desc_store_st.h b/src/feature/nodelist/desc_store_st.h
index b04a1abc7d..5f35a490a5 100644
--- a/src/feature/nodelist/desc_store_st.h
+++ b/src/feature/nodelist/desc_store_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file desc_store_st.h
+ * @brief Routerinfo/extrainfo storage structure.
+ **/
+
 #ifndef DESC_STORE_ST_H
 #define DESC_STORE_ST_H
 
@@ -36,4 +41,4 @@ struct desc_store_t {
   size_t bytes_dropped;
 };
 
-#endif
+#endif /* !defined(DESC_STORE_ST_H) */
diff --git a/src/feature/nodelist/describe.c b/src/feature/nodelist/describe.c
index 5c376408c0..00896d5a44 100644
--- a/src/feature/nodelist/describe.c
+++ b/src/feature/nodelist/describe.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,66 +9,108 @@
  * \brief Format short descriptions of relays.
  */
 
+#define DESCRIBE_PRIVATE
+
 #include "core/or/or.h"
 #include "feature/nodelist/describe.h"
-#include "feature/nodelist/routerinfo.h"
 
 #include "core/or/extend_info_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
-
-/**
- * Longest allowed output of format_node_description, plus 1 character for
- * NUL.  This allows space for:
- * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
- * " [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
- * plus a terminating NUL.
- */
-#define NODE_DESC_BUF_LEN (MAX_VERBOSE_NICKNAME_LEN+4+TOR_ADDR_BUF_LEN)
+#include "feature/nodelist/microdesc_st.h"
 
 /** Use <b>buf</b> (which must be at least NODE_DESC_BUF_LEN bytes long) to
  * hold a human-readable description of a node with identity digest
- * <b>id_digest</b>, named-status <b>is_named</b>, nickname <b>nickname</b>,
- * and address <b>addr</b> or <b>addr32h</b>.
+ * <b>id_digest</b>, nickname <b>nickname</b>, and addresses <b>addr32h</b> and
+ * <b>addr</b>.
  *
  * The <b>nickname</b> and <b>addr</b> fields are optional and may be set to
- * NULL.  The <b>addr32h</b> field is optional and may be set to 0.
+ * NULL or the null address.  The <b>addr32h</b> field is optional and may be
+ * set to 0.
  *
  * Return a pointer to the front of <b>buf</b>.
+ * If buf is NULL, return a string constant describing the error.
  */
-static const char *
+STATIC const char *
 format_node_description(char *buf,
                         const char *id_digest,
-                        int is_named,
                         const char *nickname,
                         const tor_addr_t *addr,
                         uint32_t addr32h)
 {
-  char *cp;
+  size_t rv = 0;
+  bool has_addr = addr && !tor_addr_is_null(addr);
 
   if (!buf)
     return "<NULL BUFFER>";
 
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, id_digest, DIGEST_LEN);
-  cp = buf+1+HEX_DIGEST_LEN;
+  memset(buf, 0, NODE_DESC_BUF_LEN);
+
+  if (!id_digest) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<NULL ID DIGEST>", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    return buf;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", NODE_DESC_BUF_LEN);
+  tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  id_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   if (nickname) {
-    buf[1+HEX_DIGEST_LEN] = is_named ? '=' : '~';
-    strlcpy(buf+1+HEX_DIGEST_LEN+1, nickname, MAX_NICKNAME_LEN+1);
-    cp += strlen(cp);
+    rv = strlcat(buf, "~", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+    rv = strlcat(buf, nickname, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
-  if (addr32h || addr) {
-    memcpy(cp, " at ", 4);
-    cp += 4;
-    if (addr) {
-      tor_addr_to_str(cp, addr, TOR_ADDR_BUF_LEN, 0);
-    } else {
-      struct in_addr in;
-      in.s_addr = htonl(addr32h);
-      tor_inet_ntoa(&in, cp, INET_NTOA_BUF_LEN);
-    }
+  if (addr32h || has_addr) {
+    rv = strlcat(buf, " at ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
   }
+  if (addr32h) {
+    int ntoa_rv = 0;
+    char ipv4_addr_str[INET_NTOA_BUF_LEN];
+    memset(ipv4_addr_str, 0, sizeof(ipv4_addr_str));
+    struct in_addr in;
+    memset(&in, 0, sizeof(in));
+
+    in.s_addr = htonl(addr32h);
+    ntoa_rv = tor_inet_ntoa(&in, ipv4_addr_str, sizeof(ipv4_addr_str));
+    tor_assert_nonfatal(ntoa_rv >= 0);
+
+    rv = strlcat(buf, ipv4_addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  /* Both addresses are valid */
+  if (addr32h && has_addr) {
+    rv = strlcat(buf, " and ", NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+  if (has_addr) {
+    const char *str_rv = NULL;
+    char addr_str[TOR_ADDR_BUF_LEN];
+    memset(addr_str, 0, sizeof(addr_str));
+
+    str_rv = tor_addr_to_str(addr_str, addr, sizeof(addr_str), 1);
+    tor_assert_nonfatal(str_rv == addr_str);
+
+    rv = strlcat(buf, addr_str, NODE_DESC_BUF_LEN);
+    tor_assert_nonfatal(rv < NODE_DESC_BUF_LEN);
+  }
+
   return buf;
 }
 
@@ -84,11 +126,11 @@ router_describe(const routerinfo_t *ri)
 
   if (!ri)
     return "<null>";
+
   return format_node_description(buf,
                                  ri->cache_info.identity_digest,
-                                 0,
                                  ri->nickname,
-                                 NULL,
+                                 &ri->ipv6_addr,
                                  ri->addr);
 }
 
@@ -103,25 +145,33 @@ node_describe(const node_t *node)
   static char buf[NODE_DESC_BUF_LEN];
   const char *nickname = NULL;
   uint32_t addr32h = 0;
-  int is_named = 0;
+  const tor_addr_t *ipv6_addr = NULL;
 
   if (!node)
     return "<null>";
 
   if (node->rs) {
     nickname = node->rs->nickname;
-    is_named = node->rs->is_named;
     addr32h = node->rs->addr;
+    ipv6_addr = &node->rs->ipv6_addr;
+    /* Support consensus versions less than 28, when IPv6 addresses were in
+     * microdescs. This code can be removed when 0.2.9 is no longer supported,
+     * and the MIN_METHOD_FOR_NO_A_LINES_IN_MICRODESC macro is removed. */
+    if (node->md && tor_addr_is_null(ipv6_addr)) {
+      ipv6_addr = &node->md->ipv6_addr;
+    }
   } else if (node->ri) {
     nickname = node->ri->nickname;
     addr32h = node->ri->addr;
+    ipv6_addr = &node->ri->ipv6_addr;
+  } else {
+    return "<null rs and ri>";
   }
 
   return format_node_description(buf,
                                  node->identity,
-                                 is_named,
                                  nickname,
-                                 NULL,
+                                 ipv6_addr,
                                  addr32h);
 }
 
@@ -137,11 +187,11 @@ routerstatus_describe(const routerstatus_t *rs)
 
   if (!rs)
     return "<null>";
+
   return format_node_description(buf,
                                  rs->identity_digest,
-                                 rs->is_named,
                                  rs->nickname,
-                                 NULL,
+                                 &rs->ipv6_addr,
                                  rs->addr);
 }
 
@@ -157,9 +207,9 @@ extend_info_describe(const extend_info_t *ei)
 
   if (!ei)
     return "<null>";
+
   return format_node_description(buf,
                                  ei->identity_digest,
-                                 0,
                                  ei->nickname,
                                  &ei->addr,
                                  0);
@@ -175,9 +225,39 @@ extend_info_describe(const extend_info_t *ei)
 void
 router_get_verbose_nickname(char *buf, const routerinfo_t *router)
 {
-  buf[0] = '$';
-  base16_encode(buf+1, HEX_DIGEST_LEN+1, router->cache_info.identity_digest,
-                DIGEST_LEN);
-  buf[1+HEX_DIGEST_LEN] = '~';
-  strlcpy(buf+1+HEX_DIGEST_LEN+1, router->nickname, MAX_NICKNAME_LEN+1);
+  size_t rv = 0;
+
+  if (!buf)
+    return;
+
+  memset(buf, 0, MAX_VERBOSE_NICKNAME_LEN+1);
+
+  if (!router) {
+    /* strlcpy() returns the length of the source string it attempted to copy,
+     * ignoring any required truncation due to the buffer length. */
+    rv = strlcpy(buf, "<null>", MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+    return;
+  }
+
+  /* strlcat() returns the length of the concatenated string it attempted to
+   * create, ignoring any required truncation due to the buffer length.  */
+  rv = strlcat(buf, "$", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  {
+    char hex_digest[HEX_DIGEST_LEN+1];
+    memset(hex_digest, 0, sizeof(hex_digest));
+
+    base16_encode(hex_digest, sizeof(hex_digest),
+                  router->cache_info.identity_digest, DIGEST_LEN);
+    rv = strlcat(buf, hex_digest, MAX_VERBOSE_NICKNAME_LEN+1);
+    tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+  }
+
+  rv = strlcat(buf, "~", MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
+
+  rv = strlcat(buf, router->nickname, MAX_VERBOSE_NICKNAME_LEN+1);
+  tor_assert_nonfatal(rv < MAX_VERBOSE_NICKNAME_LEN+1);
 }
diff --git a/src/feature/nodelist/describe.h b/src/feature/nodelist/describe.h
index 018af6470e..d0fa1af263 100644
--- a/src/feature/nodelist/describe.h
+++ b/src/feature/nodelist/describe.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,36 @@ const char *node_describe(const struct node_t *node);
 const char *router_describe(const struct routerinfo_t *ri);
 const char *routerstatus_describe(const struct routerstatus_t *ri);
 
-#endif
+void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
+
+#if defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS)
+
+/**
+ * Longest allowed output for an IPv4 address "255.255.255.255", with NO
+ * terminating NUL.
+ */
+#define IPV4_BUF_LEN_NO_NUL 15
+
+/**
+ * Longest allowed output of format_node_description, plus 1 character for
+ * NUL.  This allows space for:
+ * "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF~xxxxxxxxxxxxxxxxxxx at"
+ * " 255.255.255.255 and [ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]"
+ * plus a terminating NUL.
+ */
+#define NODE_DESC_BUF_LEN \
+  (MAX_VERBOSE_NICKNAME_LEN+4+IPV4_BUF_LEN_NO_NUL+5+TOR_ADDR_BUF_LEN)
+
+#endif /* defined(DESCRIBE_PRIVATE) || defined(TOR_UNIT_TESTS) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC const char *format_node_description(char *buf,
+                                           const char *id_digest,
+                                           const char *nickname,
+                                           const tor_addr_t *addr,
+                                           uint32_t addr32h);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_DESCRIBE_H) */
diff --git a/src/feature/nodelist/dirlist.c b/src/feature/nodelist/dirlist.c
index b4abffad67..4317491043 100644
--- a/src/feature/nodelist/dirlist.c
+++ b/src/feature/nodelist/dirlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,8 +27,9 @@
 #include "core/or/or.h"
 
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "core/or/policies.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/dirlist.h"
@@ -70,7 +71,7 @@ add_trusted_dir_to_nodelist_addr_set(const dir_server_t *dir)
 }
 
 /** Go over the trusted directory server list and add their address(es) to the
- * nodelist address set. This is called every time a new consensus is set. */
+ * nodelist address set. This is called everytime a new consensus is set. */
 MOCK_IMPL(void,
 dirlist_add_trusted_dir_addresses, (void))
 {
@@ -362,6 +363,9 @@ trusted_dir_server_new(const char *nickname, const char *address,
     }
     if (!hostname)
       hostname = tor_dup_ip(a);
+
+    if (!hostname)
+      return NULL;
   } else {
     if (tor_lookup_hostname(address, &a)) {
       log_warn(LD_CONFIG,
diff --git a/src/feature/nodelist/dirlist.h b/src/feature/nodelist/dirlist.h
index 527af35427..9201e76a9c 100644
--- a/src/feature/nodelist/dirlist.h
+++ b/src/feature/nodelist/dirlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -46,4 +46,4 @@ void dirlist_free_all(void);
 
 MOCK_DECL(void, dirlist_add_trusted_dir_addresses, (void));
 
-#endif
+#endif /* !defined(TOR_DIRLIST_H) */
diff --git a/src/feature/nodelist/document_signature_st.h b/src/feature/nodelist/document_signature_st.h
index 66e32c422f..4bde9d89ec 100644
--- a/src/feature/nodelist/document_signature_st.h
+++ b/src/feature/nodelist/document_signature_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file document_signature_st.h
+ * @brief Authority signature structure
+ **/
+
 #ifndef DOCUMENT_SIGNATURE_ST_H
 #define DOCUMENT_SIGNATURE_ST_H
 
@@ -25,5 +30,4 @@ struct document_signature_t {
                                      * as good. */
 };
 
-#endif
-
+#endif /* !defined(DOCUMENT_SIGNATURE_ST_H) */
diff --git a/src/feature/nodelist/extrainfo_st.h b/src/feature/nodelist/extrainfo_st.h
index c54277b05e..6bd6232cd8 100644
--- a/src/feature/nodelist/extrainfo_st.h
+++ b/src/feature/nodelist/extrainfo_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file extrainfo_st.h
+ * @brief A relay's extra-info structure.
+ **/
+
 #ifndef EXTRAINFO_ST_H
 #define EXTRAINFO_ST_H
 
@@ -26,5 +31,4 @@ struct extrainfo_t {
   size_t pending_sig_len;
 };
 
-#endif
-
+#endif /* !defined(EXTRAINFO_ST_H) */
diff --git a/src/feature/nodelist/feature_nodelist.md b/src/feature/nodelist/feature_nodelist.md
new file mode 100644
index 0000000000..9d715308c2
--- /dev/null
+++ b/src/feature/nodelist/feature_nodelist.md
@@ -0,0 +1,2 @@
+@dir /feature/nodelist
+@brief feature/nodelist: Download and manage a list of relays
diff --git a/src/feature/nodelist/fmt_routerstatus.c b/src/feature/nodelist/fmt_routerstatus.c
index 75cab7a0af..ca4a312639 100644
--- a/src/feature/nodelist/fmt_routerstatus.c
+++ b/src/feature/nodelist/fmt_routerstatus.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,64 +14,19 @@
 #include "core/or/or.h"
 #include "feature/nodelist/fmt_routerstatus.h"
 
-/* #include "lib/container/buffers.h" */
-/* #include "app/config/config.h" */
-/* #include "app/config/confparse.h" */
-/* #include "core/or/channel.h" */
-/* #include "core/or/channeltls.h" */
-/* #include "core/or/command.h" */
-/* #include "core/mainloop/connection.h" */
-/* #include "core/or/connection_or.h" */
-/* #include "feature/dircache/conscache.h" */
-/* #include "feature/dircache/consdiffmgr.h" */
-/* #include "feature/control/control.h" */
-/* #include "feature/dircache/directory.h" */
-/* #include "feature/dircache/dirserv.h" */
-/* #include "feature/hibernate/hibernate.h" */
-/* #include "feature/dirauth/keypin.h" */
-/* #include "core/mainloop/mainloop.h" */
-/* #include "feature/nodelist/microdesc.h" */
-/* #include "feature/nodelist/networkstatus.h" */
-/* #include "feature/nodelist/nodelist.h" */
 #include "core/or/policies.h"
-/* #include "core/or/protover.h" */
-/* #include "feature/stats/rephist.h" */
-/* #include "feature/relay/router.h" */
-/* #include "feature/nodelist/dirlist.h" */
 #include "feature/nodelist/routerlist.h"
-
-/* #include "feature/nodelist/routerparse.h" */
-/* #include "feature/nodelist/routerset.h" */
-/* #include "feature/nodelist/torcert.h" */
-/* #include "feature/dircommon/voting_schedule.h" */
-
 #include "feature/dirauth/dirvote.h"
 
-/* #include "feature/dircache/cached_dir_st.h" */
-/* #include "feature/dircommon/dir_connection_st.h" */
-/* #include "feature/nodelist/extrainfo_st.h" */
-/* #include "feature/nodelist/microdesc_st.h" */
-/* #include "feature/nodelist/node_st.h" */
 #include "feature/nodelist/routerinfo_st.h"
-/* #include "feature/nodelist/routerlist_st.h" */
-/* #include "core/or/tor_version_st.h" */
 #include "feature/nodelist/vote_routerstatus_st.h"
 
-/* #include "lib/compress/compress.h" */
-/* #include "lib/container/order.h" */
 #include "lib/crypt_ops/crypto_format.h"
-/* #include "lib/encoding/confline.h" */
-
-/* #include "lib/encoding/keyval.h" */
 
 /** Helper: write the router-status information in <b>rs</b> into a newly
  * allocated character buffer.  Use the same format as in network-status
  * documents.  If <b>version</b> is non-NULL, add a "v" line for the platform.
  *
- * consensus_method is the current consensus method when format is
- * NS_V3_CONSENSUS or NS_V3_CONSENSUS_MICRODESC. It is ignored for other
- * formats: pass ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD.
- *
  * Return 0 on success, -1 on failure.
  *
  * The format argument has one of the following values:
@@ -88,7 +43,6 @@ char *
 routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                           const char *protocols,
                           routerstatus_format_type_t format,
-                          int consensus_method,
                           const vote_routerstatus_t *vrs)
 {
   char *summary;
@@ -99,6 +53,10 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
   char digest64[BASE64_DIGEST_LEN+1];
   smartlist_t *chunks = smartlist_new();
 
+  const char *ip_str = fmt_addr32(rs->addr);
+  if (ip_str[0] == '\0')
+    goto err;
+
   format_iso_time(published, rs->published_on);
   digest_to_base64(identity64, rs->identity_digest);
   digest_to_base64(digest64, rs->descriptor_digest);
@@ -110,7 +68,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    (format==NS_V3_CONSENSUS_MICRODESC)?"":digest64,
                    (format==NS_V3_CONSENSUS_MICRODESC)?"":" ",
                    published,
-                   fmt_addr32(rs->addr),
+                   ip_str,
                    (int)rs->or_port,
                    (int)rs->dir_port);
 
@@ -119,12 +77,6 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
    * networkstatus_type_t values, with an additional control port value
    * added -MP */
 
-  /* V3 microdesc consensuses only have "a" lines in later consensus methods
-   */
-  if (format == NS_V3_CONSENSUS_MICRODESC &&
-      consensus_method < MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS)
-    goto done;
-
   /* Possible "a" line. At most one for now. */
   if (!tor_addr_is_null(&rs->ipv6_addr)) {
     smartlist_add_asprintf(chunks, "a %s\n",
@@ -135,7 +87,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     goto done;
 
   smartlist_add_asprintf(chunks,
-                   "s%s%s%s%s%s%s%s%s%s%s\n",
+                   "s%s%s%s%s%s%s%s%s%s%s%s\n",
                   /* These must stay in alphabetical order. */
                    rs->is_authority?" Authority":"",
                    rs->is_bad_exit?" BadExit":"",
@@ -145,6 +97,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
                    rs->is_hs_dir?" HSDir":"",
                    rs->is_flagged_running?" Running":"",
                    rs->is_stable?" Stable":"",
+                   rs->is_staledesc?" StaleDesc":"",
                    rs->is_v2_dir?" V2Dir":"",
                    rs->is_valid?" Valid":"");
 
@@ -164,6 +117,8 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     if (format != NS_CONTROL_PORT) {
       /* Blow up more or less nicely if we didn't get anything or not the
        * thing we expected.
+       * This should be kept in sync with the function
+       * routerstatus_has_visibly_changed and the struct routerstatus_t
        */
       if (!desc) {
         char id[HEX_DIGEST_LEN+1];
@@ -232,7 +187,7 @@ routerstatus_format_entry(const routerstatus_t *rs, const char *version,
     }
 
     if (format == NS_V3_VOTE && vrs) {
-      if (tor_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
+      if (fast_mem_is_zero((char*)vrs->ed25519_id, ED25519_PUBKEY_LEN)) {
         smartlist_add_strdup(chunks, "id ed25519 none\n");
       } else {
         char ed_b64[BASE64_DIGEST256_LEN+1];
diff --git a/src/feature/nodelist/fmt_routerstatus.h b/src/feature/nodelist/fmt_routerstatus.h
index ddd7a7cf37..a007989af3 100644
--- a/src/feature/nodelist/fmt_routerstatus.h
+++ b/src/feature/nodelist/fmt_routerstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -35,7 +35,6 @@ char *routerstatus_format_entry(
                               const char *version,
                               const char *protocols,
                               routerstatus_format_type_t format,
-                              int consensus_method,
                               const vote_routerstatus_t *vrs);
 
 #endif /* !defined(TOR_FMT_ROUTERSTATUS_H) */
diff --git a/src/feature/nodelist/include.am b/src/feature/nodelist/include.am
new file mode 100644
index 0000000000..2f5d58ec1c
--- /dev/null
+++ b/src/feature/nodelist/include.am
@@ -0,0 +1,49 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 				\
+	src/feature/nodelist/authcert.c		\
+	src/feature/nodelist/describe.c		\
+	src/feature/nodelist/dirlist.c		\
+	src/feature/nodelist/microdesc.c	\
+	src/feature/nodelist/networkstatus.c	\
+	src/feature/nodelist/nickname.c		\
+	src/feature/nodelist/nodefamily.c	\
+	src/feature/nodelist/nodelist.c		\
+	src/feature/nodelist/node_select.c	\
+	src/feature/nodelist/routerinfo.c	\
+	src/feature/nodelist/routerlist.c	\
+	src/feature/nodelist/routerset.c	\
+	src/feature/nodelist/fmt_routerstatus.c	\
+	src/feature/nodelist/torcert.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/nodelist/authcert.h			\
+	src/feature/nodelist/authority_cert_st.h	\
+	src/feature/nodelist/describe.h			\
+	src/feature/nodelist/desc_store_st.h		\
+	src/feature/nodelist/dirlist.h			\
+	src/feature/nodelist/document_signature_st.h	\
+	src/feature/nodelist/extrainfo_st.h		\
+	src/feature/nodelist/microdesc.h		\
+	src/feature/nodelist/microdesc_st.h		\
+	src/feature/nodelist/networkstatus.h		\
+	src/feature/nodelist/networkstatus_sr_info_st.h	\
+	src/feature/nodelist/networkstatus_st.h		\
+	src/feature/nodelist/networkstatus_voter_info_st.h	\
+	src/feature/nodelist/nickname.h			\
+	src/feature/nodelist/node_st.h			\
+	src/feature/nodelist/nodefamily.h		\
+	src/feature/nodelist/nodefamily_st.h		\
+	src/feature/nodelist/nodelist.h			\
+	src/feature/nodelist/node_select.h		\
+	src/feature/nodelist/routerinfo.h		\
+	src/feature/nodelist/routerinfo_st.h		\
+	src/feature/nodelist/routerlist.h		\
+	src/feature/nodelist/routerlist_st.h		\
+	src/feature/nodelist/routerset.h		\
+	src/feature/nodelist/fmt_routerstatus.h		\
+	src/feature/nodelist/routerstatus_st.h		\
+	src/feature/nodelist/signed_descriptor_st.h	\
+	src/feature/nodelist/torcert.h			\
+	src/feature/nodelist/vote_routerstatus_st.h
diff --git a/src/feature/nodelist/microdesc.c b/src/feature/nodelist/microdesc.c
index dafaabb5e5..cf7732b8dc 100644
--- a/src/feature/nodelist/microdesc.c
+++ b/src/feature/nodelist/microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* Copyright (c) 2009-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,11 +18,13 @@
 #include "feature/client/entrynodes.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dlstatus.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dirparse/microdesc_parse.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/relay/router.h"
@@ -69,6 +71,8 @@ struct microdesc_cache_t {
 };
 
 static microdesc_cache_t *get_microdesc_cache_noload(void);
+static void warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                              const char *activity);
 
 /** Helper: computes a hash of <b>md</b> to place it in a hash table. */
 static inline unsigned int
@@ -86,10 +90,10 @@ microdesc_eq_(microdesc_t *a, microdesc_t *b)
 }
 
 HT_PROTOTYPE(microdesc_map, microdesc_t, node,
-             microdesc_hash_, microdesc_eq_)
+             microdesc_hash_, microdesc_eq_);
 HT_GENERATE2(microdesc_map, microdesc_t, node,
              microdesc_hash_, microdesc_eq_, 0.6,
-             tor_reallocarray_, tor_free_)
+             tor_reallocarray_, tor_free_);
 
 /************************* md fetch fail cache *****************************/
 
@@ -110,8 +114,9 @@ microdesc_note_outdated_dirserver(const char *relay_digest)
 
   /* If we have a reasonably live consensus, then most of our dirservers should
    * still be caching all the microdescriptors in it. Reasonably live
-   * consensuses are up to a day old. But microdescriptors expire 7 days after
-   * the last consensus that referenced them. */
+   * consensuses are up to a day old (or a day in the future). But
+   * microdescriptors expire 7 days after the last consensus that referenced
+   * them. */
   if (!networkstatus_get_reasonably_live_consensus(approx_time(),
                                                    FLAV_MICRODESC)) {
     return;
@@ -221,6 +226,8 @@ dump_microdescriptor(int fd, microdesc_t *md, size_t *annotation_len_out)
   }
 
   md->off = tor_fd_getpos(fd);
+  warn_if_nul_found(md->body, md->bodylen, (int64_t) md->off,
+                    "dumping a microdescriptor");
   written = write_all_to_fd(fd, md->body, md->bodylen);
   if (written != (ssize_t)md->bodylen) {
     written = written < 0 ? 0 : written;
@@ -480,6 +487,27 @@ microdesc_cache_clear(microdesc_cache_t *cache)
   cache->bytes_dropped = 0;
 }
 
+static void
+warn_if_nul_found(const char *inp, size_t len, int64_t offset,
+                  const char *activity)
+{
+  const char *nul_found = memchr(inp, 0, len);
+  if (BUG(nul_found)) {
+    log_warn(LD_BUG, "Found unexpected NUL while %s, offset %"PRId64
+             "at position %"TOR_PRIuSZ"/%"TOR_PRIuSZ".",
+             activity, offset, (nul_found - inp), len);
+    const char *start_excerpt_at, *eos = inp + len;
+    if ((nul_found - inp) >= 16)
+      start_excerpt_at = nul_found - 16;
+    else
+      start_excerpt_at = inp;
+    size_t excerpt_len = MIN(32, eos - start_excerpt_at);
+    char tmp[65];
+    base16_encode(tmp, sizeof(tmp), start_excerpt_at, excerpt_len);
+    log_warn(LD_BUG, "      surrounding string: %s", tmp);
+  }
+}
+
 /** Reload the contents of <b>cache</b> from disk.  If it is empty, load it
  * for the first time.  Return 0 on success, -1 on failure. */
 int
@@ -497,6 +525,7 @@ microdesc_cache_reload(microdesc_cache_t *cache)
 
   mm = cache->cache_content = tor_mmap_file(cache->cache_fname);
   if (mm) {
+    warn_if_nul_found(mm->data, mm->size, 0, "scanning microdesc cache");
     added = microdescs_add_to_cache(cache, mm->data, mm->data+mm->size,
                                     SAVED_IN_CACHE, 0, -1, NULL);
     if (added) {
@@ -508,7 +537,9 @@ microdesc_cache_reload(microdesc_cache_t *cache)
   journal_content = read_file_to_str(cache->journal_fname,
                                      RFTS_IGNORE_MISSING, &st);
   if (journal_content) {
-    cache->journal_len = (size_t) st.st_size;
+    cache->journal_len = strlen(journal_content);
+    warn_if_nul_found(journal_content, (size_t)st.st_size, 0,
+                      "reading microdesc journal");
     added = microdescs_add_to_cache(cache, journal_content,
                                     journal_content+st.st_size,
                                     SAVED_IN_JOURNAL, 0, -1, NULL);
@@ -544,8 +575,8 @@ microdesc_cache_clean(microdesc_cache_t *cache, time_t cutoff, int force)
   size_t bytes_dropped = 0;
   time_t now = time(NULL);
 
-  /* If we don't know a live consensus, don't believe last_listed values: we
-   * might be starting up after being down for a while. */
+  /* If we don't know a reasonably live consensus, don't believe last_listed
+   * values: we might be starting up after being down for a while. */
   if (! force &&
       ! networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC))
       return;
@@ -884,10 +915,7 @@ microdesc_free_(microdesc_t *md, const char *fname, int lineno)
   if (md->body && md->saved_location != SAVED_IN_CACHE)
     tor_free(md->body);
 
-  if (md->family) {
-    SMARTLIST_FOREACH(md->family, char *, cp, tor_free(cp));
-    smartlist_free(md->family);
-  }
+  nodefamily_free(md->family);
   short_policy_free(md->exit_policy);
   short_policy_free(md->ipv6_exit_policy);
 
@@ -943,7 +971,7 @@ microdesc_list_missing_digest256(networkstatus_t *ns, microdesc_cache_t *cache,
       continue;
     if (skip && digest256map_get(skip, (const uint8_t*)rs->descriptor_digest))
       continue;
-    if (tor_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
+    if (fast_mem_is_zero(rs->descriptor_digest, DIGEST256_LEN))
       continue;
     /* XXXX Also skip if we're a noncache and wouldn't use this router.
      * XXXX NM Microdesc
@@ -970,9 +998,10 @@ update_microdesc_downloads(time_t now)
 
   if (should_delay_dir_fetches(options, NULL))
     return;
-  if (directory_too_idle_to_fetch_descriptors(options, now))
+  if (dirclient_too_idle_to_fetch_descriptors(options, now))
     return;
 
+  /* Give up if we don't have a reasonably live consensus. */
   consensus = networkstatus_get_reasonably_live_consensus(now, FLAV_MICRODESC);
   if (!consensus)
     return;
diff --git a/src/feature/nodelist/microdesc.h b/src/feature/nodelist/microdesc.h
index c18099d540..b352f58e34 100644
--- a/src/feature/nodelist/microdesc.h
+++ b/src/feature/nodelist/microdesc.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/nodelist/microdesc_st.h b/src/feature/nodelist/microdesc_st.h
index bb8b23d664..410403e965 100644
--- a/src/feature/nodelist/microdesc_st.h
+++ b/src/feature/nodelist/microdesc_st.h
@@ -1,16 +1,24 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file microdesc_st.h
+ * @brief Microdescriptor structure
+ **/
+
 #ifndef MICRODESC_ST_H
 #define MICRODESC_ST_H
 
 struct curve25519_public_key_t;
 struct ed25519_public_key_t;
+struct nodefamily_t;
 struct short_policy_t;
 
+#include "ext/ht.h"
+
 /** A microdescriptor is the smallest amount of information needed to build a
  * circuit through a router.  They are generated by the directory authorities,
  * using information from the uploaded routerinfo documents.  They are not
@@ -32,6 +40,8 @@ struct microdesc_t {
   unsigned int no_save : 1;
   /** If true, this microdesc has an entry in the microdesc_map */
   unsigned int held_in_map : 1;
+  /** True iff the exit policy for this router rejects everything. */
+  unsigned int policy_is_reject_star : 1;
   /** Reference count: how many node_ts have a reference to this microdesc? */
   unsigned int held_by_nodes;
 
@@ -69,12 +79,12 @@ struct microdesc_t {
   tor_addr_t ipv6_addr;
   /** As routerinfo_t.ipv6_orport */
   uint16_t ipv6_orport;
-  /** As routerinfo_t.family */
-  smartlist_t *family;
+  /** As routerinfo_t.family, with readable members parsed. */
+  struct nodefamily_t *family;
   /** IPv4 exit policy summary */
   struct short_policy_t *exit_policy;
   /** IPv6 exit policy summary */
   struct short_policy_t *ipv6_exit_policy;
 };
 
-#endif
+#endif /* !defined(MICRODESC_ST_H) */
diff --git a/src/feature/nodelist/networkstatus.c b/src/feature/nodelist/networkstatus.c
index c74acd8b74..e07d58c91c 100644
--- a/src/feature/nodelist/networkstatus.c
+++ b/src/feature/nodelist/networkstatus.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -44,6 +44,7 @@
 #include "core/mainloop/netstatus.h"
 #include "core/or/channel.h"
 #include "core/or/channelpadding.h"
+#include "core/or/circuitpadding.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitstats.h"
@@ -57,16 +58,18 @@
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dircache/consdiffmgr.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/fmt_routerstatus.h"
@@ -81,6 +84,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 
+#include "feature/dirauth/dirauth_periodic.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/shared_random.h"
@@ -98,6 +102,7 @@
 #include "feature/nodelist/routerlist_st.h"
 #include "feature/dirauth/vote_microdesc_hash_st.h"
 #include "feature/nodelist/vote_routerstatus_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -116,8 +121,6 @@ STATIC networkstatus_t *current_md_consensus = NULL;
 typedef struct consensus_waiting_for_certs_t {
   /** The consensus itself. */
   networkstatus_t *consensus;
-  /** The encoded version of the consensus, nul-terminated. */
-  char *body;
   /** When did we set the current value of consensus_waiting_for_certs?  If
    * this is too recent, we shouldn't try to fetch a new consensus for a
    * little while, to give ourselves time to get certificates for this one. */
@@ -179,6 +182,10 @@ static void update_consensus_bootstrap_multiple_downloads(
 static int networkstatus_check_required_protocols(const networkstatus_t *ns,
                                                   int client_mode,
                                                   char **warning_out);
+static int reload_consensus_from_file(const char *fname,
+                                      const char *flavor,
+                                      unsigned flags,
+                                      const char *source_dir);
 
 /** Forget that we've warned about anything networkstatus-related, so we will
  * give fresh warnings if the same behavior happens again. */
@@ -210,14 +217,11 @@ networkstatus_reset_download_failures(void)
     download_status_reset(&consensus_bootstrap_dl_status[i]);
 }
 
-/**
- * Read and and return the cached consensus of type <b>flavorname</b>.  If
- * <b>unverified</b> is true, get the one we haven't verified. Return NULL if
- * the file isn't there. */
-static char *
-networkstatus_read_cached_consensus_impl(int flav,
-                                         const char *flavorname,
-                                         int unverified_consensus)
+/** Return the filename used to cache the consensus of a given flavor */
+MOCK_IMPL(char *,
+networkstatus_get_cache_fname,(int flav,
+                               const char *flavorname,
+                               int unverified_consensus))
 {
   char buf[128];
   const char *prefix;
@@ -232,21 +236,35 @@ networkstatus_read_cached_consensus_impl(int flav,
     tor_snprintf(buf, sizeof(buf), "%s-%s-consensus", prefix, flavorname);
   }
 
-  char *filename = get_cachedir_fname(buf);
-  char *result = read_file_to_str(filename, RFTS_IGNORE_MISSING, NULL);
+  return get_cachedir_fname(buf);
+}
+
+/**
+ * Read and and return the cached consensus of type <b>flavorname</b>.  If
+ * <b>unverified</b> is false, get the one we haven't verified. Return NULL if
+ * the file isn't there. */
+static tor_mmap_t *
+networkstatus_map_cached_consensus_impl(int flav,
+                                        const char *flavorname,
+                                        int unverified_consensus)
+{
+  char *filename = networkstatus_get_cache_fname(flav,
+                                                 flavorname,
+                                                 unverified_consensus);
+  tor_mmap_t *result = tor_mmap_file(filename);
   tor_free(filename);
   return result;
 }
 
-/** Return a new string containing the current cached consensus of flavor
- * <b>flavorname</b>. */
-char *
-networkstatus_read_cached_consensus(const char *flavorname)
- {
+/** Map the file containing the current cached consensus of flavor
+ * <b>flavorname</b> */
+tor_mmap_t *
+networkstatus_map_cached_consensus(const char *flavorname)
+{
   int flav = networkstatus_parse_flavor_name(flavorname);
   if (flav < 0)
     return NULL;
-  return networkstatus_read_cached_consensus_impl(flav, flavorname, 0);
+  return networkstatus_map_cached_consensus_impl(flav, flavorname, 0);
 }
 
 /** Read every cached v3 consensus networkstatus from the disk. */
@@ -259,25 +277,15 @@ router_reload_consensus_networkstatus(void)
   /* FFFF Suppress warnings if cached consensus is bad? */
   for (flav = 0; flav < N_CONSENSUS_FLAVORS; ++flav) {
     const char *flavor = networkstatus_get_flavor_name(flav);
-    char *s = networkstatus_read_cached_consensus_impl(flav, flavor, 0);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor, flags, NULL) < -1) {
-        log_warn(LD_FS, "Couldn't load consensus %s networkstatus from cache",
-                 flavor);
-      }
-      tor_free(s);
-    }
+    char *fname = networkstatus_get_cache_fname(flav, flavor, 0);
+    reload_consensus_from_file(fname, flavor, flags, NULL);
+    tor_free(fname);
 
-    s = networkstatus_read_cached_consensus_impl(flav, flavor, 1);
-    if (s) {
-      if (networkstatus_set_current_consensus(s, flavor,
-                                     flags | NSSET_WAS_WAITING_FOR_CERTS,
-                                     NULL)) {
-        log_info(LD_FS, "Couldn't load unverified consensus %s networkstatus "
-                 "from cache", flavor);
-      }
-      tor_free(s);
-    }
+    fname = networkstatus_get_cache_fname(flav, flavor, 1);
+    reload_consensus_from_file(fname, flavor,
+                               flags | NSSET_WAS_WAITING_FOR_CERTS,
+                               NULL);
+    tor_free(fname);
   }
 
   update_certificate_downloads(time(NULL));
@@ -713,8 +721,8 @@ networkstatus_vote_find_mutable_entry(networkstatus_t *ns, const char *digest)
 
 /** Return the entry in <b>ns</b> for the identity digest <b>digest</b>, or
  * NULL if none was found. */
-const routerstatus_t *
-networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+MOCK_IMPL(const routerstatus_t *,
+networkstatus_vote_find_entry,(networkstatus_t *ns, const char *digest))
 {
   return networkstatus_vote_find_mutable_entry(ns, digest);
 }
@@ -1156,7 +1164,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
       }
     }
 
-    if (directory_fetches_dir_info_early(options)) {
+    if (dirclient_fetches_dir_info_early(options)) {
       /* We want to cache the next one at some point after this one
        * is no longer fresh... */
       start = (time_t)(c->fresh_until + min_sec_before_caching);
@@ -1178,7 +1186,7 @@ update_consensus_networkstatus_fetch_time_impl(time_t now, int flav)
 
       /* If we're a bridge user, make use of the numbers we just computed
        * to choose the rest of the interval *after* them. */
-      if (directory_fetches_dir_info_later(options)) {
+      if (dirclient_fetches_dir_info_later(options)) {
         /* Give all the *clients* enough time to download the consensus. */
         start = (time_t)(start + dl_interval + min_sec_before_caching);
         /* But try to get it before ours actually expires. */
@@ -1377,7 +1385,7 @@ networkstatus_get_dl_status_by_flavor_running,(consensus_flavor_t flavor))
 }
 
 /** Return the most recent consensus that we have downloaded, or NULL if we
- * don't have one. */
+ * don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus,(void))
 {
@@ -1388,7 +1396,7 @@ networkstatus_get_latest_consensus,(void))
 }
 
 /** Return the latest consensus we have whose flavor matches <b>f</b>, or NULL
- * if we don't have one. */
+ * if we don't have one. May return future or expired consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_latest_consensus_by_flavor,(consensus_flavor_t f))
 {
@@ -1422,10 +1430,11 @@ networkstatus_is_live(const networkstatus_t *ns, time_t now)
   return (ns->valid_after <= now && now <= ns->valid_until);
 }
 
-/** Determine if <b>consensus</b> is valid or expired recently enough that
- * we can still use it.
+/** Determine if <b>consensus</b> is valid, or expired recently enough, or not
+ * too far in the future, so that we can still use it.
  *
- * Return 1 if the consensus is reasonably live, or 0 if it is too old.
+ * Return 1 if the consensus is reasonably live, or 0 if it is too old or
+ * too new.
  */
 int
 networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
@@ -1434,29 +1443,42 @@ networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
   if (BUG(!consensus))
     return 0;
 
-  return networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
+  return networkstatus_valid_after_is_reasonably_live(consensus->valid_after,
+                                                      now) &&
+         networkstatus_valid_until_is_reasonably_live(consensus->valid_until,
                                                       now);
 }
 
+#define REASONABLY_LIVE_TIME (24*60*60)
+
+/** As networkstatus_consensus_reasonably_live, but takes a valid_after
+ * time, and checks to see if it is in the past, or not too far in the future.
+ */
+int
+networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                             time_t now)
+{
+  return (now >= valid_after - REASONABLY_LIVE_TIME);
+}
+
 /** As networkstatus_consensus_reasonably_live, but takes a valid_until
- * time rather than an entire consensus. */
+ * time, and checks to see if it is in the future, or not too far in the past.
+ */
 int
 networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                              time_t now)
 {
-#define REASONABLY_LIVE_TIME (24*60*60)
   return (now <= valid_until + REASONABLY_LIVE_TIME);
 }
 
 /** As networkstatus_get_live_consensus(), but is way more tolerant of expired
- * consensuses. */
+ *  and future consensuses. */
 MOCK_IMPL(networkstatus_t *,
 networkstatus_get_reasonably_live_consensus,(time_t now, int flavor))
 {
   networkstatus_t *consensus =
     networkstatus_get_latest_consensus_by_flavor(flavor);
   if (consensus &&
-      consensus->valid_after <= now &&
       networkstatus_consensus_reasonably_live(consensus, now))
     return consensus;
   else
@@ -1517,7 +1539,7 @@ networkstatus_consensus_can_use_extra_fallbacks,(const or_options_t *options))
              >= smartlist_len(router_get_trusted_dir_servers()));
   /* If we don't fetch from the authorities, and we have additional mirrors,
    * we can use them. */
-  return (!directory_fetches_from_authorities(options)
+  return (!dirclient_fetches_from_authorities(options)
           && (smartlist_len(router_get_fallback_dir_servers())
               > smartlist_len(router_get_trusted_dir_servers())));
 }
@@ -1557,36 +1579,16 @@ networkstatus_consensus_is_already_downloading(const char *resource)
   return answer;
 }
 
-/* Does the current, reasonably live consensus have IPv6 addresses?
- * Returns 1 if there is a reasonably live consensus and its consensus method
- * includes IPv6 addresses in the consensus.
- * Otherwise, if there is no consensus, or the method does not include IPv6
- * addresses, returns 0. */
-int
-networkstatus_consensus_has_ipv6(const or_options_t* options)
-{
-  const networkstatus_t *cons = networkstatus_get_reasonably_live_consensus(
-                                                    approx_time(),
-                                                    usable_consensus_flavor());
-
-  /* If we have no consensus, we have no IPv6 in it */
-  if (!cons) {
-    return 0;
-  }
-
-  /* Different flavours of consensus gained IPv6 at different times */
-  if (we_use_microdescriptors_for_circuits(options)) {
-    return
-       cons->consensus_method >= MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
-  } else {
-    return 1;
-  }
-}
-
-/** Given two router status entries for the same router identity, return 1 if
- * if the contents have changed between them. Otherwise, return 0. */
-static int
-routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b)
+/** Given two router status entries for the same router identity, return 1
+ * if the contents have changed between them. Otherwise, return 0.
+ * It only checks for fields that are output by control port.
+ * This should be kept in sync with the struct routerstatus_t
+ * and the printing function routerstatus_format_entry in
+ * NS_CONTROL_PORT mode.
+ **/
+STATIC int
+routerstatus_has_visibly_changed(const routerstatus_t *a,
+                                 const routerstatus_t *b)
 {
   tor_assert(tor_memeq(a->identity_digest, b->identity_digest, DIGEST_LEN));
 
@@ -1605,9 +1607,14 @@ routerstatus_has_changed(const routerstatus_t *a, const routerstatus_t *b)
          a->is_valid != b->is_valid ||
          a->is_possible_guard != b->is_possible_guard ||
          a->is_bad_exit != b->is_bad_exit ||
-         a->is_hs_dir != b->is_hs_dir;
-  // XXXX this function needs a huge refactoring; it has gotten out
-  // XXXX of sync with routerstatus_t, and it will do so again.
+         a->is_hs_dir != b->is_hs_dir ||
+         a->is_staledesc != b->is_staledesc ||
+         a->has_bandwidth != b->has_bandwidth ||
+         a->published_on != b->published_on ||
+         a->ipv6_orport != b->ipv6_orport ||
+         a->is_v2_dir != b->is_v2_dir ||
+         a->bandwidth_kb != b->bandwidth_kb ||
+         tor_addr_compare(&a->ipv6_addr, &b->ipv6_addr, CMP_EXACT);
 }
 
 /** Notify controllers of any router status entries that changed between
@@ -1639,7 +1646,7 @@ notify_control_networkstatus_changed(const networkstatus_t *old_c,
                      tor_memcmp(rs_old->identity_digest,
                             rs_new->identity_digest, DIGEST_LEN),
                      smartlist_add(changed, (void*) rs_new)) {
-    if (routerstatus_has_changed(rs_old, rs_new))
+    if (routerstatus_has_visibly_changed(rs_old, rs_new))
       smartlist_add(changed, (void*)rs_new);
   } SMARTLIST_FOREACH_JOIN_END(rs_old, rs_new);
 
@@ -1655,6 +1662,7 @@ notify_before_networkstatus_changes(const networkstatus_t *old_c,
   notify_control_networkstatus_changed(old_c, new_c);
   dos_consensus_has_changed(new_c);
   relay_consensus_has_changed(new_c);
+  hs_dos_consensus_has_changed(new_c);
 }
 
 /* Called after a new consensus has been put in the global state. It is safe
@@ -1725,6 +1733,44 @@ networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
 #endif /* defined(TOR_UNIT_TESTS) */
 
 /**
+ * Helper: Read the current consensus of type <b>flavor</b> from
+ * <b>fname</b>.  Flags and return values are as for
+ * networkstatus_set_current_consensus().
+ **/
+static int
+reload_consensus_from_file(const char *fname,
+                           const char *flavor,
+                           unsigned flags,
+                           const char *source_dir)
+{
+  tor_mmap_t *map = tor_mmap_file(fname);
+  if (!map)
+    return 0;
+
+  int rv = networkstatus_set_current_consensus(map->data, map->size,
+                                               flavor, flags, source_dir);
+#ifdef _WIN32
+  if (rv < 0 && tor_memstr(map->data, map->size, "\r\n")) {
+    log_notice(LD_GENERAL, "Looks like the above failures are probably "
+               "because of a CRLF in consensus file %s; falling back to "
+               "read_file_to_string. Nothing to worry about: this file "
+               "was probably saved by an earlier version of Tor.",
+               escaped(fname));
+    char *content = read_file_to_str(fname, RFTS_IGNORE_MISSING, NULL);
+    rv = networkstatus_set_current_consensus(content, strlen(content),
+                                             flavor, flags, source_dir);
+    tor_free(content);
+  }
+#endif /* defined(_WIN32) */
+  if (rv < -1) {
+    log_warn(LD_GENERAL, "Couldn't set consensus from cache file %s",
+             escaped(fname));
+  }
+  tor_munmap_file(map);
+  return rv;
+}
+
+/**
  * Helper for handle_missing_protocol_warning: handles either the
  * client case (if <b>is_client</b> is set) or the server case otherwise.
  */
@@ -1841,6 +1887,7 @@ warn_early_consensus(const networkstatus_t *c, const char *flavor,
  */
 int
 networkstatus_set_current_consensus(const char *consensus,
+                                    size_t consensus_len,
                                     const char *flavor,
                                     unsigned flags,
                                     const char *source_dir)
@@ -1869,7 +1916,9 @@ networkstatus_set_current_consensus(const char *consensus,
   }
 
   /* Make sure it's parseable. */
-  c = networkstatus_parse_vote_from_string(consensus, NULL, NS_TYPE_CONSENSUS);
+  c = networkstatus_parse_vote_from_string(consensus,
+                                           consensus_len,
+                                           NULL, NS_TYPE_CONSENSUS);
   if (!c) {
     log_warn(LD_DIR, "Unable to parse networkstatus consensus");
     result = -2;
@@ -1957,14 +2006,12 @@ networkstatus_set_current_consensus(const char *consensus,
           c->valid_after > current_valid_after) {
         waiting = &consensus_waiting_for_certs[flav];
         networkstatus_vote_free(waiting->consensus);
-        tor_free(waiting->body);
         waiting->consensus = c;
         free_consensus = 0;
-        waiting->body = tor_strdup(consensus);
         waiting->set_at = now;
         waiting->dl_failed = 0;
         if (!from_cache) {
-          write_str_to_file(unverified_fname, consensus, 0);
+          write_bytes_to_file(unverified_fname, consensus, consensus_len, 1);
         }
         if (dl_certs)
           authority_certs_fetch_missing(c, now, source_dir);
@@ -1976,9 +2023,9 @@ networkstatus_set_current_consensus(const char *consensus,
          * latest consensus. */
         if (was_waiting_for_certs && from_cache)
           if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
           }
       }
       goto done;
@@ -1991,9 +2038,9 @@ networkstatus_set_current_consensus(const char *consensus,
       }
       if (was_waiting_for_certs && (r < -1) && from_cache) {
         if (unlink(unverified_fname) != 0) {
-            log_warn(LD_FS,
-                     "Failed to unlink %s: %s",
-                     unverified_fname, strerror(errno));
+            log_debug(LD_FS,
+                      "Failed to unlink %s: %s",
+                      unverified_fname, strerror(errno));
         }
       }
       goto done;
@@ -2055,16 +2102,12 @@ networkstatus_set_current_consensus(const char *consensus,
       waiting->consensus->valid_after <= c->valid_after) {
     networkstatus_vote_free(waiting->consensus);
     waiting->consensus = NULL;
-    if (consensus != waiting->body)
-      tor_free(waiting->body);
-    else
-      waiting->body = NULL;
     waiting->set_at = 0;
     waiting->dl_failed = 0;
     if (unlink(unverified_fname) != 0) {
-      log_warn(LD_FS,
-               "Failed to unlink %s: %s",
-               unverified_fname, strerror(errno));
+      log_debug(LD_FS,
+                "Failed to unlink %s: %s",
+                unverified_fname, strerror(errno));
     }
   }
 
@@ -2077,12 +2120,11 @@ networkstatus_set_current_consensus(const char *consensus,
      * the first thing we need to do is recalculate the voting schedule static
      * object so we can use the timings in there needed by some subsystems
      * such as hidden service and shared random. */
-    voting_schedule_recalculate_timing(options, now);
+    dirauth_sched_recalculate_timing(options, now);
     reschedule_dirvote(options);
 
     nodelist_set_consensus(c);
 
-    /* XXXXNM Microdescs: needs a non-ns variant. ???? NM*/
     update_consensus_networkstatus_fetch_time(now);
 
     /* Change the cell EWMA settings */
@@ -2095,6 +2137,7 @@ networkstatus_set_current_consensus(const char *consensus,
     circuit_build_times_new_consensus_params(
                                get_circuit_build_times_mutable(), c);
     channelpadding_new_consensus_params(c);
+    circpad_new_consensus_params(c);
   }
 
   /* Reset the failure count only if this consensus is actually valid. */
@@ -2108,22 +2151,23 @@ networkstatus_set_current_consensus(const char *consensus,
   if (we_want_to_fetch_flavor(options, flav)) {
     if (dir_server_mode(get_options())) {
       dirserv_set_cached_consensus_networkstatus(consensus,
+                                                 consensus_len,
                                                  flavor,
                                                  &c->digests,
                                                  c->digest_sha3_as_signed,
                                                  c->valid_after);
 
-      consdiffmgr_add_consensus(consensus, c);
+      consdiffmgr_add_consensus(consensus, consensus_len, c);
     }
   }
 
   if (!from_cache) {
-    write_str_to_file(consensus_fname, consensus, 0);
+    write_bytes_to_file(consensus_fname, consensus, consensus_len, 1);
   }
 
   warn_early_consensus(c, flavor, now);
 
-  /* We got a new consesus. Reset our md fetch fail cache */
+  /* We got a new consensus. Reset our md fetch fail cache */
   microdesc_reset_outdated_dirservers_list();
 
   router_dir_info_changed();
@@ -2154,14 +2198,10 @@ networkstatus_note_certs_arrived(const char *source_dir)
     if (!waiting->consensus)
       continue;
     if (networkstatus_check_consensus_signature(waiting->consensus, 0)>=0) {
-      char *waiting_body = waiting->body;
-      if (!networkstatus_set_current_consensus(
-                                 waiting_body,
-                                 flavor_name,
-                                 NSSET_WAS_WAITING_FOR_CERTS,
-                                 source_dir)) {
-        tor_free(waiting_body);
-      }
+      char *fname = networkstatus_get_cache_fname(i, flavor_name, 1);
+      reload_consensus_from_file(fname, flavor_name,
+                                 NSSET_WAS_WAITING_FOR_CERTS, source_dir);
+      tor_free(fname);
     }
   }
 }
@@ -2311,10 +2351,52 @@ char *
 networkstatus_getinfo_helper_single(const routerstatus_t *rs)
 {
   return routerstatus_format_entry(rs, NULL, NULL, NS_CONTROL_PORT,
-                                   ROUTERSTATUS_FORMAT_NO_CONSENSUS_METHOD,
                                    NULL);
 }
 
+/**
+ * Extract status information from <b>ri</b> and from other authority
+ * functions and store it in <b>rs</b>. <b>rs</b> is zeroed out before it is
+ * set.
+ *
+ * We assume that node-\>is_running has already been set, e.g. by
+ *   dirserv_set_router_is_running(ri, now);
+ */
+void
+set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                 const node_t *node,
+                                 const routerinfo_t *ri)
+{
+  memset(rs, 0, sizeof(routerstatus_t));
+
+  rs->is_authority =
+    router_digest_is_trusted_dir(ri->cache_info.identity_digest);
+
+  /* Set by compute_performance_thresholds or from consensus */
+  rs->is_exit = node->is_exit;
+  rs->is_stable = node->is_stable;
+  rs->is_fast = node->is_fast;
+  rs->is_flagged_running = node->is_running;
+  rs->is_valid = node->is_valid;
+  rs->is_possible_guard = node->is_possible_guard;
+  rs->is_bad_exit = node->is_bad_exit;
+  rs->is_hs_dir = node->is_hs_dir;
+  rs->is_named = rs->is_unnamed = 0;
+
+  rs->published_on = ri->cache_info.published_on;
+  memcpy(rs->identity_digest, node->identity, DIGEST_LEN);
+  memcpy(rs->descriptor_digest, ri->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  rs->addr = ri->addr;
+  strlcpy(rs->nickname, ri->nickname, sizeof(rs->nickname));
+  rs->or_port = ri->or_port;
+  rs->dir_port = ri->dir_port;
+  rs->is_v2_dir = ri->supports_tunnelled_dir_requests;
+
+  tor_addr_copy(&rs->ipv6_addr, &ri->ipv6_addr);
+  rs->ipv6_orport = ri->ipv6_orport;
+}
+
 /** Alloc and return a string describing routerstatuses for the most
  * recent info of each router we know about that is of purpose
  * <b>purpose_string</b>. Return NULL if unrecognized purpose.
@@ -2331,7 +2413,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   smartlist_t *statuses;
   const uint8_t purpose = router_purpose_from_string(purpose_string);
   routerstatus_t rs;
-  const int bridge_auth = authdir_mode_bridge(get_options());
 
   if (purpose == ROUTER_PURPOSE_UNKNOWN) {
     log_info(LD_DIR, "Unrecognized purpose '%s' when listing router statuses.",
@@ -2348,11 +2429,7 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
       continue;
     if (ri->purpose != purpose)
       continue;
-    /* TODO: modifying the running flag in a getinfo is a bad idea */
-    if (bridge_auth && ri->purpose == ROUTER_PURPOSE_BRIDGE)
-      dirserv_set_router_is_running(ri, now);
-    /* then generate and write out status lines for each of them */
-    set_routerstatus_from_routerinfo(&rs, node, ri, now, 0);
+    set_routerstatus_from_routerinfo(&rs, node, ri);
     smartlist_add(statuses, networkstatus_getinfo_helper_single(&rs));
   } SMARTLIST_FOREACH_END(ri);
 
@@ -2362,41 +2439,6 @@ networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now)
   return answer;
 }
 
-/** Write out router status entries for all our bridge descriptors. */
-void
-networkstatus_dump_bridge_status_to_file(time_t now)
-{
-  char *status = networkstatus_getinfo_by_purpose("bridge", now);
-  char *fname = NULL;
-  char *thresholds = NULL;
-  char *published_thresholds_and_status = NULL;
-  char published[ISO_TIME_LEN+1];
-  const routerinfo_t *me = router_get_my_routerinfo();
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *fingerprint_line = NULL;
-
-  if (me && crypto_pk_get_fingerprint(me->identity_pkey,
-                                      fingerprint, 0) >= 0) {
-    tor_asprintf(&fingerprint_line, "fingerprint %s\n", fingerprint);
-  } else {
-    log_warn(LD_BUG, "Error computing fingerprint for bridge status.");
-  }
-  format_iso_time(published, now);
-  dirserv_compute_bridge_flag_thresholds();
-  thresholds = dirserv_get_flag_thresholds_line();
-  tor_asprintf(&published_thresholds_and_status,
-               "published %s\nflag-thresholds %s\n%s%s",
-               published, thresholds, fingerprint_line ? fingerprint_line : "",
-               status);
-  fname = get_datadir_fname("networkstatus-bridges");
-  write_str_to_file(fname,published_thresholds_and_status,0);
-  tor_free(thresholds);
-  tor_free(published_thresholds_and_status);
-  tor_free(fname);
-  tor_free(status);
-  tor_free(fingerprint_line);
-}
-
 /* DOCDOC get_net_param_from_list */
 static int32_t
 get_net_param_from_list(smartlist_t *net_params, const char *param_name,
@@ -2668,6 +2710,9 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
   const char *required, *recommended;
   char *missing = NULL;
 
+  const bool consensus_postdates_this_release =
+    ns->valid_after >= tor_get_approx_release_date();
+
   tor_assert(warning_out);
 
   if (client_mode) {
@@ -2685,7 +2730,7 @@ networkstatus_check_required_protocols(const networkstatus_t *ns,
                  "%s on the Tor network. The missing protocols are: %s",
                  func, missing);
     tor_free(missing);
-    return 1;
+    return consensus_postdates_this_release ? 1 : 0;
   }
 
   if (! protover_all_supported(recommended, &missing)) {
@@ -2718,6 +2763,49 @@ networkstatus_free_all(void)
       networkstatus_vote_free(waiting->consensus);
       waiting->consensus = NULL;
     }
-    tor_free(waiting->body);
   }
 }
+
+/** Return the start of the next interval of size <b>interval</b> (in
+ * seconds) after <b>now</b>, plus <b>offset</b>. Midnight always
+ * starts a fresh interval, and if the last interval of a day would be
+ * truncated to less than half its size, it is rolled into the
+ * previous interval. */
+time_t
+voting_sched_get_start_of_interval_after(time_t now, int interval,
+                                           int offset)
+{
+  struct tm tm;
+  time_t midnight_today=0;
+  time_t midnight_tomorrow;
+  time_t next;
+
+  tor_gmtime_r(&now, &tm);
+  tm.tm_hour = 0;
+  tm.tm_min = 0;
+  tm.tm_sec = 0;
+
+  if (tor_timegm(&tm, &midnight_today) < 0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Ran into an invalid time when trying to find midnight.");
+    // LCOV_EXCL_STOP
+  }
+  midnight_tomorrow = midnight_today + (24*60*60);
+
+  next = midnight_today + ((now-midnight_today)/interval + 1)*interval;
+
+  /* Intervals never cross midnight. */
+  if (next > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  /* If the interval would only last half as long as it's supposed to, then
+   * skip over to the next day. */
+  if (next + interval/2 > midnight_tomorrow)
+    next = midnight_tomorrow;
+
+  next += offset;
+  if (next - interval > now)
+    next -= interval;
+
+  return next;
+}
diff --git a/src/feature/nodelist/networkstatus.h b/src/feature/nodelist/networkstatus.h
index 9e7b0f1bb0..ce050aeadc 100644
--- a/src/feature/nodelist/networkstatus.h
+++ b/src/feature/nodelist/networkstatus.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,10 @@
 
 void networkstatus_reset_warnings(void);
 void networkstatus_reset_download_failures(void);
-char *networkstatus_read_cached_consensus(const char *flavorname);
+MOCK_DECL(char *,networkstatus_get_cache_fname,(int flav,
+                                                const char *flavorname,
+                                                int unverified_consensus));
+tor_mmap_t *networkstatus_map_cached_consensus(const char *flavorname);
 int router_reload_consensus_networkstatus(void);
 void routerstatus_free_(routerstatus_t *rs);
 #define routerstatus_free(rs) \
@@ -40,8 +43,9 @@ int compare_digest_to_routerstatus_entry(const void *_key,
                                          const void **_member);
 int compare_digest_to_vote_routerstatus_entry(const void *_key,
                                               const void **_member);
-const routerstatus_t *networkstatus_vote_find_entry(networkstatus_t *ns,
-                                              const char *digest);
+MOCK_DECL(const routerstatus_t *,networkstatus_vote_find_entry,(
+                                              networkstatus_t *ns,
+                                              const char *digest));
 routerstatus_t *networkstatus_vote_find_mutable_entry(networkstatus_t *ns,
                                               const char *digest);
 int networkstatus_vote_find_entry_idx(networkstatus_t *ns,
@@ -87,6 +91,8 @@ MOCK_DECL(networkstatus_t *, networkstatus_get_live_consensus,(time_t now));
 int networkstatus_is_live(const networkstatus_t *ns, time_t now);
 int networkstatus_consensus_reasonably_live(const networkstatus_t *consensus,
                                             time_t now);
+int networkstatus_valid_after_is_reasonably_live(time_t valid_after,
+                                                 time_t now);
 int networkstatus_valid_until_is_reasonably_live(time_t valid_until,
                                                  time_t now);
 MOCK_DECL(networkstatus_t *,networkstatus_get_reasonably_live_consensus,
@@ -98,7 +104,6 @@ int networkstatus_consensus_can_use_multiple_directories(
 MOCK_DECL(int, networkstatus_consensus_can_use_extra_fallbacks,(
                                                 const or_options_t *options));
 int networkstatus_consensus_is_already_downloading(const char *resource);
-int networkstatus_consensus_has_ipv6(const or_options_t* options);
 
 #define NSSET_FROM_CACHE 1
 #define NSSET_WAS_WAITING_FOR_CERTS 2
@@ -106,6 +111,7 @@ int networkstatus_consensus_has_ipv6(const or_options_t* options);
 #define NSSET_ACCEPT_OBSOLETE 8
 #define NSSET_REQUIRE_FLAVOR 16
 int networkstatus_set_current_consensus(const char *consensus,
+                                        size_t consensus_len,
                                         const char *flavor,
                                         unsigned flags,
                                         const char *source_dir);
@@ -118,7 +124,6 @@ void signed_descs_update_status_from_consensus_networkstatus(
 
 char *networkstatus_getinfo_helper_single(const routerstatus_t *rs);
 char *networkstatus_getinfo_by_purpose(const char *purpose_string, time_t now);
-void networkstatus_dump_bridge_status_to_file(time_t now);
 MOCK_DECL(int32_t, networkstatus_get_param,
           (const networkstatus_t *ns, const char *param_name,
            int32_t default_val, int32_t min_val, int32_t max_val));
@@ -145,6 +150,13 @@ void vote_routerstatus_free_(vote_routerstatus_t *rs);
 #define vote_routerstatus_free(rs) \
   FREE_AND_NULL(vote_routerstatus_t, vote_routerstatus_free_, (rs))
 
+void set_routerstatus_from_routerinfo(routerstatus_t *rs,
+                                      const node_t *node,
+                                      const routerinfo_t *ri);
+time_t voting_sched_get_start_of_interval_after(time_t now,
+                                                  int interval,
+                                                  int offset);
+
 #ifdef NETWORKSTATUS_PRIVATE
 #ifdef TOR_UNIT_TESTS
 STATIC int networkstatus_set_current_consensus_from_ns(networkstatus_t *c,
@@ -154,7 +166,8 @@ STATIC void warn_early_consensus(const networkstatus_t *c, const char *flavor,
 extern networkstatus_t *current_ns_consensus;
 extern networkstatus_t *current_md_consensus;
 #endif /* defined(TOR_UNIT_TESTS) */
+STATIC int routerstatus_has_visibly_changed(const routerstatus_t *a,
+                                    const routerstatus_t *b);
 #endif /* defined(NETWORKSTATUS_PRIVATE) */
 
 #endif /* !defined(TOR_NETWORKSTATUS_H) */
-
diff --git a/src/feature/nodelist/networkstatus_sr_info_st.h b/src/feature/nodelist/networkstatus_sr_info_st.h
index 677d8ed811..04d0dfe8f6 100644
--- a/src/feature/nodelist/networkstatus_sr_info_st.h
+++ b/src/feature/nodelist/networkstatus_sr_info_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file networkstatus_sr_info_st.h
+ * @brief Shared-randomness structure.
+ **/
+
 #ifndef NETWORKSTATUS_SR_INFO_ST_H
 #define NETWORKSTATUS_SR_INFO_ST_H
 
@@ -19,5 +24,4 @@ struct networkstatus_sr_info_t {
   smartlist_t *commits;
 };
 
-#endif
-
+#endif /* !defined(NETWORKSTATUS_SR_INFO_ST_H) */
diff --git a/src/feature/nodelist/networkstatus_st.h b/src/feature/nodelist/networkstatus_st.h
index 6160f12361..021168d3ca 100644
--- a/src/feature/nodelist/networkstatus_st.h
+++ b/src/feature/nodelist/networkstatus_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file networkstatus_st.h
+ * @brief Networkstatus consensus/vote structure.
+ **/
+
 #ifndef NETWORKSTATUS_ST_H
 #define NETWORKSTATUS_ST_H
 
@@ -99,6 +104,9 @@ struct networkstatus_t {
 
   /** List of key=value strings from the headers of the bandwidth list file */
   smartlist_t *bw_file_headers;
+
+  /** A SHA256 digest of the bandwidth file used in a vote. */
+  uint8_t bw_file_digest256[DIGEST256_LEN];
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_ST_H) */
diff --git a/src/feature/nodelist/networkstatus_voter_info_st.h b/src/feature/nodelist/networkstatus_voter_info_st.h
index 4037fcdeca..b4d0b1dd17 100644
--- a/src/feature/nodelist/networkstatus_voter_info_st.h
+++ b/src/feature/nodelist/networkstatus_voter_info_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file networkstatus_voter_info_st.h
+ * @brief Single consensus voter structure.
+ **/
+
 #ifndef NETWORKSTATUS_VOTER_INFO_ST_H
 #define NETWORKSTATUS_VOTER_INFO_ST_H
 
@@ -27,4 +32,4 @@ struct networkstatus_voter_info_t {
   smartlist_t *sigs;
 };
 
-#endif
+#endif /* !defined(NETWORKSTATUS_VOTER_INFO_ST_H) */
diff --git a/src/feature/nodelist/nickname.c b/src/feature/nodelist/nickname.c
index 5378b749ca..c022dd6bc4 100644
--- a/src/feature/nodelist/nickname.c
+++ b/src/feature/nodelist/nickname.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/nodelist/nickname.h b/src/feature/nodelist/nickname.h
index 9bdc6b50e8..11c6416f3a 100644
--- a/src/feature/nodelist/nickname.h
+++ b/src/feature/nodelist/nickname.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,4 +16,4 @@ int is_legal_nickname(const char *s);
 int is_legal_nickname_or_hexdigest(const char *s);
 int is_legal_hexdigest(const char *s);
 
-#endif
+#endif /* !defined(TOR_NICKNAME_H) */
diff --git a/src/feature/nodelist/node_select.c b/src/feature/nodelist/node_select.c
index 7b9e241e5b..e831248413 100644
--- a/src/feature/nodelist/node_select.c
+++ b/src/feature/nodelist/node_select.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,7 @@
 #include "core/or/reasons.h"
 #include "feature/client/entrynodes.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/describe.h"
 #include "feature/nodelist/dirlist.h"
@@ -30,6 +31,7 @@
 #include "feature/nodelist/routerset.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
+#include "lib/container/bitarray.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/math/fp.h"
 
@@ -146,7 +148,7 @@ router_pick_dirserver_generic(smartlist_t *sourcelist,
       try_ip_pref = 0;                                                        \
       goto retry_label;                                                       \
     }                                                                         \
-  STMT_END                                                                    \
+  STMT_END
 
 /* Common retry code for router_pick_directory_server_impl and
  * router_pick_trusteddirserver_impl. Retry without excluding nodes, but with
@@ -321,7 +323,7 @@ router_pick_directory_server_impl(dirinfo_type_t type, int flags,
 
   const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
   const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
-  const int must_have_or = directory_must_use_begindir(options);
+  const int must_have_or = dirclient_must_use_begindir(options);
 
   /* Find all the running dirservers we know about. */
   SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), const node_t *, node) {
@@ -630,6 +632,7 @@ compute_weighted_bandwidths(const smartlist_t *sl,
   }
 
   weight_scale = networkstatus_get_weight_scale_param(NULL);
+  tor_assert(weight_scale >= 1);
 
   if (rule == WEIGHT_FOR_GUARD) {
     Wg = networkstatus_get_bw_weight(NULL, "Wgg", -1);
@@ -871,6 +874,58 @@ routerlist_add_node_and_family(smartlist_t *sl, const routerinfo_t *router)
   nodelist_add_node_and_family(sl, node);
 }
 
+/**
+ * Remove every node_t that appears in <b>excluded</b> from <b>sl</b>.
+ *
+ * Behaves like smartlist_subtract, but uses nodelist_idx values to deliver
+ * linear performance when smartlist_subtract would be quadratic.
+ **/
+static void
+nodelist_subtract(smartlist_t *sl, const smartlist_t *excluded)
+{
+  const smartlist_t *nodelist = nodelist_get_list();
+  const int nodelist_len = smartlist_len(nodelist);
+  bitarray_t *excluded_idx = bitarray_init_zero(nodelist_len);
+
+  /* We haven't used nodelist_idx in this way previously, so I'm going to be
+   * paranoid in this code, and check that nodelist_idx is correct for every
+   * node before we use it.  If we fail, we fall back to smartlist_subtract().
+   */
+
+  /* Set the excluded_idx bit corresponding to every excluded node...
+   */
+  SMARTLIST_FOREACH_BEGIN(excluded, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    bitarray_set(excluded_idx, idx);
+  } SMARTLIST_FOREACH_END(node);
+
+  /* Then remove them from sl.
+   */
+  SMARTLIST_FOREACH_BEGIN(sl, const node_t *, node) {
+    const int idx = node->nodelist_idx;
+    if (BUG(idx < 0) || BUG(idx >= nodelist_len) ||
+        BUG(node != smartlist_get(nodelist, idx))) {
+      goto internal_error;
+    }
+    if (bitarray_is_set(excluded_idx, idx)) {
+      SMARTLIST_DEL_CURRENT(sl, node);
+    }
+  } SMARTLIST_FOREACH_END(node);
+
+  bitarray_free(excluded_idx);
+  return;
+
+ internal_error:
+  log_warn(LD_BUG, "Internal error prevented us from using the fast method "
+           "for subtracting nodelists. Falling back to the quadratic way.");
+  smartlist_subtract(sl, excluded);
+  bitarray_free(excluded_idx);
+}
+
 /** Return a random running node from the nodelist. Never
  * pick a node that is in
  * <b>excludedsmartlist</b>, or which matches <b>excludedset</b>,
@@ -905,6 +960,7 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   const int direct_conn = (flags & CRN_DIRECT_CONN) != 0;
   const int rendezvous_v3 = (flags & CRN_RENDEZVOUS_V3) != 0;
 
+  const smartlist_t *node_list = nodelist_get_list();
   smartlist_t *sl=smartlist_new(),
     *excludednodes=smartlist_new();
   const node_t *choice = NULL;
@@ -915,17 +971,17 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
   rule = weight_for_exit ? WEIGHT_FOR_EXIT :
     (need_guard ? WEIGHT_FOR_GUARD : WEIGHT_FOR_MID);
 
-  SMARTLIST_FOREACH_BEGIN(nodelist_get_list(), node_t *, node) {
+  SMARTLIST_FOREACH_BEGIN(node_list, const node_t *, node) {
     if (node_allows_single_hop_exits(node)) {
       /* Exclude relays that allow single hop exit circuits. This is an
        * obsolete option since 0.2.9.2-alpha and done by default in
        * 0.3.1.0-alpha. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     } else if (rendezvous_v3 &&
                !node_supports_v3_rendezvous_point(node)) {
       /* Exclude relays that do not support to rendezvous for a hidden service
        * version 3. */
-      smartlist_add(excludednodes, node);
+      smartlist_add(excludednodes, (node_t*)node);
     }
   } SMARTLIST_FOREACH_END(node);
 
@@ -942,19 +998,11 @@ router_choose_random_node(smartlist_t *excludedsmartlist,
            "We found %d running nodes.",
             smartlist_len(sl));
 
-  smartlist_subtract(sl,excludednodes);
-  log_debug(LD_CIRC,
-            "We removed %d excludednodes, leaving %d nodes.",
-            smartlist_len(excludednodes),
-            smartlist_len(sl));
-
   if (excludedsmartlist) {
-    smartlist_subtract(sl,excludedsmartlist);
-    log_debug(LD_CIRC,
-              "We removed %d excludedsmartlist, leaving %d nodes.",
-              smartlist_len(excludedsmartlist),
-              smartlist_len(sl));
+    smartlist_add_all(excludednodes, excludedsmartlist);
   }
+  nodelist_subtract(sl, excludednodes);
+
   if (excludedset) {
     routerset_subtract_nodes(sl,excludedset);
     log_debug(LD_CIRC,
@@ -1074,7 +1122,7 @@ router_pick_trusteddirserver_impl(const smartlist_t *sourcelist,
 
   const int skip_or_fw = router_skip_or_reachability(options, try_ip_pref);
   const int skip_dir_fw = router_skip_dir_reachability(options, try_ip_pref);
-  const int must_have_or = directory_must_use_begindir(options);
+  const int must_have_or = dirclient_must_use_begindir(options);
 
   SMARTLIST_FOREACH_BEGIN(sourcelist, const dir_server_t *, d)
     {
diff --git a/src/feature/nodelist/node_select.h b/src/feature/nodelist/node_select.h
index ed7450b92c..2e67f990f6 100644
--- a/src/feature/nodelist/node_select.h
+++ b/src/feature/nodelist/node_select.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -97,6 +97,6 @@ STATIC const routerstatus_t *router_pick_directory_server_impl(
                                            int *n_busy_out);
 STATIC int router_is_already_dir_fetching(const tor_addr_port_t *ap,
                                           int serverdesc, int microdesc);
-#endif
+#endif /* defined(NODE_SELECT_PRIVATE) */
 
-#endif
+#endif /* !defined(TOR_NODE_SELECT_H) */
diff --git a/src/feature/nodelist/node_st.h b/src/feature/nodelist/node_st.h
index 53ffde29e4..b1ec4db202 100644
--- a/src/feature/nodelist/node_st.h
+++ b/src/feature/nodelist/node_st.h
@@ -1,14 +1,20 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file node_st.h
+ * @brief Node information structure.
+ **/
+
 #ifndef NODE_ST_H
 #define NODE_ST_H
 
 #include "feature/hs/hsdir_index_st.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
+#include "ext/ht.h"
 
 /** A node_t represents a Tor router.
  *
@@ -99,4 +105,4 @@ struct node_t {
   struct hsdir_index_t hsdir_index;
 };
 
-#endif
+#endif /* !defined(NODE_ST_H) */
diff --git a/src/feature/nodelist/nodefamily.c b/src/feature/nodelist/nodefamily.c
new file mode 100644
index 0000000000..feaa3730dc
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.c
@@ -0,0 +1,416 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.c
+ * \brief Code to manipulate encoded, reference-counted node families.  We
+ *  use these tricks to save space, since these families would otherwise
+ *  require a large number of tiny allocations.
+ **/
+
+#include "core/or/or.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
+#include "feature/nodelist/nodefamily_st.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/router.h"
+#include "feature/nodelist/routerlist.h"
+
+#include "ht.h"
+#include "siphash.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/defs/digest_sizes.h"
+#include "lib/log/util_bug.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+/**
+ * Allocate and return a blank node family with space to hold <b>n_members</b>
+ * members.
+ */
+static nodefamily_t *
+nodefamily_alloc(int n_members)
+{
+  size_t alloc_len = offsetof(nodefamily_t, family_members) +
+    NODEFAMILY_ARRAY_SIZE(n_members);
+  nodefamily_t *nf = tor_malloc_zero(alloc_len);
+  nf->n_members = n_members;
+  return nf;
+}
+
+/**
+ * Hashtable hash implementation.
+ */
+static inline unsigned int
+nodefamily_hash(const nodefamily_t *nf)
+{
+  return (unsigned) siphash24g(nf->family_members,
+                               NODEFAMILY_ARRAY_SIZE(nf->n_members));
+}
+
+/**
+ * Hashtable equality implementation.
+ */
+static inline unsigned int
+nodefamily_eq(const nodefamily_t *a, const nodefamily_t *b)
+{
+  return (a->n_members == b->n_members) &&
+    fast_memeq(a->family_members, b->family_members,
+               NODEFAMILY_ARRAY_SIZE(a->n_members));
+}
+
+static HT_HEAD(nodefamily_map, nodefamily_t) the_node_families
+  = HT_INITIALIZER();
+
+HT_PROTOTYPE(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             nodefamily_eq);
+HT_GENERATE2(nodefamily_map, nodefamily_t, ht_ent, nodefamily_hash,
+             node_family_eq, 0.6, tor_reallocarray_, tor_free_);
+
+/**
+ * Parse the family declaration in <b>s</b>, returning the canonical
+ * <b>nodefamily_t</b> for its members.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * If NF_WARN_MALFORMED is set in <b>flags</b>, warn about any
+ * elements that we can't parse.  (By default, we log at info.)
+ *
+ * If NF_REJECT_MALFORMED is set in <b>flags</b>, treat any unparseable
+ * elements as an error. (By default, we simply omit them.)
+ **/
+nodefamily_t *
+nodefamily_parse(const char *s, const uint8_t *rsa_id_self,
+                 unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *result = nodefamily_from_members(sl, rsa_id_self, flags, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Canonicalize the family list <b>s</b>, returning a newly allocated string.
+ *
+ * The canonicalization rules are fully specified in dir-spec.txt, but,
+ * briefly: $hexid entries are put in caps, $hexid[=~]foo entries are
+ * truncated, nicknames are put into lowercase, unrecognized entries are left
+ * alone, and everything is sorted.
+ **/
+char *
+nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                        unsigned flags)
+{
+  smartlist_t *sl = smartlist_new();
+  smartlist_t *result_members = smartlist_new();
+  smartlist_split_string(sl, s, NULL, SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  nodefamily_t *nf = nodefamily_from_members(sl, rsa_id_self, flags,
+                                             result_members);
+
+  char *formatted = nodefamily_format(nf);
+  smartlist_split_string(result_members, formatted, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  smartlist_sort_strings(result_members);
+  char *combined = smartlist_join_strings(result_members, " ", 0, NULL);
+
+  nodefamily_free(nf);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  SMARTLIST_FOREACH(result_members, char *, cp, tor_free(cp));
+  smartlist_free(result_members);
+  tor_free(formatted);
+
+  return combined;
+}
+
+/**
+ * qsort helper for encoded nodefamily elements.
+ **/
+static int
+compare_members(const void *a, const void *b)
+{
+  return fast_memcmp(a, b, NODEFAMILY_MEMBER_LEN);
+}
+
+/**
+ * Parse the member strings in <b>members</b>, returning their canonical
+ * <b>nodefamily_t</b>.  Return NULL on error.
+ *
+ * If <b>rsa_id_self</b> is provided, it is a DIGEST_LEN-byte digest
+ * for the router that declared this family: insert it into the
+ * family declaration if it is not there already.
+ *
+ * The <b>flags</b> element is interpreted as in nodefamily_parse().
+ *
+ * If <b>unrecognized</b> is provided, fill it copies of any unrecognized
+ * members.  (Note that malformed $hexids are not considered unrecognized.)
+ **/
+nodefamily_t *
+nodefamily_from_members(const smartlist_t *members,
+                        const uint8_t *rsa_id_self,
+                        unsigned flags,
+                        smartlist_t *unrecognized_out)
+{
+  const int n_self = rsa_id_self ? 1 : 0;
+  int n_bad_elements = 0;
+  int n_members = smartlist_len(members) + n_self;
+  nodefamily_t *tmp = nodefamily_alloc(n_members);
+  uint8_t *ptr = NODEFAMILY_MEMBER_PTR(tmp, 0);
+
+  SMARTLIST_FOREACH_BEGIN(members, const char *, cp) {
+    bool bad_element = true;
+    if (is_legal_nickname(cp)) {
+      ptr[0] = NODEFAMILY_BY_NICKNAME;
+      tor_assert(strlen(cp) < DIGEST_LEN); // guaranteed by is_legal_nickname
+      memcpy(ptr+1, cp, strlen(cp));
+      tor_strlower((char*) ptr+1);
+      bad_element = false;
+    } else if (is_legal_hexdigest(cp)) {
+      char digest_buf[DIGEST_LEN];
+      char nn_buf[MAX_NICKNAME_LEN+1];
+      char nn_char=0;
+      if (hex_digest_nickname_decode(cp, digest_buf, &nn_char, nn_buf)==0) {
+        bad_element = false;
+        ptr[0] = NODEFAMILY_BY_RSA_ID;
+        memcpy(ptr+1, digest_buf, DIGEST_LEN);
+      }
+    } else {
+      if (unrecognized_out)
+        smartlist_add_strdup(unrecognized_out, cp);
+    }
+
+    if (bad_element) {
+      const int severity = (flags & NF_WARN_MALFORMED) ? LOG_WARN : LOG_INFO;
+      log_fn(severity, LD_GENERAL,
+             "Bad element %s while parsing a node family.",
+             escaped(cp));
+      ++n_bad_elements;
+    } else {
+      ptr += NODEFAMILY_MEMBER_LEN;
+    }
+  } SMARTLIST_FOREACH_END(cp);
+
+  if (n_bad_elements && (flags & NF_REJECT_MALFORMED))
+    goto err;
+
+  if (rsa_id_self) {
+    /* Add self. */
+    ptr[0] = NODEFAMILY_BY_RSA_ID;
+    memcpy(ptr+1, rsa_id_self, DIGEST_LEN);
+  }
+
+  n_members -= n_bad_elements;
+
+  /* Sort tmp into canonical order. */
+  qsort(tmp->family_members, n_members, NODEFAMILY_MEMBER_LEN,
+        compare_members);
+
+  /* Remove duplicates. */
+  int i;
+  for (i = 0; i < n_members-1; ++i) {
+    uint8_t *thisptr = NODEFAMILY_MEMBER_PTR(tmp, i);
+    uint8_t *nextptr = NODEFAMILY_MEMBER_PTR(tmp, i+1);
+    if (fast_memeq(thisptr, nextptr, NODEFAMILY_MEMBER_LEN)) {
+      memmove(thisptr, nextptr, (n_members-i-1)*NODEFAMILY_MEMBER_LEN);
+      --n_members;
+      --i;
+    }
+  }
+  int n_members_alloc = tmp->n_members;
+  tmp->n_members = n_members;
+
+  /* See if we already allocated this family. */
+  nodefamily_t *found = HT_FIND(nodefamily_map, &the_node_families, tmp);
+  if (found) {
+    /* If we did, great: incref it and return it. */
+    ++found->refcnt;
+    tor_free(tmp);
+    return found;
+  } else {
+    /* If not, insert it into the hashtable. */
+    if (n_members_alloc != n_members) {
+      /* Compact the family if needed */
+      nodefamily_t *tmp2 = nodefamily_alloc(n_members);
+      memcpy(tmp2->family_members, tmp->family_members,
+             n_members * NODEFAMILY_MEMBER_LEN);
+      tor_free(tmp);
+      tmp = tmp2;
+    }
+
+    tmp->refcnt = 1;
+    HT_INSERT(nodefamily_map, &the_node_families, tmp);
+    return tmp;
+  }
+
+ err:
+  tor_free(tmp);
+  return NULL;
+}
+
+/**
+ * Drop our reference to <b>family</b>, freeing it if there are no more
+ * references.
+ */
+void
+nodefamily_free_(nodefamily_t *family)
+{
+  if (family == NULL)
+    return;
+
+  --family->refcnt;
+
+  if (family->refcnt == 0) {
+    HT_REMOVE(nodefamily_map, &the_node_families, family);
+    tor_free(family);
+  }
+}
+
+/**
+ * Return true iff <b>family</b> contains the SHA1 RSA1024 identity
+ * <b>rsa_id</b>.
+ */
+bool
+nodefamily_contains_rsa_id(const nodefamily_t *family,
+                           const uint8_t *rsa_id)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    if (ptr[0] == NODEFAMILY_BY_RSA_ID &&
+        fast_memeq(ptr+1, rsa_id, DIGEST_LEN)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff <b>family</b> contains the nickname <b>name</b>.
+ */
+bool
+nodefamily_contains_nickname(const nodefamily_t *family,
+                             const char *name)
+{
+  if (family == NULL)
+    return false;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    // note that the strcasecmp() is safe because there is always at least one
+    // NUL in the encoded nickname, because all legal nicknames are less than
+    // DIGEST_LEN bytes long.
+    if (ptr[0] == NODEFAMILY_BY_NICKNAME && !strcasecmp((char*)ptr+1, name)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true if <b>family</b> contains the nickname or the RSA ID for
+ * <b>node</b>
+ **/
+bool
+nodefamily_contains_node(const nodefamily_t *family,
+                         const node_t *node)
+{
+  return
+    nodefamily_contains_nickname(family, node_get_nickname(node))
+    ||
+    nodefamily_contains_rsa_id(family, node_get_rsa_id_digest(node));
+}
+
+/**
+ * Look up every entry in <b>family</b>, and add add the corresponding
+ * node_t to <b>out</b>.
+ **/
+void
+nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                  smartlist_t *out)
+{
+  if (!family)
+    return;
+
+  unsigned i;
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    const node_t *node = NULL;
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        node = node_get_by_nickname((char*)ptr+1, NNF_NO_WARN_UNNAMED);
+        break;
+      case NODEFAMILY_BY_RSA_ID:
+        node = node_get_by_id((char*)ptr+1);
+        break;
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+    if (node)
+      smartlist_add(out, (void *)node);
+  }
+}
+
+/**
+ * Encode <b>family</b> as a space-separated string.
+ */
+char *
+nodefamily_format(const nodefamily_t *family)
+{
+  if (!family)
+    return tor_strdup("");
+
+  unsigned i;
+  smartlist_t *sl = smartlist_new();
+  for (i = 0; i < family->n_members; ++i) {
+    const uint8_t *ptr = NODEFAMILY_MEMBER_PTR(family, i);
+    switch (ptr[0]) {
+      case NODEFAMILY_BY_NICKNAME:
+        smartlist_add_strdup(sl, (char*)ptr+1);
+        break;
+      case NODEFAMILY_BY_RSA_ID: {
+        char buf[HEX_DIGEST_LEN+2];
+        buf[0]='$';
+        base16_encode(buf+1, sizeof(buf)-1, (char*)ptr+1, DIGEST_LEN);
+        tor_strupper(buf);
+        smartlist_add_strdup(sl, buf);
+        break;
+      }
+      default:
+        /* LCOV_EXCL_START */
+        tor_assert_nonfatal_unreached();
+        break;
+        /* LCOV_EXCL_STOP */
+    }
+  }
+
+  char *result = smartlist_join_strings(sl, " ", 0, NULL);
+  SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));
+  smartlist_free(sl);
+  return result;
+}
+
+/**
+ * Free all storage held in the nodefamily map.
+ **/
+void
+nodefamily_free_all(void)
+{
+  HT_CLEAR(nodefamily_map, &the_node_families);
+}
diff --git a/src/feature/nodelist/nodefamily.h b/src/feature/nodelist/nodefamily.h
new file mode 100644
index 0000000000..16e161ba82
--- /dev/null
+++ b/src/feature/nodelist/nodefamily.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file nodefamily.h
+ * \brief Header file for nodefamily.c.
+ **/
+
+#ifndef TOR_NODEFAMILY_H
+#define TOR_NODEFAMILY_H
+
+#include "lib/malloc/malloc.h"
+#include <stdbool.h>
+
+typedef struct nodefamily_t nodefamily_t;
+struct node_t;
+struct smartlist_t;
+
+#define NF_WARN_MALFORMED    (1u<<0)
+#define NF_REJECT_MALFORMED  (1u<<1)
+
+nodefamily_t *nodefamily_parse(const char *s,
+                               const uint8_t *rsa_id_self,
+                               unsigned flags);
+nodefamily_t *nodefamily_from_members(const struct smartlist_t *members,
+                                      const uint8_t *rsa_id_self,
+                                      unsigned flags,
+                                      smartlist_t *unrecognized_out);
+void nodefamily_free_(nodefamily_t *family);
+#define nodefamily_free(family) \
+  FREE_AND_NULL(nodefamily_t, nodefamily_free_, (family))
+
+bool nodefamily_contains_rsa_id(const nodefamily_t *family,
+                                const uint8_t *rsa_id);
+bool nodefamily_contains_nickname(const nodefamily_t *family,
+                                  const char *name);
+bool nodefamily_contains_node(const nodefamily_t *family,
+                              const struct node_t *node);
+void nodefamily_add_nodes_to_smartlist(const nodefamily_t *family,
+                                       struct smartlist_t *out);
+char *nodefamily_format(const nodefamily_t *family);
+char *nodefamily_canonicalize(const char *s, const uint8_t *rsa_id_self,
+                              unsigned flags);
+
+void nodefamily_free_all(void);
+
+#endif /* !defined(TOR_NODEFAMILY_H) */
diff --git a/src/feature/nodelist/nodefamily_st.h b/src/feature/nodelist/nodefamily_st.h
new file mode 100644
index 0000000000..c581c917a9
--- /dev/null
+++ b/src/feature/nodelist/nodefamily_st.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file nodefamily_st.h
+ * @brief Compact node-family structure
+ **/
+
+#ifndef TOR_NODEFAMILY_ST_H
+#define TOR_NODEFAMILY_ST_H
+
+#include "orconfig.h"
+#include "ht.h"
+
+struct nodefamily_t {
+  /** Entry for this nodefamily_t within the hashtable. */
+  HT_ENTRY(nodefamily_t) ht_ent;
+  /** Reference count.  (The hashtable is not treated as a reference */
+  uint32_t refcnt;
+  /** Number of items encoded in <b>family_members</b>. */
+  uint32_t n_members;
+  /* A byte-array encoding the members of this family.  We encode each member
+   * as one byte to indicate whether it's a nickname or a fingerprint, plus
+   * DIGEST_LEN bytes of data.  The entries are lexically sorted.
+   */
+  uint8_t family_members[FLEXIBLE_ARRAY_MEMBER];
+};
+
+#define NODEFAMILY_MEMBER_LEN (1+DIGEST_LEN)
+
+/** Tag byte, indicates that the following bytes are a RSA1024 SHA1 ID.
+ */
+#define NODEFAMILY_BY_RSA_ID 0
+/** Tag byte, indicates that the following bytes are a NUL-padded nickname.
+ */
+#define NODEFAMILY_BY_NICKNAME 1
+
+/**
+ * Number of bytes to allocate in the array for a nodefamily_t with N members.
+ **/
+#define NODEFAMILY_ARRAY_SIZE(n) \
+  ((n) * NODEFAMILY_MEMBER_LEN)
+
+/**
+ * Pointer to the i'th member of <b>nf</b>, as encoded.
+ */
+#define NODEFAMILY_MEMBER_PTR(nf, i) \
+  (&((nf)->family_members[(i) * NODEFAMILY_MEMBER_LEN]))
+
+#endif /* !defined(TOR_NODEFAMILY_ST_H) */
diff --git a/src/feature/nodelist/nodelist.c b/src/feature/nodelist/nodelist.c
index 9d56f2c3aa..7ebc4f5fda 100644
--- a/src/feature/nodelist/nodelist.c
+++ b/src/feature/nodelist/nodelist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,9 +49,9 @@
 #include "core/or/protover.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
-#include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/describe.h"
@@ -59,6 +59,7 @@
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/node_select.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/routerset.h"
@@ -156,9 +157,9 @@ node_id_eq(const node_t *node1, const node_t *node2)
   return tor_memeq(node1->identity, node2->identity, DIGEST_LEN);
 }
 
-HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq)
+HT_PROTOTYPE(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq);
 HT_GENERATE2(nodelist_map, node_t, ht_ent, node_id_hash, node_id_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 static inline unsigned int
 node_ed_id_hash(const node_t *node)
@@ -173,9 +174,9 @@ node_ed_id_eq(const node_t *node1, const node_t *node2)
 }
 
 HT_PROTOTYPE(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
-             node_ed_id_eq)
+             node_ed_id_eq);
 HT_GENERATE2(nodelist_ed_map, node_t, ed_ht_ent, node_ed_id_hash,
-             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_)
+             node_ed_id_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** The global nodelist. */
 static nodelist_t *the_nodelist=NULL;
@@ -702,7 +703,8 @@ nodelist_set_consensus(networkstatus_t *ns)
   SMARTLIST_FOREACH(the_nodelist->nodes, node_t *, node,
                     node->rs = NULL);
 
-  /* Conservatively estimate that every node will have 2 addresses. */
+  /* Conservatively estimate that every node will have 2 addresses (v4 and
+   * v6). Then we add the number of configured trusted authorities we have. */
   int estimated_addresses = smartlist_len(ns->routerstatus_list) *
                             get_estimated_address_per_node();
   estimated_addresses += (get_n_authorities(V3_DIRINFO | BRIDGE_DIRINFO) *
@@ -1045,7 +1047,7 @@ nodelist_ensure_freshness(networkstatus_t *ns)
 /** Return a list of a node_t * for every node we know about.  The caller
  * MUST NOT modify the list. (You can set and clear flags in the nodes if
  * you must, but you must not add or remove nodes.) */
-MOCK_IMPL(smartlist_t *,
+MOCK_IMPL(const smartlist_t *,
 nodelist_get_list,(void))
 {
   init_nodelist();
@@ -1148,8 +1150,8 @@ node_get_by_nickname,(const char *nickname, unsigned flags))
 
 /** Return the Ed25519 identity key for the provided node, or NULL if it
  * doesn't have one. */
-const ed25519_public_key_t *
-node_get_ed25519_id(const node_t *node)
+MOCK_IMPL(const ed25519_public_key_t *,
+node_get_ed25519_id,(const node_t *node))
 {
   const ed25519_public_key_t *ri_pk = NULL;
   const ed25519_public_key_t *md_pk = NULL;
@@ -1207,7 +1209,7 @@ node_ed25519_id_matches(const node_t *node, const ed25519_public_key_t *id)
 /** Dummy object that should be unreturnable.  Used to ensure that
  * node_get_protover_summary_flags() always returns non-NULL. */
 static const protover_summary_flags_t zero_protover_flags = {
-  0,0,0,0,0,0,0
+  0,0,0,0,0,0,0,0,0
 };
 
 /** Return the protover_summary_flags for a given node. */
@@ -1232,9 +1234,9 @@ node_get_protover_summary_flags(const node_t *node)
  * by ed25519 ID during the link handshake.  If <b>compatible_with_us</b>,
  * it needs to be using a link authentication method that we understand.
  * If not, any plausible link authentication method will do. */
-int
-node_supports_ed25519_link_authentication(const node_t *node,
-                                          int compatible_with_us)
+MOCK_IMPL(int,
+node_supports_ed25519_link_authentication,(const node_t *node,
+                                           int compatible_with_us))
 {
   if (! node_get_ed25519_id(node))
     return 0;
@@ -1267,6 +1269,17 @@ node_supports_ed25519_hs_intro(const node_t *node)
   return node_get_protover_summary_flags(node)->supports_ed25519_hs_intro;
 }
 
+/** Return true iff <b>node</b> supports the DoS ESTABLISH_INTRO cell
+ * extenstion. */
+int
+node_supports_establish_intro_dos_extension(const node_t *node)
+{
+  tor_assert(node);
+
+  return node_get_protover_summary_flags(node)->
+                           supports_establish_intro_dos_extension;
+}
+
 /** Return true iff <b>node</b> supports to be a rendezvous point for hidden
  * service version 3 (HSRend=2). */
 int
@@ -1290,6 +1303,102 @@ node_get_rsa_id_digest(const node_t *node)
   return (const uint8_t*)node->identity;
 }
 
+/* Returns a new smartlist with all possible link specifiers from node:
+ *  - legacy ID is mandatory thus MUST be present in node;
+ *  - include ed25519 link specifier if present in the node, and the node
+ *    supports ed25519 link authentication, and:
+ *    - if direct_conn is true, its link versions are compatible with us,
+ *    - if direct_conn is false, regardless of its link versions;
+ *  - include IPv4 link specifier, if the primary address is not IPv4, log a
+ *    BUG() warning, and return an empty smartlist;
+ *  - include IPv6 link specifier if present in the node.
+ *
+ * If node is NULL, returns an empty smartlist.
+ *
+ * The smartlist must be freed using link_specifier_smartlist_free(). */
+MOCK_IMPL(smartlist_t *,
+node_get_link_specifier_smartlist,(const node_t *node, bool direct_conn))
+{
+  link_specifier_t *ls;
+  tor_addr_port_t ap;
+  smartlist_t *lspecs = smartlist_new();
+
+  if (!node)
+    return lspecs;
+
+  /* Get the relay's IPv4 address. */
+  node_get_prim_orport(node, &ap);
+
+  /* We expect the node's primary address to be a valid IPv4 address.
+   * This conforms to the protocol, which requires either an IPv4 or IPv6
+   * address (or both). */
+  if (BUG(!tor_addr_is_v4(&ap.addr)) ||
+      BUG(!tor_addr_port_is_valid_ap(&ap, 0))) {
+    return lspecs;
+  }
+
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(ls, tor_addr_to_ipv4h(&ap.addr));
+  link_specifier_set_un_ipv4_port(ls, ap.port);
+  /* Four bytes IPv4 and two bytes port. */
+  link_specifier_set_ls_len(ls, sizeof(ap.addr.addr.in_addr) +
+                            sizeof(ap.port));
+  smartlist_add(lspecs, ls);
+
+  /* Legacy ID is mandatory and will always be present in node. */
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memcpy(link_specifier_getarray_un_legacy_id(ls), node->identity,
+         link_specifier_getlen_un_legacy_id(ls));
+  link_specifier_set_ls_len(ls, link_specifier_getlen_un_legacy_id(ls));
+  smartlist_add(lspecs, ls);
+
+  /* ed25519 ID is only included if the node has it, and the node declares a
+   protocol version that supports ed25519 link authentication.
+   If direct_conn is true, we also require that the node's link version is
+   compatible with us. (Otherwise, we will be sending the ed25519 key
+   to another tor, which may support different link versions.) */
+  if (!ed25519_public_key_is_zero(&node->ed25519_id) &&
+      node_supports_ed25519_link_authentication(node, direct_conn)) {
+    ls = link_specifier_new();
+    link_specifier_set_ls_type(ls, LS_ED25519_ID);
+    memcpy(link_specifier_getarray_un_ed25519_id(ls), &node->ed25519_id,
+           link_specifier_getlen_un_ed25519_id(ls));
+    link_specifier_set_ls_len(ls, link_specifier_getlen_un_ed25519_id(ls));
+    smartlist_add(lspecs, ls);
+  }
+
+  /* Check for IPv6. If so, include it as well. */
+  if (node_has_ipv6_orport(node)) {
+    ls = link_specifier_new();
+    node_get_pref_ipv6_orport(node, &ap);
+    link_specifier_set_ls_type(ls, LS_IPV6);
+    size_t addr_len = link_specifier_getlen_un_ipv6_addr(ls);
+    const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ap.addr);
+    uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(ls);
+    memcpy(ipv6_array, in6_addr, addr_len);
+    link_specifier_set_un_ipv6_port(ls, ap.port);
+    /* Sixteen bytes IPv6 and two bytes port. */
+    link_specifier_set_ls_len(ls, addr_len + sizeof(ap.port));
+    smartlist_add(lspecs, ls);
+  }
+
+  return lspecs;
+}
+
+/* Free a link specifier list. */
+void
+link_specifier_smartlist_free_(smartlist_t *ls_list)
+{
+  if (!ls_list)
+    return;
+
+  SMARTLIST_FOREACH(ls_list, link_specifier_t *, lspec,
+                    link_specifier_free(lspec));
+  smartlist_free(ls_list);
+}
+
 /** Return the nickname of <b>node</b>, or NULL if we can't find one. */
 const char *
 node_get_nickname(const node_t *node)
@@ -1429,8 +1538,7 @@ node_exit_policy_rejects_all(const node_t *node)
   if (node->ri)
     return node->ri->policy_is_reject_star;
   else if (node->md)
-    return node->md->exit_policy == NULL ||
-      short_policy_is_reject_star(node->md->exit_policy);
+    return node->md->policy_is_reject_star;
   else
     return 1;
 }
@@ -1605,19 +1713,6 @@ node_is_me(const node_t *node)
   return router_digest_is_me(node->identity);
 }
 
-/** Return <b>node</b> declared family (as a list of names), or NULL if
- * the node didn't declare a family. */
-const smartlist_t *
-node_get_declared_family(const node_t *node)
-{
-  if (node->ri && node->ri->declared_family)
-    return node->ri->declared_family;
-  else if (node->md && node->md->family)
-    return node->md->family;
-  else
-    return NULL;
-}
-
 /* Does this node have a valid IPv6 address?
  * Prefer node_has_ipv6_orport() or node_has_ipv6_dirport() for
  * checking specific ports. */
@@ -1877,7 +1972,7 @@ microdesc_has_curve25519_onion_key(const microdesc_t *md)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)md->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -1957,7 +2052,7 @@ node_set_country(node_t *node)
 void
 nodelist_refresh_countries(void)
 {
-  smartlist_t *nodes = nodelist_get_list();
+  const smartlist_t *nodes = nodelist_get_list();
   SMARTLIST_FOREACH(nodes, node_t *, node,
                     node_set_country(node));
 }
@@ -1968,6 +2063,9 @@ int
 addrs_in_same_network_family(const tor_addr_t *a1,
                              const tor_addr_t *a2)
 {
+  if (tor_addr_is_null(a1) || tor_addr_is_null(a2))
+    return 0;
+
   switch (tor_addr_family(a1)) {
     case AF_INET:
       return 0 == tor_addr_compare_masked(a1, a2, 16, CMP_SEMANTIC);
@@ -1983,7 +2081,7 @@ addrs_in_same_network_family(const tor_addr_t *a1,
  * (case-insensitive), or if <b>node's</b> identity key digest
  * matches a hexadecimal value stored in <b>nickname</b>.  Return
  * false otherwise. */
-static int
+STATIC int
 node_nickname_matches(const node_t *node, const char *nickname)
 {
   const char *n = node_get_nickname(node);
@@ -1995,7 +2093,7 @@ node_nickname_matches(const node_t *node, const char *nickname)
 }
 
 /** Return true iff <b>node</b> is named by some nickname in <b>lst</b>. */
-static inline int
+STATIC int
 node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
 {
   if (!lst) return 0;
@@ -2006,6 +2104,61 @@ node_in_nickname_smartlist(const smartlist_t *lst, const node_t *node)
   return 0;
 }
 
+/** Return true iff n1's declared family contains n2. */
+STATIC int
+node_family_contains(const node_t *n1, const node_t *n2)
+{
+  if (n1->ri && n1->ri->declared_family) {
+    return node_in_nickname_smartlist(n1->ri->declared_family, n2);
+  } else if (n1->md) {
+    return nodefamily_contains_node(n1->md->family, n2);
+  } else {
+    return 0;
+  }
+}
+
+/**
+ * Return true iff <b>node</b> has declared a nonempty family.
+ **/
+STATIC bool
+node_has_declared_family(const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    return true;
+  }
+
+  if (node->md && node->md->family) {
+    return true;
+  }
+
+  return false;
+}
+
+/**
+ * Add to <b>out</b> every node_t that is listed by <b>node</b> as being in
+ * its family.  (Note that these nodes are not in node's family unless they
+ * also agree that node is in their family.)
+ **/
+STATIC void
+node_lookup_declared_family(smartlist_t *out, const node_t *node)
+{
+  if (node->ri && node->ri->declared_family &&
+      smartlist_len(node->ri->declared_family)) {
+    SMARTLIST_FOREACH_BEGIN(node->ri->declared_family, const char *, name) {
+      const node_t *n2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED);
+      if (n2) {
+        smartlist_add(out, (node_t *)n2);
+      }
+    } SMARTLIST_FOREACH_END(name);
+    return;
+  }
+
+  if (node->md && node->md->family) {
+    nodefamily_add_nodes_to_smartlist(node->md->family, out);
+  }
+}
+
 /** Return true iff r1 and r2 are in the same family, but not the same
  * router. */
 int
@@ -2018,19 +2171,20 @@ nodes_in_same_family(const node_t *node1, const node_t *node2)
     tor_addr_t a1, a2;
     node_get_addr(node1, &a1);
     node_get_addr(node2, &a2);
-    if (addrs_in_same_network_family(&a1, &a2))
+
+    tor_addr_port_t ap6_1, ap6_2;
+    node_get_pref_ipv6_orport(node1, &ap6_1);
+    node_get_pref_ipv6_orport(node2, &ap6_2);
+
+    if (addrs_in_same_network_family(&a1, &a2) ||
+        addrs_in_same_network_family(&ap6_1.addr, &ap6_2.addr))
       return 1;
   }
 
   /* Are they in the same family because the agree they are? */
-  {
-    const smartlist_t *f1, *f2;
-    f1 = node_get_declared_family(node1);
-    f2 = node_get_declared_family(node2);
-    if (f1 && f2 &&
-        node_in_nickname_smartlist(f1, node2) &&
-        node_in_nickname_smartlist(f2, node1))
-      return 1;
+  if (node_family_contains(node1, node2) &&
+      node_family_contains(node2, node1)) {
+    return 1;
   }
 
   /* Are they in the same family because the user says they are? */
@@ -2058,13 +2212,10 @@ void
 nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
 {
   const smartlist_t *all_nodes = nodelist_get_list();
-  const smartlist_t *declared_family;
   const or_options_t *options = get_options();
 
   tor_assert(node);
 
-  declared_family = node_get_declared_family(node);
-
   /* Let's make sure that we have the node itself, if it's a real node. */
   {
     const node_t *real_node = node_get_by_id(node->identity);
@@ -2075,35 +2226,35 @@ nodelist_add_node_and_family(smartlist_t *sl, const node_t *node)
   /* First, add any nodes with similar network addresses. */
   if (options->EnforceDistinctSubnets) {
     tor_addr_t node_addr;
+    tor_addr_port_t node_ap6;
     node_get_addr(node, &node_addr);
+    node_get_pref_ipv6_orport(node, &node_ap6);
 
     SMARTLIST_FOREACH_BEGIN(all_nodes, const node_t *, node2) {
       tor_addr_t a;
+      tor_addr_port_t ap6;
       node_get_addr(node2, &a);
-      if (addrs_in_same_network_family(&a, &node_addr))
+      node_get_pref_ipv6_orport(node2, &ap6);
+      if (addrs_in_same_network_family(&a, &node_addr) ||
+          addrs_in_same_network_family(&ap6.addr, &node_ap6.addr))
         smartlist_add(sl, (void*)node2);
     } SMARTLIST_FOREACH_END(node2);
   }
 
-  /* Now, add all nodes in the declared_family of this node, if they
+  /* Now, add all nodes in the declared family of this node, if they
    * also declare this node to be in their family. */
-  if (declared_family) {
+  if (node_has_declared_family(node)) {
+    smartlist_t *declared_family = smartlist_new();
+    node_lookup_declared_family(declared_family, node);
+
     /* Add every r such that router declares familyness with node, and node
      * declares familyhood with router. */
-    SMARTLIST_FOREACH_BEGIN(declared_family, const char *, name) {
-      const node_t *node2;
-      const smartlist_t *family2;
-      if (!(node2 = node_get_by_nickname(name, NNF_NO_WARN_UNNAMED)))
-        continue;
-      if (!(family2 = node_get_declared_family(node2)))
-        continue;
-      SMARTLIST_FOREACH_BEGIN(family2, const char *, name2) {
-          if (node_nickname_matches(node, name2)) {
-            smartlist_add(sl, (void*)node2);
-            break;
-          }
-      } SMARTLIST_FOREACH_END(name2);
-    } SMARTLIST_FOREACH_END(name);
+    SMARTLIST_FOREACH_BEGIN(declared_family, const node_t *, node2) {
+      if (node_family_contains(node2, node)) {
+        smartlist_add(sl, (void*)node2);
+      }
+    } SMARTLIST_FOREACH_END(node2);
+    smartlist_free(declared_family);
   }
 
   /* If the user declared any families locally, honor those too. */
@@ -2408,7 +2559,7 @@ compute_frac_paths_available(const networkstatus_t *consensus,
   const int authdir = authdir_mode_v3(options);
 
   count_usable_descriptors(num_present_out, num_usable_out,
-                           mid, consensus, now, NULL,
+                           mid, consensus, now, options->MiddleNodes,
                            USABLE_DESCRIPTOR_ALL);
   log_debug(LD_NET,
             "%s: %d present, %d usable",
@@ -2610,7 +2761,7 @@ count_loading_descriptors_progress(void)
   if (fraction > 1.0)
     return 0; /* it's not the number of descriptors holding us back */
   return BOOTSTRAP_STATUS_LOADING_DESCRIPTORS + (int)
-    (fraction*(BOOTSTRAP_STATUS_CONN_OR-1 -
+    (fraction*(BOOTSTRAP_STATUS_ENOUGH_DIRINFO-1 -
                BOOTSTRAP_STATUS_LOADING_DESCRIPTORS));
 }
 
@@ -2697,14 +2848,14 @@ update_router_have_minimum_dir_info(void)
   /* If paths have just become available in this update. */
   if (res && !have_min_dir_info) {
     control_event_client_status(LOG_NOTICE, "ENOUGH_DIR_INFO");
-    control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+    control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
     log_info(LD_DIR,
              "We now have enough directory information to build circuits.");
   }
 
   /* If paths have just become unavailable in this update. */
   if (!res && have_min_dir_info) {
-    int quiet = directory_too_idle_to_fetch_descriptors(options, now);
+    int quiet = dirclient_too_idle_to_fetch_descriptors(options, now);
     tor_log(quiet ? LOG_INFO : LOG_NOTICE, LD_DIR,
         "Our directory information is no longer up-to-date "
         "enough to build circuits: %s", dir_info_status);
diff --git a/src/feature/nodelist/nodelist.h b/src/feature/nodelist/nodelist.h
index bc09731ce2..0e06326a9c 100644
--- a/src/feature/nodelist/nodelist.h
+++ b/src/feature/nodelist/nodelist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,16 +71,23 @@ const char *node_get_platform(const node_t *node);
 uint32_t node_get_prim_addr_ipv4h(const node_t *node);
 void node_get_address_string(const node_t *node, char *cp, size_t len);
 long node_get_declared_uptime(const node_t *node);
-const smartlist_t *node_get_declared_family(const node_t *node);
-const struct ed25519_public_key_t *node_get_ed25519_id(const node_t *node);
+MOCK_DECL(const struct ed25519_public_key_t *,node_get_ed25519_id,
+          (const node_t *node));
 int node_ed25519_id_matches(const node_t *node,
                             const struct ed25519_public_key_t *id);
-int node_supports_ed25519_link_authentication(const node_t *node,
-                                              int compatible_with_us);
+MOCK_DECL(int,node_supports_ed25519_link_authentication,
+          (const node_t *node,
+           int compatible_with_us));
 int node_supports_v3_hsdir(const node_t *node);
 int node_supports_ed25519_hs_intro(const node_t *node);
 int node_supports_v3_rendezvous_point(const node_t *node);
+int node_supports_establish_intro_dos_extension(const node_t *node);
 const uint8_t *node_get_rsa_id_digest(const node_t *node);
+MOCK_DECL(smartlist_t *,node_get_link_specifier_smartlist,(const node_t *node,
+                                                           bool direct_conn));
+void link_specifier_smartlist_free_(smartlist_t *ls_list);
+#define link_specifier_smartlist_free(ls_list) \
+  FREE_AND_NULL(smartlist_t, link_specifier_smartlist_free_, (ls_list))
 
 int node_has_ipv6_addr(const node_t *node);
 int node_has_ipv6_orport(const node_t *node);
@@ -100,7 +107,7 @@ const struct curve25519_public_key_t *node_get_curve25519_onion_key(
                                   const node_t *node);
 crypto_pk_t *node_get_rsa_onion_key(const node_t *node);
 
-MOCK_DECL(smartlist_t *, nodelist_get_list, (void));
+MOCK_DECL(const smartlist_t *, nodelist_get_list, (void));
 
 /* Temporary during transition to multiple addresses.  */
 void node_get_addr(const node_t *node, tor_addr_t *addr_out);
@@ -158,10 +165,16 @@ int count_loading_descriptors_progress(void);
 
 #ifdef NODELIST_PRIVATE
 
+STATIC int node_nickname_matches(const node_t *node, const char *nickname);
+STATIC int node_in_nickname_smartlist(const smartlist_t *lst,
+                                      const node_t *node);
+STATIC int node_family_contains(const node_t *n1, const node_t *n2);
+STATIC bool node_has_declared_family(const node_t *node);
+STATIC void node_lookup_declared_family(smartlist_t *out, const node_t *node);
+
 #ifdef TOR_UNIT_TESTS
 
-STATIC void
-node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
+STATIC void node_set_hsdir_index(node_t *node, const networkstatus_t *ns);
 
 #endif /* defined(TOR_UNIT_TESTS) */
 
diff --git a/src/feature/nodelist/routerinfo.c b/src/feature/nodelist/routerinfo.c
index 975b503615..0bf2a977f5 100644
--- a/src/feature/nodelist/routerinfo.c
+++ b/src/feature/nodelist/routerinfo.c
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerinfo.c
+ * @brief Manipulate full router descriptors.
+ **/
+
 #include "core/or/or.h"
 
 #include "feature/nodelist/nodelist.h"
diff --git a/src/feature/nodelist/routerinfo.h b/src/feature/nodelist/routerinfo.h
index bfa28c7754..604e478999 100644
--- a/src/feature/nodelist/routerinfo.h
+++ b/src/feature/nodelist/routerinfo.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,11 +17,9 @@ void router_get_prim_orport(const routerinfo_t *router,
 int router_has_orport(const routerinfo_t *router,
                       const tor_addr_port_t *orport);
 
-void router_get_verbose_nickname(char *buf, const routerinfo_t *router);
-
 smartlist_t *router_get_all_orports(const routerinfo_t *ri);
 
 const char *router_purpose_to_string(uint8_t p);
 uint8_t router_purpose_from_string(const char *s);
 
-#endif
+#endif /* !defined(TOR_ROUTERINFO_H) */
diff --git a/src/feature/nodelist/routerinfo_st.h b/src/feature/nodelist/routerinfo_st.h
index 59656818c1..36ead50e33 100644
--- a/src/feature/nodelist/routerinfo_st.h
+++ b/src/feature/nodelist/routerinfo_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerinfo_st.h
+ * @brief Router descriptor structure.
+ **/
+
 #ifndef ROUTERINFO_ST_H
 #define ROUTERINFO_ST_H
 
@@ -21,9 +26,6 @@ struct routerinfo_t {
   uint16_t dir_port; /**< Port for HTTP directory connections. */
 
   /** A router's IPv6 address, if it has one. */
-  /* XXXXX187 Actually these should probably be part of a list of addresses,
-   * not just a special case.  Use abstractions to access these; don't do it
-   * directly. */
   tor_addr_t ipv6_addr;
   uint16_t ipv6_orport;
 
@@ -112,4 +114,4 @@ struct routerinfo_t {
   uint8_t purpose;
 };
 
-#endif
+#endif /* !defined(ROUTERINFO_ST_H) */
diff --git a/src/feature/nodelist/routerlist.c b/src/feature/nodelist/routerlist.c
index c7fa868929..96bea5a670 100644
--- a/src/feature/nodelist/routerlist.c
+++ b/src/feature/nodelist/routerlist.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -67,12 +67,13 @@
 #include "core/mainloop/mainloop.h"
 #include "core/or/policies.h"
 #include "feature/client/bridges.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/authmode.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dirauth/reachability.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/authcert.h"
@@ -116,9 +117,9 @@
 /* Typed wrappers for different digestmap types; used to avoid type
  * confusion. */
 
-DECLARE_TYPED_DIGESTMAP_FNS(sdmap_, digest_sd_map_t, signed_descriptor_t)
-DECLARE_TYPED_DIGESTMAP_FNS(rimap_, digest_ri_map_t, routerinfo_t)
-DECLARE_TYPED_DIGESTMAP_FNS(eimap_, digest_ei_map_t, extrainfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(sdmap, digest_sd_map_t, signed_descriptor_t)
+DECLARE_TYPED_DIGESTMAP_FNS(rimap, digest_ri_map_t, routerinfo_t)
+DECLARE_TYPED_DIGESTMAP_FNS(eimap, digest_ei_map_t, extrainfo_t)
 #define SDMAP_FOREACH(map, keyvar, valvar)                              \
   DIGESTMAP_FOREACH(sdmap_to_digestmap(map), keyvar, signed_descriptor_t *, \
                     valvar)
@@ -160,7 +161,7 @@ static time_t last_descriptor_download_attempted = 0;
  *
  * From time to time, we replace "cached-descriptors" with a new file
  * containing only the live, non-superseded descriptors, and clear
- * cached-routers.new.
+ * cached-descriptors.new.
  *
  * On startup, we read both files.
  */
@@ -1463,12 +1464,13 @@ router_descriptor_is_older_than,(const routerinfo_t *router, int seconds))
 }
 
 /** Add <b>router</b> to the routerlist, if we don't already have it.  Replace
- * older entries (if any) with the same key.  Note: Callers should not hold
- * their pointers to <b>router</b> if this function fails; <b>router</b>
- * will either be inserted into the routerlist or freed. Similarly, even
- * if this call succeeds, they should not hold their pointers to
- * <b>router</b> after subsequent calls with other routerinfo's -- they
- * might cause the original routerinfo to get freed.
+ * older entries (if any) with the same key.
+ *
+ * Note: Callers should not hold their pointers to <b>router</b> if this
+ * function fails; <b>router</b> will either be inserted into the routerlist or
+ * freed. Similarly, even if this call succeeds, they should not hold their
+ * pointers to <b>router</b> after subsequent calls with other routerinfo's --
+ * they might cause the original routerinfo to get freed.
  *
  * Returns the status for the operation. Might set *<b>msg</b> if it wants
  * the poster of the router to know something.
@@ -1930,6 +1932,8 @@ routerlist_remove_old_routers(void)
 void
 routerlist_descriptors_added(smartlist_t *sl, int from_cache)
 {
+  // XXXX use pubsub mechanism here.
+
   tor_assert(sl);
   control_event_descriptors_changed(sl);
   SMARTLIST_FOREACH_BEGIN(sl, routerinfo_t *, ri) {
@@ -2405,7 +2409,7 @@ max_dl_per_request(const or_options_t *options, int purpose)
   }
   /* If we're going to tunnel our connections, we can ask for a lot more
    * in a request. */
-  if (directory_must_use_begindir(options)) {
+  if (dirclient_must_use_begindir(options)) {
     max = 500;
   }
   return max;
@@ -2448,7 +2452,7 @@ launch_descriptor_downloads(int purpose,
   if (!n_downloadable)
     return;
 
-  if (!directory_fetches_dir_info_early(options)) {
+  if (!dirclient_fetches_dir_info_early(options)) {
     if (n_downloadable >= MAX_DL_TO_DELAY) {
       log_debug(LD_DIR,
                 "There are enough downloadable %ss to launch requests.",
@@ -2539,7 +2543,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
   int n_delayed=0, n_have=0, n_would_reject=0, n_wouldnt_use=0,
     n_inprogress=0, n_in_oldrouters=0;
 
-  if (directory_too_idle_to_fetch_descriptors(options, now))
+  if (dirclient_too_idle_to_fetch_descriptors(options, now))
     goto done;
   if (!consensus)
     goto done;
@@ -2559,8 +2563,15 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
   map = digestmap_new();
   list_pending_descriptor_downloads(map, 0);
   SMARTLIST_FOREACH_BEGIN(consensus->routerstatus_list, void *, rsp) {
-      routerstatus_t *rs =
-        is_vote ? &(((vote_routerstatus_t *)rsp)->status) : rsp;
+      routerstatus_t *rs;
+      vote_routerstatus_t *vrs;
+      if (is_vote) {
+        rs = &(((vote_routerstatus_t *)rsp)->status);
+        vrs = rsp;
+      } else {
+        rs = rsp;
+        vrs = NULL;
+      }
       signed_descriptor_t *sd;
       if ((sd = router_get_by_descriptor_digest(rs->descriptor_digest))) {
         const routerinfo_t *ri;
@@ -2585,7 +2596,7 @@ update_consensus_router_descriptor_downloads(time_t now, int is_vote,
         ++n_delayed; /* Not ready for retry. */
         continue;
       }
-      if (authdir && dirserv_would_reject_router(rs)) {
+      if (authdir && is_vote && dirserv_would_reject_router(rs, vrs)) {
         ++n_would_reject;
         continue; /* We would throw it out immediately. */
       }
@@ -2915,7 +2926,7 @@ router_differences_are_cosmetic(const routerinfo_t *r1, const routerinfo_t *r2)
       (r1->bandwidthburst != r2->bandwidthburst))
     return 0;
 
-  /* Did more than 12 hours pass? */
+  /* Has enough time passed between the publication times? */
   if (r1->cache_info.published_on + ROUTER_MAX_COSMETIC_TIME_DIFFERENCE
       < r2->cache_info.published_on)
     return 0;
@@ -2975,7 +2986,7 @@ routerinfo_incompatible_with_extrainfo(const crypto_pk_t *identity_pkey,
   digest256_matches = tor_memeq(ei->digest256,
                                 sd->extra_info_digest256, DIGEST256_LEN);
   digest256_matches |=
-    tor_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
+    fast_mem_is_zero(sd->extra_info_digest256, DIGEST256_LEN);
 
   /* The identity must match exactly to have been generated at the same time
    * by the same router. */
@@ -3059,7 +3070,7 @@ routerinfo_has_curve25519_onion_key(const routerinfo_t *ri)
     return 0;
   }
 
-  if (tor_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
+  if (fast_mem_is_zero((const char*)ri->onion_curve25519_pkey->public_key,
                       CURVE25519_PUBKEY_LEN)) {
     return 0;
   }
@@ -3227,6 +3238,8 @@ refresh_all_country_info(void)
     routerset_refresh_countries(options->EntryNodes);
   if (options->ExitNodes)
     routerset_refresh_countries(options->ExitNodes);
+  if (options->MiddleNodes)
+    routerset_refresh_countries(options->MiddleNodes);
   if (options->ExcludeNodes)
     routerset_refresh_countries(options->ExcludeNodes);
   if (options->ExcludeExitNodes)
diff --git a/src/feature/nodelist/routerlist.h b/src/feature/nodelist/routerlist.h
index 5771ebb1ab..81a2343540 100644
--- a/src/feature/nodelist/routerlist.h
+++ b/src/feature/nodelist/routerlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -37,9 +37,12 @@ typedef enum was_router_added_t {
   ROUTER_WAS_NOT_WANTED = -6,
   /* Router descriptor was rejected because it was older than
    * OLD_ROUTER_DESC_MAX_AGE. */
-  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'NOT_NEW' */
-  /* DOCDOC */
-  ROUTER_CERTS_EXPIRED = -8
+  ROUTER_WAS_TOO_OLD = -7, /* note contrast with 'ROUTER_IS_ALREADY_KNOWN' */
+  /* Some certs on this router are expired. */
+  ROUTER_CERTS_EXPIRED = -8,
+  /* We couldn't format the annotations for this router. This is a directory
+   * authority bug. */
+  ROUTER_AUTHDIR_BUG_ANNOTATIONS = -10
 } was_router_added_t;
 
 /** How long do we avoid using a directory server after it's given us a 503? */
diff --git a/src/feature/nodelist/routerlist_st.h b/src/feature/nodelist/routerlist_st.h
index 7446ead3cb..ec8933c7cb 100644
--- a/src/feature/nodelist/routerlist_st.h
+++ b/src/feature/nodelist/routerlist_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerlist_st.h
+ * @brief Router descriptor list structure.
+ **/
+
 #ifndef ROUTERLIST_ST_H
 #define ROUTERLIST_ST_H
 
@@ -36,5 +41,4 @@ struct routerlist_t {
   desc_store_t extrainfo_store;
 };
 
-#endif
-
+#endif /* !defined(ROUTERLIST_ST_H) */
diff --git a/src/feature/nodelist/routerset.c b/src/feature/nodelist/routerset.c
index 55e2756959..2e06ecbf04 100644
--- a/src/feature/nodelist/routerset.c
+++ b/src/feature/nodelist/routerset.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
-n * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,7 +17,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
  *
  * Routersets are typically used for user-specified restrictions, and
  * are created by invoking routerset_new and routerset_parse from
- * config.c and confparse.c.  To use a routerset, invoke one of
+ * config.c and confmgt.c.  To use a routerset, invoke one of
  * routerset_contains_...() functions , or use
  * routerstatus_get_all_nodes() / routerstatus_subtract_nodes() to
  * manipulate a smartlist of node_t pointers.
@@ -34,6 +34,9 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/nickname.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerset.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
 #include "lib/geoip/geoip.h"
 
 #include "core/or/addr_policy_st.h"
@@ -41,6 +44,7 @@ n * Copyright (c) 2001-2004, Roger Dingledine.
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
+#include "lib/confmgt/var_type_def_st.h"
 
 /** Return a new empty routerset. */
 routerset_t *
@@ -378,7 +382,7 @@ routerset_get_all_nodes(smartlist_t *out, const routerset_t *routerset,
   } else {
     /* We need to iterate over the routerlist to get all the ones of the
      * right kind. */
-    smartlist_t *nodes = nodelist_get_list();
+    const smartlist_t *nodes = nodelist_get_list();
     SMARTLIST_FOREACH(nodes, const node_t *, node, {
         if (running_only && !node->is_running)
           continue;
@@ -461,3 +465,111 @@ routerset_free_(routerset_t *routerset)
   bitarray_free(routerset->countries);
   tor_free(routerset);
 }
+
+/**
+ * config helper: parse a routerset-typed variable.
+ *
+ * Takes as input as a single line in <b>line</b>; writes its results into a
+ * routerset_t** passed as <b>target</b>.  On success return 0; on failure
+ * return -1 and store an error message into *<b>errmsg</b>.
+ **/
+/*
+ * Warning: For this type, the default value (NULL) and "" are sometimes
+ * considered different values.  That is generally risky, and best avoided for
+ * other types in the future.  For cases where we want the default to be "all
+ * routers" (like EntryNodes) we should add a new routerset value indicating
+ * "all routers" (see #31908)
+ */
+static int
+routerset_kv_parse(void *target, const config_line_t *line, char **errmsg,
+                  const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)target;
+  routerset_free(*p); // clear the old value, if any.
+  routerset_t *rs = routerset_new();
+  if (routerset_parse(rs, line->value, line->key) < 0) {
+    routerset_free(rs);
+    *errmsg = tor_strdup("Invalid router list.");
+    return -1;
+  } else {
+    if (routerset_is_empty(rs)) {
+      /* Represent empty sets as NULL. */
+      routerset_free(rs);
+    }
+    *p = rs;
+    return 0;
+  }
+}
+
+/**
+ * config helper: encode a routerset-typed variable.
+ *
+ * Return a newly allocated string containing the value of the
+ * routerset_t** passed as <b>value</b>.
+ */
+static char *
+routerset_encode(const void *value, const void *params)
+{
+  (void)params;
+  const routerset_t **p = (const routerset_t**)value;
+  return routerset_to_string(*p);
+}
+
+/**
+ * config helper: free and clear a routerset-typed variable.
+ *
+ * Clear the routerset_t** passed as <b>value</b>.
+ */
+static void
+routerset_clear(void *value, const void *params)
+{
+  (void)params;
+  routerset_t **p = (routerset_t**)value;
+  routerset_free(*p); // sets *p to NULL.
+}
+
+/**
+ * config helper: copy a routerset-typed variable.
+ *
+ * Takes it input from a routerset_t** in <b>src</b>; writes its output to a
+ * routerset_t** in <b>dest</b>.  Returns 0 on success, -1 on (impossible)
+ * failure.
+ **/
+static int
+routerset_copy(void *dest, const void *src, const void *params)
+{
+  (void)params;
+  routerset_t **output = (routerset_t**)dest;
+  const routerset_t *input = *(routerset_t**)src;
+  routerset_free(*output); // sets *output to NULL
+  if (! routerset_is_empty(input)) {
+    *output = routerset_new();
+    routerset_union(*output, input);
+  }
+  return 0;
+}
+
+/**
+ * Function table to implement a routerset_t-based configuration type.
+ **/
+static const var_type_fns_t routerset_type_fns = {
+  .kv_parse = routerset_kv_parse,
+  .encode = routerset_encode,
+  .clear = routerset_clear,
+  .copy = routerset_copy
+};
+
+/**
+ * Definition of a routerset_t-based configuration type.
+ *
+ * Values are mapped to and from strings using the format defined in
+ * routerset_parse(): nicknames, IP address patterns, and fingerprints--with
+ * optional space, separated by commas.
+ *
+ * Empty sets are represented as NULL.
+ **/
+const var_type_def_t ROUTERSET_type_defn = {
+  .name = "RouterList",
+  .fns = &routerset_type_fns
+};
diff --git a/src/feature/nodelist/routerset.h b/src/feature/nodelist/routerset.h
index ca8b6fed93..0e4fedf64e 100644
--- a/src/feature/nodelist/routerset.h
+++ b/src/feature/nodelist/routerset.h
@@ -1,10 +1,10 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file routerlist.h
+ * \file routerset.h
  * \brief Header file for routerset.c
  **/
 
@@ -44,6 +44,10 @@ void routerset_free_(routerset_t *routerset);
 #define routerset_free(rs) FREE_AND_NULL(routerset_t, routerset_free_, (rs))
 int routerset_len(const routerset_t *set);
 
+struct var_type_def_t;
+extern const struct var_type_def_t ROUTERSET_type_defn;
+typedef routerset_t *config_decl_ROUTERSET;
+
 #ifdef ROUTERSET_PRIVATE
 #include "lib/container/bitarray.h"
 
diff --git a/src/feature/nodelist/routerstatus_st.h b/src/feature/nodelist/routerstatus_st.h
index 288edf5982..735c754b31 100644
--- a/src/feature/nodelist/routerstatus_st.h
+++ b/src/feature/nodelist/routerstatus_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerstatus_st.h
+ * @brief Routerstatus (consensus entry) structure
+ **/
+
 #ifndef ROUTERSTATUS_ST_H
 #define ROUTERSTATUS_ST_H
 
@@ -12,6 +17,10 @@
 /** Contents of a single router entry in a network status object.
  */
 struct routerstatus_t {
+ /* This should be kept in sync with the function
+ * routerstatus_has_visibly_changed and the printing function
+ * routerstatus_format_entry in NS_CONTROL_PORT mode.
+ */
   time_t published_on; /**< When was this router published? */
   char nickname[MAX_NICKNAME_LEN+1]; /**< The nickname this router says it
                                       * has. */
@@ -47,6 +56,8 @@ struct routerstatus_t {
   unsigned int is_v2_dir:1; /** True iff this router publishes an open DirPort
                              * or it claims to accept tunnelled dir requests.
                              */
+  unsigned int is_staledesc:1; /** True iff the authorities think this router
+                                * should upload a new descriptor soon. */
 
   unsigned int has_bandwidth:1; /**< The vote/consensus had bw info */
   unsigned int has_exitsummary:1; /**< The vote/consensus had exit summaries */
@@ -76,5 +87,4 @@ struct routerstatus_t {
 
 };
 
-#endif
-
+#endif /* !defined(ROUTERSTATUS_ST_H) */
diff --git a/src/feature/nodelist/signed_descriptor_st.h b/src/feature/nodelist/signed_descriptor_st.h
index bdcebf184a..068f2a733c 100644
--- a/src/feature/nodelist/signed_descriptor_st.h
+++ b/src/feature/nodelist/signed_descriptor_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file signed_descriptor_st.h
+ * @brief Descriptor/extrainfo signature structure
+ **/
+
 #ifndef SIGNED_DESCRIPTOR_ST_H
 #define SIGNED_DESCRIPTOR_ST_H
 
@@ -57,5 +62,4 @@ struct signed_descriptor_t {
   unsigned int send_unencrypted : 1;
 };
 
-#endif
-
+#endif /* !defined(SIGNED_DESCRIPTOR_ST_H) */
diff --git a/src/feature/nodelist/torcert.c b/src/feature/nodelist/torcert.c
index b0197e9f13..89cc9c88fb 100644
--- a/src/feature/nodelist/torcert.c
+++ b/src/feature/nodelist/torcert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,7 +74,7 @@ tor_cert_sign_impl(const ed25519_keypair_t *signing_key,
   tor_assert(real_len == alloc_len);
   tor_assert(real_len > ED25519_SIG_LEN);
   uint8_t *sig = encoded + (real_len - ED25519_SIG_LEN);
-  tor_assert(tor_mem_is_zero((char*)sig, ED25519_SIG_LEN));
+  tor_assert(fast_mem_is_zero((char*)sig, ED25519_SIG_LEN));
 
   ed25519_signature_t signature;
   if (ed25519_sign(&signature, encoded,
@@ -290,8 +290,8 @@ tor_cert_describe_signature_status(const tor_cert_t *cert)
 }
 
 /** Return a new copy of <b>cert</b> */
-tor_cert_t *
-tor_cert_dup(const tor_cert_t *cert)
+MOCK_IMPL(tor_cert_t *,
+tor_cert_dup,(const tor_cert_t *cert))
 {
   tor_cert_t *newcert = tor_memdup(cert, sizeof(tor_cert_t));
   if (cert->encoded)
diff --git a/src/feature/nodelist/torcert.h b/src/feature/nodelist/torcert.h
index 492275b514..f8fba2b794 100644
--- a/src/feature/nodelist/torcert.h
+++ b/src/feature/nodelist/torcert.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file torcert.h
+ * @brief Header for torcert.c
+ **/
+
 #ifndef TORCERT_H_INCLUDED
 #define TORCERT_H_INCLUDED
 
@@ -71,7 +76,7 @@ int tor_cert_checksig(tor_cert_t *cert,
                       const ed25519_public_key_t *pubkey, time_t now);
 const char *tor_cert_describe_signature_status(const tor_cert_t *cert);
 
-tor_cert_t *tor_cert_dup(const tor_cert_t *cert);
+MOCK_DECL(tor_cert_t *,tor_cert_dup,(const tor_cert_t *cert));
 int tor_cert_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 int tor_cert_opt_eq(const tor_cert_t *cert1, const tor_cert_t *cert2);
 
diff --git a/src/feature/nodelist/vote_routerstatus_st.h b/src/feature/nodelist/vote_routerstatus_st.h
index 366754c166..ad0ee3f23b 100644
--- a/src/feature/nodelist/vote_routerstatus_st.h
+++ b/src/feature/nodelist/vote_routerstatus_st.h
@@ -1,9 +1,13 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file vote_routerstatus_st.h
+ * @brief Routerstatus (vote entry) structure
+ **/
 #ifndef VOTE_ROUTERSTATUS_ST_H
 #define VOTE_ROUTERSTATUS_ST_H
 
@@ -38,4 +42,4 @@ struct vote_routerstatus_t {
   uint8_t ed25519_id[ED25519_PUBKEY_LEN];
 };
 
-#endif
+#endif /* !defined(VOTE_ROUTERSTATUS_ST_H) */
diff --git a/src/feature/relay/.may_include b/src/feature/relay/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/relay/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/relay/circuitbuild_relay.c b/src/feature/relay/circuitbuild_relay.c
new file mode 100644
index 0000000000..b89866b477
--- /dev/null
+++ b/src/feature/relay/circuitbuild_relay.c
@@ -0,0 +1,549 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file circuitbuild_relay.c
+ * @brief Implements the details of exteding circuits (by relaying extend
+ * cells as create cells, and answering create cells).
+ *
+ * On the server side, this module handles the logic of responding to
+ * RELAY_EXTEND requests, using circuit_extend() and onionskin_answer().
+ *
+ * The shared client and server code is in core/or/circuitbuild.c.
+ **/
+
+#include "orconfig.h"
+#include "feature/relay/circuitbuild_relay.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/crypto/relay_crypto.h"
+
+#include "core/or/cell_st.h"
+#include "core/or/circuit_st.h"
+#include "core/or/extend_info_st.h"
+#include "core/or/or_circuit_st.h"
+
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/onion.h"
+#include "core/or/relay.h"
+
+#include "feature/nodelist/nodelist.h"
+
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+
+/* Before replying to an extend cell, check the state of the circuit
+ * <b>circ</b>, and the configured tor mode.
+ *
+ * <b>circ</b> must not be NULL.
+ *
+ * If the state and mode are valid, return 0.
+ * Otherwise, if they are invalid, log a protocol warning, and return -1.
+ */
+STATIC int
+circuit_extend_state_valid_helper(const struct circuit_t *circ)
+{
+  if (!server_mode(get_options())) {
+    circuitbuild_warn_client_extend();
+    return -1;
+  }
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  if (circ->n_chan) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "n_chan already set. Bug/attack. Closing.");
+    return -1;
+  }
+
+  if (circ->n_hop) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "conn to next hop already launched. Bug/attack. Closing.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Make sure the extend cell <b>ec</b> has an ed25519 link specifier.
+ *
+ * First, check that the RSA node id is valid.
+ * If the node id is valid, add the ed25519 link specifier (if required),
+ * and return 0.
+ *
+ * Otherwise, if the node id is invalid, log a protocol warning,
+ * and return -1.(And do not modify the extend cell.)
+ *
+ * Must be called before circuit_extend_lspec_valid_helper().
+ */
+STATIC int
+circuit_extend_add_ed25519_helper(struct extend_cell_t *ec)
+{
+  IF_BUG_ONCE(!ec) {
+    return -1;
+  }
+
+  /* Check if they asked us for 0000..0000. We support using
+   * an empty fingerprint for the first hop (e.g. for a bridge relay),
+   * but we don't want to let clients send us extend cells for empty
+   * fingerprints -- a) because it opens the user up to a mitm attack,
+   * and b) because it lets an attacker force the relay to hold open a
+   * new TLS connection for each extend request. */
+  if (tor_digest_is_zero((const char*)ec->node_id)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend without specifying an id_digest.");
+    return -1;
+  }
+
+  /* Fill in ed_pubkey if it was not provided and we can infer it from
+   * our networkstatus */
+  if (ed25519_public_key_is_zero(&ec->ed_pubkey)) {
+    const node_t *node = node_get_by_id((const char*)ec->node_id);
+    const ed25519_public_key_t *node_ed_id = NULL;
+    if (node &&
+        node_supports_ed25519_link_authentication(node, 1) &&
+        (node_ed_id = node_get_ed25519_id(node))) {
+      ed25519_pubkey_copy(&ec->ed_pubkey, node_ed_id);
+    }
+  }
+
+  return 0;
+}
+
+/* Check if the address and port in the tor_addr_port_t <b>ap</b> are valid,
+ * and are allowed by the current ExtendAllowPrivateAddresses config.
+ *
+ * If they are valid, return true.
+ * Otherwise, if they are invalid, return false.
+ *
+ * If <b>log_zero_addrs</b> is true, log warnings about zero addresses at
+ * <b>log_level</b>. If <b>log_internal_addrs</b> is true, log warnings about
+ * internal addresses at <b>log_level</b>.
+ */
+static bool
+circuit_extend_addr_port_is_valid(const struct tor_addr_port_t *ap,
+                                  bool log_zero_addrs, bool log_internal_addrs,
+                                  int log_level)
+{
+  /* It's safe to print the family. But we don't want to print the address,
+   * unless specifically configured to do so. (Zero addresses aren't sensitive,
+   * But some internal addresses might be.)*/
+
+  if (!tor_addr_port_is_valid_ap(ap, 0)) {
+    if (log_zero_addrs) {
+      log_fn(log_level, LD_PROTOCOL,
+             "Client asked me to extend to a zero destination port or "
+             "%s address '%s'.",
+             fmt_addr_family(&ap->addr), safe_str(fmt_addrport_ap(ap)));
+    }
+    return false;
+  }
+
+  if (tor_addr_is_internal(&ap->addr, 0) &&
+      !get_options()->ExtendAllowPrivateAddresses) {
+    if (log_internal_addrs) {
+      log_fn(log_level, LD_PROTOCOL,
+             "Client asked me to extend to a private %s address '%s'.",
+             fmt_addr_family(&ap->addr),
+             safe_str(fmt_and_decorate_addr(&ap->addr)));
+    }
+    return false;
+  }
+
+  return true;
+}
+
+/* Before replying to an extend cell, check the link specifiers in the extend
+ * cell <b>ec</b>, which was received on the circuit <b>circ</b>.
+ *
+ * If they are valid, return 0.
+ * Otherwise, if they are invalid, log a protocol warning, and return -1.
+ *
+ * Must be called after circuit_extend_add_ed25519_helper().
+ */
+STATIC int
+circuit_extend_lspec_valid_helper(const struct extend_cell_t *ec,
+                                  const struct circuit_t *circ)
+{
+  IF_BUG_ONCE(!ec) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  /* Check the addresses, without logging */
+  const int ipv4_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                                           false, false, 0);
+  const int ipv6_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                                           false, false, 0);
+  /* We need at least one valid address */
+  if (!ipv4_valid && !ipv6_valid) {
+    /* Now, log the invalid addresses at protocol warning level */
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                      true, true, LOG_PROTOCOL_WARN);
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                      true, true, LOG_PROTOCOL_WARN);
+    /* And fail */
+    return -1;
+  } else if (!ipv4_valid) {
+    /* Always log unexpected internal addresses, but go on to use the other
+     * valid address */
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                      false, true, LOG_PROTOCOL_WARN);
+  } else if (!ipv6_valid) {
+    circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                      false, true, LOG_PROTOCOL_WARN);
+  }
+
+  IF_BUG_ONCE(circ->magic != OR_CIRCUIT_MAGIC) {
+    return -1;
+  }
+
+  const channel_t *p_chan = CONST_TO_OR_CIRCUIT(circ)->p_chan;
+
+  IF_BUG_ONCE(!p_chan) {
+    return -1;
+  }
+
+  /* Next, check if we're being asked to connect to the hop that the
+   * extend cell came from. There isn't any reason for that, and it can
+   * assist circular-path attacks. */
+  if (tor_memeq(ec->node_id, p_chan->identity_digest, DIGEST_LEN)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop.");
+    return -1;
+  }
+
+  /* Check the previous hop Ed25519 ID too */
+  if (! ed25519_public_key_is_zero(&ec->ed_pubkey) &&
+      ed25519_pubkey_eq(&ec->ed_pubkey, &p_chan->ed25519_identity)) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Client asked me to extend back to the previous hop "
+           "(by Ed25519 ID).");
+    return -1;
+  }
+
+  return 0;
+}
+
+/* If possible, return a supported, non-NULL IP address.
+ *
+ * If both addresses are supported and non-NULL, choose one uniformly at
+ * random.
+ *
+ * If we have an IPv6-only extend, but IPv6 is not supported, returns NULL.
+ * If both addresses are NULL, also returns NULL. */
+STATIC const tor_addr_port_t *
+circuit_choose_ip_ap_for_extend(const tor_addr_port_t *ipv4_ap,
+                                const tor_addr_port_t *ipv6_ap)
+{
+  const bool ipv6_supported = router_can_extend_over_ipv6(get_options());
+
+  /* If IPv6 is not supported, we can't use the IPv6 address. */
+  if (!ipv6_supported) {
+    ipv6_ap = NULL;
+  }
+
+  /* If there is no IPv6 address, IPv4 is always supported.
+   * Until clients include IPv6 ORPorts, and most relays support IPv6,
+   * this is the most common case. */
+  if (!ipv6_ap) {
+    return ipv4_ap;
+  }
+
+  /* If there is no IPv4 address, return the (possibly NULL) IPv6 address. */
+  if (!ipv4_ap) {
+    return ipv6_ap;
+  }
+
+  /* Now we have an IPv4 and an IPv6 address, and IPv6 is supported.
+   * So make an IPv6 connection at random, with probability 1 in N.
+   *   1 means "always IPv6 (and no IPv4)"
+   *   2 means "equal probability of IPv4 or IPv6"
+   *   ... (and so on) ...
+   *   (UINT_MAX - 1) means "almost always IPv4 (and almost never IPv6)"
+   * To disable IPv6, set ipv6_supported to 0.
+   */
+#define IPV6_CONNECTION_ONE_IN_N 2
+
+  bool choose_ipv6 = crypto_fast_rng_one_in_n(get_thread_fast_rng(),
+                                              IPV6_CONNECTION_ONE_IN_N);
+  if (choose_ipv6) {
+    return ipv6_ap;
+  } else {
+    return ipv4_ap;
+  }
+}
+
+/* When there is no open channel for an extend cell <b>ec</b>, set up the
+ * circuit <b>circ</b> to wait for a new connection.
+ *
+ * If <b>should_launch</b> is true, open a new connection. (Otherwise, we are
+ * already waiting for a new connection to the same relay.)
+ *
+ * Check if IPv6 extends are supported by our current configuration. If they
+ * are, new connections may be made over IPv4 or IPv6. (IPv4 connections are
+ * always supported.)
+ */
+STATIC void
+circuit_open_connection_for_extend(const struct extend_cell_t *ec,
+                                   struct circuit_t *circ,
+                                   int should_launch)
+{
+  /* We have to check circ first, so we can close it on all other failures */
+  IF_BUG_ONCE(!circ) {
+    /* We can't mark a NULL circuit for close. */
+    return;
+  }
+
+  /* Now we know that circ is not NULL */
+  IF_BUG_ONCE(!ec) {
+    circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+    return;
+  }
+
+  /* Check the addresses, without logging */
+  const int ipv4_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv4,
+                                                           false, false, 0);
+  const int ipv6_valid = circuit_extend_addr_port_is_valid(&ec->orport_ipv6,
+                                                           false, false, 0);
+
+  IF_BUG_ONCE(!ipv4_valid && !ipv6_valid) {
+    /* circuit_extend_lspec_valid_helper() should have caught this */
+    circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+    return;
+  }
+
+  const tor_addr_port_t *chosen_ap = circuit_choose_ip_ap_for_extend(
+                                        ipv4_valid ? &ec->orport_ipv4 : NULL,
+                                        ipv6_valid ? &ec->orport_ipv6 : NULL);
+  if (!chosen_ap) {
+    /* An IPv6-only extend, but IPv6 is not supported */
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Received IPv6-only extend, but we don't have an IPv6 ORPort.");
+    circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+    return;
+  }
+
+  circ->n_hop = extend_info_new(NULL /*nickname*/,
+                                (const char*)ec->node_id,
+                                &ec->ed_pubkey,
+                                NULL, /*onion_key*/
+                                NULL, /*curve25519_key*/
+                                &chosen_ap->addr,
+                                chosen_ap->port);
+
+  circ->n_chan_create_cell = tor_memdup(&ec->create_cell,
+                                        sizeof(ec->create_cell));
+
+  circuit_set_state(circ, CIRCUIT_STATE_CHAN_WAIT);
+
+  if (should_launch) {
+    /* we should try to open a connection */
+    channel_t *n_chan = channel_connect_for_circuit(
+                                                &circ->n_hop->addr,
+                                                circ->n_hop->port,
+                                                circ->n_hop->identity_digest,
+                                                &circ->n_hop->ed_identity);
+    if (!n_chan) {
+      log_info(LD_CIRC,"Launching n_chan failed. Closing circuit.");
+      circuit_mark_for_close(circ, END_CIRC_REASON_CONNECTFAILED);
+      return;
+    }
+    log_debug(LD_CIRC,"connecting in progress (or finished). Good.");
+  }
+}
+
+/** Take the 'extend' <b>cell</b>, pull out addr/port plus the onion
+ * skin and identity digest for the next hop. If we're already connected,
+ * pass the onion skin to the next hop using a create cell; otherwise
+ * launch a new OR connection, and <b>circ</b> will notice when the
+ * connection succeeds or fails.
+ *
+ * Return -1 if we want to warn and tear down the circuit, else return 0.
+ */
+int
+circuit_extend(struct cell_t *cell, struct circuit_t *circ)
+{
+  channel_t *n_chan;
+  relay_header_t rh;
+  extend_cell_t ec;
+  const char *msg = NULL;
+  int should_launch = 0;
+
+  IF_BUG_ONCE(!cell) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  if (circuit_extend_state_valid_helper(circ) < 0)
+    return -1;
+
+  relay_header_unpack(&rh, cell->payload);
+
+  if (extend_cell_parse(&ec, rh.command,
+                        cell->payload+RELAY_HEADER_SIZE,
+                        rh.length) < 0) {
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Can't parse extend cell. Closing circuit.");
+    return -1;
+  }
+
+  if (circuit_extend_add_ed25519_helper(&ec) < 0)
+    return -1;
+
+  if (circuit_extend_lspec_valid_helper(&ec, circ) < 0)
+    return -1;
+
+  /* Check the addresses, without logging */
+  const int ipv4_valid = circuit_extend_addr_port_is_valid(&ec.orport_ipv4,
+                                                           false, false, 0);
+  const int ipv6_valid = circuit_extend_addr_port_is_valid(&ec.orport_ipv6,
+                                                           false, false, 0);
+  IF_BUG_ONCE(!ipv4_valid && !ipv6_valid) {
+    /* circuit_extend_lspec_valid_helper() should have caught this */
+    return -1;
+  }
+
+  n_chan = channel_get_for_extend((const char*)ec.node_id,
+                                  &ec.ed_pubkey,
+                                  ipv4_valid ? &ec.orport_ipv4.addr : NULL,
+                                  ipv6_valid ? &ec.orport_ipv6.addr : NULL,
+                                  &msg,
+                                  &should_launch);
+
+  if (!n_chan) {
+    /* We can't use fmt_addr*() twice in the same function call,
+     * because it uses a static buffer. */
+    log_debug(LD_CIRC|LD_OR, "Next router IPv4 (%s): %s.",
+              fmt_addrport_ap(&ec.orport_ipv4),
+              msg ? msg : "????");
+    log_debug(LD_CIRC|LD_OR, "Next router IPv6 (%s).",
+              fmt_addrport_ap(&ec.orport_ipv6));
+
+    circuit_open_connection_for_extend(&ec, circ, should_launch);
+
+    /* return success. The onion/circuit/etc will be taken care of
+     * automatically (may already have been) whenever n_chan reaches
+     * OR_CONN_STATE_OPEN.
+     */
+    return 0;
+  } else {
+    /* Connection is already established.
+     * So we need to extend the circuit to the next hop. */
+    tor_assert(!circ->n_hop);
+    circ->n_chan = n_chan;
+    log_debug(LD_CIRC,
+              "n_chan is %s.",
+              channel_get_canonical_remote_descr(n_chan));
+
+    if (circuit_deliver_create_cell(circ, &ec.create_cell, 1) < 0)
+      return -1;
+
+    return 0;
+  }
+}
+
+/** On a relay, accept a create cell, initialise a circuit, and send a
+ * created cell back.
+ *
+ * Given:
+ *   - a response payload consisting of:
+ *     - the <b>created_cell</b> and
+ *     - an optional <b>rend_circ_nonce</b>, and
+ *   - <b>keys</b> of length <b>keys_len</b>, which must be
+ *     CPATH_KEY_MATERIAL_LEN;
+ * then:
+ *   - initialize the circuit <b>circ</b>'s cryptographic material,
+ *   - set the circuit's state to open, and
+ *   - send a created cell back on that circuit.
+ *
+ * If we haven't found our ORPorts reachable yet, and the channel meets the
+ * necessary conditions, mark the relevant ORPorts as reachable.
+ *
+ * Returns -1 if cell or circuit initialisation fails.
+ */
+int
+onionskin_answer(struct or_circuit_t *circ,
+                 const created_cell_t *created_cell,
+                 const char *keys, size_t keys_len,
+                 const uint8_t *rend_circ_nonce)
+{
+  cell_t cell;
+
+  IF_BUG_ONCE(!circ) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!created_cell) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!keys) {
+    return -1;
+  }
+
+  IF_BUG_ONCE(!rend_circ_nonce) {
+    return -1;
+  }
+
+  tor_assert(keys_len == CPATH_KEY_MATERIAL_LEN);
+
+  if (created_cell_format(&cell, created_cell) < 0) {
+    log_warn(LD_BUG,"couldn't format created cell (type=%d, len=%d).",
+             (int)created_cell->cell_type, (int)created_cell->handshake_len);
+    return -1;
+  }
+  cell.circ_id = circ->p_circ_id;
+
+  circuit_set_state(TO_CIRCUIT(circ), CIRCUIT_STATE_OPEN);
+
+  log_debug(LD_CIRC,"init digest forward 0x%.8x, backward 0x%.8x.",
+            (unsigned int)get_uint32(keys),
+            (unsigned int)get_uint32(keys+20));
+  if (relay_crypto_init(&circ->crypto, keys, keys_len, 0, 0)<0) {
+    log_warn(LD_BUG,"Circuit initialization failed.");
+    return -1;
+  }
+
+  memcpy(circ->rend_circ_nonce, rend_circ_nonce, DIGEST_LEN);
+
+  int used_create_fast = (created_cell->cell_type == CELL_CREATED_FAST);
+
+  append_cell_to_circuit_queue(TO_CIRCUIT(circ),
+                               circ->p_chan, &cell, CELL_DIRECTION_IN, 0);
+  log_debug(LD_CIRC,"Finished sending '%s' cell.",
+            used_create_fast ? "created_fast" : "created");
+
+  /* Ignore the local bit when ExtendAllowPrivateAddresses is set:
+   * it violates the assumption that private addresses are local.
+   * Also, many test networks run on local addresses, and
+   * TestingTorNetwork sets ExtendAllowPrivateAddresses. */
+  if ((!channel_is_local(circ->p_chan)
+       || get_options()->ExtendAllowPrivateAddresses)
+      && !channel_is_outgoing(circ->p_chan)) {
+    /* record that we could process create cells from a non-local conn
+     * that we didn't initiate; presumably this means that create cells
+     * can reach us too. */
+    router_orport_found_reachable();
+  }
+
+  return 0;
+}
diff --git a/src/feature/relay/circuitbuild_relay.h b/src/feature/relay/circuitbuild_relay.h
new file mode 100644
index 0000000000..0783161538
--- /dev/null
+++ b/src/feature/relay/circuitbuild_relay.h
@@ -0,0 +1,87 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file circuitbuild_relay.h
+ * @brief Header for feature/relay/circuitbuild_relay.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_CIRCUITBUILD_RELAY_H
+#define TOR_FEATURE_RELAY_CIRCUITBUILD_RELAY_H
+
+#include "lib/cc/torint.h"
+#include "lib/log/log.h"
+
+#include "app/config/config.h"
+
+struct cell_t;
+struct created_cell_t;
+
+struct circuit_t;
+struct or_circuit_t;
+struct extend_cell_t;
+
+/* Log a protocol warning about getting an extend cell on a client. */
+static inline void
+circuitbuild_warn_client_extend(void)
+{
+  log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+         "Got an extend cell, but running as a client. Closing.");
+}
+
+#ifdef HAVE_MODULE_RELAY
+
+int circuit_extend(struct cell_t *cell, struct circuit_t *circ);
+
+int onionskin_answer(struct or_circuit_t *circ,
+                     const struct created_cell_t *created_cell,
+                     const char *keys, size_t keys_len,
+                     const uint8_t *rend_circ_nonce);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+static inline int
+circuit_extend(struct cell_t *cell, struct circuit_t *circ)
+{
+  (void)cell;
+  (void)circ;
+  circuitbuild_warn_client_extend();
+  return -1;
+}
+
+static inline int
+onionskin_answer(struct or_circuit_t *circ,
+                 const struct created_cell_t *created_cell,
+                 const char *keys, size_t keys_len,
+                 const uint8_t *rend_circ_nonce)
+{
+  (void)circ;
+  (void)created_cell;
+  (void)keys;
+  (void)keys_len;
+  (void)rend_circ_nonce;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#ifdef TOR_UNIT_TESTS
+
+STATIC int circuit_extend_state_valid_helper(const struct circuit_t *circ);
+STATIC int circuit_extend_add_ed25519_helper(struct extend_cell_t *ec);
+STATIC int circuit_extend_lspec_valid_helper(const struct extend_cell_t *ec,
+                                             const struct circuit_t *circ);
+STATIC const tor_addr_port_t * circuit_choose_ip_ap_for_extend(
+                                             const tor_addr_port_t *ipv4_ap,
+                                             const tor_addr_port_t *ipv6_ap);
+STATIC void circuit_open_connection_for_extend(const struct extend_cell_t *ec,
+                                               struct circuit_t *circ,
+                                               int should_launch);
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_CIRCUITBUILD_RELAY_H) */
diff --git a/src/feature/relay/dns.c b/src/feature/relay/dns.c
index e20a39482f..b83bd9b758 100644
--- a/src/feature/relay/dns.c
+++ b/src/feature/relay/dns.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -59,7 +59,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/relay/dns.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
@@ -146,9 +146,9 @@ cached_resolve_hash(cached_resolve_t *a)
 }
 
 HT_PROTOTYPE(cache_map, cached_resolve_t, node, cached_resolve_hash,
-             cached_resolves_eq)
+             cached_resolves_eq);
 HT_GENERATE2(cache_map, cached_resolve_t, node, cached_resolve_hash,
-             cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_)
+             cached_resolves_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** Initialize the DNS cache. */
 static void
@@ -268,22 +268,6 @@ has_dns_init_failed(void)
   return nameserver_config_failed;
 }
 
-/** Helper: Given a TTL from a DNS response, determine what TTL to give the
- * OP that asked us to resolve it, and how long to cache that record
- * ourselves. */
-uint32_t
-dns_clip_ttl(uint32_t ttl)
-{
-  /* This logic is a defense against "DefectTor" DNS-based traffic
-   * confirmation attacks, as in https://nymity.ch/tor-dns/tor-dns.pdf .
-   * We only give two values: a "low" value and a "high" value.
-   */
-  if (ttl < MIN_DNS_TTL_AT_EXIT)
-    return MIN_DNS_TTL_AT_EXIT;
-  else
-    return MAX_DNS_TTL_AT_EXIT;
-}
-
 /** Helper: free storage held by an entry in the DNS cache. */
 static void
 free_cached_resolve_(cached_resolve_t *r)
@@ -521,7 +505,7 @@ send_resolved_cell,(edge_connection_t *conn, uint8_t answer_type,
   uint32_t ttl;
 
   buf[0] = answer_type;
-  ttl = dns_clip_ttl(conn->address_ttl);
+  ttl = clip_dns_ttl(conn->address_ttl);
 
   switch (answer_type)
     {
@@ -593,7 +577,7 @@ send_resolved_hostname_cell,(edge_connection_t *conn,
   size_t namelen = strlen(hostname);
 
   tor_assert(namelen < 256);
-  ttl = dns_clip_ttl(conn->address_ttl);
+  ttl = clip_dns_ttl(conn->address_ttl);
 
   buf[0] = RESOLVED_TYPE_HOSTNAME;
   buf[1] = (uint8_t)namelen;
@@ -987,25 +971,6 @@ assert_connection_edge_not_dns_pending(edge_connection_t *conn)
 #endif /* 1 */
 }
 
-/** Log an error and abort if any connection waiting for a DNS resolve is
- * corrupted. */
-void
-assert_all_pending_dns_resolves_ok(void)
-{
-  pending_connection_t *pend;
-  cached_resolve_t **resolve;
-
-  HT_FOREACH(resolve, cache_map, &cache_root) {
-    for (pend = (*resolve)->pending_connections;
-         pend;
-         pend = pend->next) {
-      assert_connection_ok(TO_CONN(pend->conn), 0);
-      tor_assert(!SOCKET_OK(pend->conn->base_.s));
-      tor_assert(!connection_in_array(TO_CONN(pend->conn)));
-    }
-  }
-}
-
 /** Remove <b>conn</b> from the list of connections waiting for conn-\>address.
  */
 void
@@ -1063,7 +1028,7 @@ connection_dns_remove(edge_connection_t *conn)
  * the resolve for <b>address</b> itself, and remove any cached results for
  * <b>address</b> from the cache.
  */
-MOCK_IMPL(void,
+MOCK_IMPL(STATIC void,
 dns_cancel_pending_resolve,(const char *address))
 {
   pending_connection_t *pend;
@@ -1338,7 +1303,7 @@ make_pending_resolve_cached(cached_resolve_t *resolve)
         resolve->ttl_hostname < ttl)
       ttl = resolve->ttl_hostname;
 
-    set_expiry(new_resolve, time(NULL) + dns_clip_ttl(ttl));
+    set_expiry(new_resolve, time(NULL) + clip_dns_ttl(ttl));
   }
 
   assert_cache_ok();
@@ -1360,6 +1325,42 @@ evdns_err_is_transient(int err)
   }
 }
 
+/**
+ * Return number of configured nameservers in <b>the_evdns_base</b>.
+ */
+size_t
+number_of_configured_nameservers(void)
+{
+  return evdns_base_count_nameservers(the_evdns_base);
+}
+
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+/**
+ * Return address of configured nameserver in <b>the_evdns_base</b>
+ * at index <b>idx</b>.
+ */
+tor_addr_t *
+configured_nameserver_address(const size_t idx)
+{
+ struct sockaddr_storage sa;
+ ev_socklen_t sa_len = sizeof(sa);
+
+ if (evdns_base_get_nameserver_addr(the_evdns_base, (int)idx,
+                                    (struct sockaddr *)&sa,
+                                    sa_len) > 0) {
+   tor_addr_t *tor_addr = tor_malloc(sizeof(tor_addr_t));
+   if (tor_addr_from_sockaddr(tor_addr,
+                              (const struct sockaddr *)&sa,
+                              NULL) == 0) {
+     return tor_addr;
+   }
+   tor_free(tor_addr);
+ }
+
+ return NULL;
+}
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
 /** Configure eventdns nameservers if force is true, or if the configuration
  * has changed since the last time we called this function, or if we failed on
  * our last attempt.  On Unix, this reads from /etc/resolv.conf or
@@ -1391,16 +1392,23 @@ configure_nameservers(int force)
   evdns_set_log_fn(evdns_log_cb);
   if (conf_fname) {
     log_debug(LD_FS, "stat()ing %s", conf_fname);
-    if (stat(sandbox_intern_string(conf_fname), &st)) {
+    int missing_resolv_conf = 0;
+    int stat_res = stat(sandbox_intern_string(conf_fname), &st);
+
+    if (stat_res) {
       log_warn(LD_EXIT, "Unable to stat resolver configuration in '%s': %s",
                conf_fname, strerror(errno));
-      goto err;
-    }
-    if (!force && resolv_conf_fname && !strcmp(conf_fname,resolv_conf_fname)
+      missing_resolv_conf = 1;
+    } else if (!force && resolv_conf_fname &&
+               !strcmp(conf_fname,resolv_conf_fname)
         && st.st_mtime == resolv_conf_mtime) {
       log_info(LD_EXIT, "No change to '%s'", conf_fname);
       return 0;
     }
+
+    if (stat_res == 0 && st.st_size == 0)
+      missing_resolv_conf = 1;
+
     if (nameservers_configured) {
       evdns_base_search_clear(the_evdns_base);
       evdns_base_clear_nameservers_and_suspend(the_evdns_base);
@@ -1413,20 +1421,34 @@ configure_nameservers(int force)
           sandbox_intern_string("/etc/hosts"));
     }
 #endif /* defined(DNS_OPTION_HOSTSFILE) && defined(USE_LIBSECCOMP) */
-    log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
-    if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
-        sandbox_intern_string(conf_fname)))) {
-      log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers in '%s' (%d)",
-               conf_fname, conf_fname, r);
-      goto err;
-    }
-    if (evdns_base_count_nameservers(the_evdns_base) == 0) {
-      log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.", conf_fname);
-      goto err;
+
+    if (!missing_resolv_conf) {
+      log_info(LD_EXIT, "Parsing resolver configuration in '%s'", conf_fname);
+      if ((r = evdns_base_resolv_conf_parse(the_evdns_base, flags,
+          sandbox_intern_string(conf_fname)))) {
+        log_warn(LD_EXIT, "Unable to parse '%s', or no nameservers "
+                          "in '%s' (%d)", conf_fname, conf_fname, r);
+
+        if (r != 6) // "r = 6" means "no DNS servers were in resolv.conf" -
+          goto err; // in which case we expect libevent to add 127.0.0.1 as
+                    // fallback.
+      }
+      if (evdns_base_count_nameservers(the_evdns_base) == 0) {
+        log_warn(LD_EXIT, "Unable to find any nameservers in '%s'.",
+                 conf_fname);
+      }
+
+      tor_free(resolv_conf_fname);
+      resolv_conf_fname = tor_strdup(conf_fname);
+      resolv_conf_mtime = st.st_mtime;
+    } else {
+      log_warn(LD_EXIT, "Could not read your DNS config from '%s' - "
+                        "please investigate your DNS configuration. "
+                        "This is possibly a problem. Meanwhile, falling"
+                        " back to local DNS at 127.0.0.1.", conf_fname);
+      evdns_base_nameserver_ip_add(the_evdns_base, "127.0.0.1");
     }
-    tor_free(resolv_conf_fname);
-    resolv_conf_fname = tor_strdup(conf_fname);
-    resolv_conf_mtime = st.st_mtime;
+
     if (nameservers_configured)
       evdns_base_resume(the_evdns_base);
   }
@@ -1569,12 +1591,17 @@ evdns_callback(int result, char type, int count, int ttl, void *addresses,
     } else if (type == DNS_IPv6_AAAA && count) {
       char answer_buf[TOR_ADDR_BUF_LEN];
       char *escaped_address;
+      const char *ip_str;
       struct in6_addr *addrs = addresses;
       tor_addr_from_in6(&addr, &addrs[0]);
-      tor_inet_ntop(AF_INET6, &addrs[0], answer_buf, sizeof(answer_buf));
+      ip_str = tor_inet_ntop(AF_INET6, &addrs[0], answer_buf,
+                             sizeof(answer_buf));
       escaped_address = esc_for_log(string_address);
 
-      if (answer_is_wildcarded(answer_buf)) {
+      if (BUG(ip_str == NULL)) {
+        log_warn(LD_EXIT, "tor_inet_ntop() failed!");
+        result = DNS_ERR_NOTEXIST;
+      } else if (answer_is_wildcarded(answer_buf)) {
         log_debug(LD_EXIT, "eventdns said that %s resolves to ISP-hijacked "
                   "address %s; treating as a failure.",
                   safe_str(escaped_address),
@@ -1841,6 +1868,7 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl,
                               void *addresses, void *arg)
 {
   (void)ttl;
+  const char *ip_str;
   ++n_wildcard_requests;
   if (result == DNS_ERR_NONE && count) {
     char *string_address = arg;
@@ -1850,16 +1878,22 @@ evdns_wildcard_check_callback(int result, char type, int count, int ttl,
       for (i = 0; i < count; ++i) {
         char answer_buf[INET_NTOA_BUF_LEN+1];
         struct in_addr in;
+        int ntoa_res;
         in.s_addr = addrs[i];
-        tor_inet_ntoa(&in, answer_buf, sizeof(answer_buf));
-        wildcard_increment_answer(answer_buf);
+        ntoa_res = tor_inet_ntoa(&in, answer_buf, sizeof(answer_buf));
+        tor_assert_nonfatal(ntoa_res >= 0);
+        if (ntoa_res > 0)
+          wildcard_increment_answer(answer_buf);
       }
     } else if (type == DNS_IPv6_AAAA) {
       const struct in6_addr *addrs = addresses;
       for (i = 0; i < count; ++i) {
         char answer_buf[TOR_ADDR_BUF_LEN+1];
-        tor_inet_ntop(AF_INET6, &addrs[i], answer_buf, sizeof(answer_buf));
-        wildcard_increment_answer(answer_buf);
+        ip_str = tor_inet_ntop(AF_INET6, &addrs[i], answer_buf,
+                               sizeof(answer_buf));
+        tor_assert_nonfatal(ip_str);
+        if (ip_str)
+          wildcard_increment_answer(answer_buf);
       }
     }
 
@@ -1994,12 +2028,12 @@ dns_launch_correctness_checks(void)
 
   /* Wait a while before launching requests for test addresses, so we can
    * get the results from checking for wildcarding. */
-  if (! launch_event)
+  if (!launch_event)
     launch_event = tor_evtimer_new(tor_libevent_get_base(),
                                    launch_test_addresses, NULL);
   timeout.tv_sec = 30;
   timeout.tv_usec = 0;
-  if (evtimer_add(launch_event, &timeout)<0) {
+  if (evtimer_add(launch_event, &timeout) < 0) {
     log_warn(LD_BUG, "Couldn't add timer for checking for dns hijacking");
   }
 }
@@ -2131,7 +2165,7 @@ dns_cache_handle_oom(time_t now, size_t min_remove_bytes)
     total_bytes_removed += bytes_removed;
 
     /* Increase time_inc by a reasonable fraction. */
-    time_inc += (MAX_DNS_TTL_AT_EXIT / 4);
+    time_inc += (MAX_DNS_TTL / 4);
   } while (total_bytes_removed < min_remove_bytes);
 
   return total_bytes_removed;
diff --git a/src/feature/relay/dns.h b/src/feature/relay/dns.h
index e4474cdf43..120b75bf8d 100644
--- a/src/feature/relay/dns.h
+++ b/src/feature/relay/dns.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,29 +12,14 @@
 #ifndef TOR_DNS_H
 #define TOR_DNS_H
 
-/** Lowest value for DNS ttl that a server will give. */
-#define MIN_DNS_TTL_AT_EXIT (5*60)
-/** Highest value for DNS ttl that a server will give. */
-#define MAX_DNS_TTL_AT_EXIT (60*60)
-
-/** How long do we keep DNS cache entries before purging them (regardless of
- * their TTL)? */
-#define MAX_DNS_ENTRY_AGE (3*60*60)
-/** How long do we cache/tell clients to cache DNS records when no TTL is
- * known? */
-#define DEFAULT_DNS_TTL (30*60)
+#ifdef HAVE_MODULE_RELAY
 
 int dns_init(void);
 int has_dns_init_failed(void);
-void dns_free_all(void);
-uint32_t dns_clip_ttl(uint32_t ttl);
 int dns_reset(void);
 void connection_dns_remove(edge_connection_t *conn);
 void assert_connection_edge_not_dns_pending(edge_connection_t *conn);
-void assert_all_pending_dns_resolves_ok(void);
-MOCK_DECL(void,dns_cancel_pending_resolve,(const char *question));
 int dns_resolve(edge_connection_t *exitconn);
-void dns_launch_correctness_checks(void);
 int dns_seems_to_be_broken(void);
 int dns_seems_to_be_broken_for_ipv6(void);
 void dns_reset_correctness_checks(void);
@@ -42,9 +27,57 @@ size_t dns_cache_total_allocation(void);
 void dump_dns_mem_usage(int severity);
 size_t dns_cache_handle_oom(time_t now, size_t min_remove_bytes);
 
+/* These functions are only used within the feature/relay module, and don't
+ * need stubs. */
+void dns_free_all(void);
+void dns_launch_correctness_checks(void);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define dns_init() (0)
+#define dns_seems_to_be_broken() (0)
+#define has_dns_init_failed() (0)
+#define dns_cache_total_allocation() (0)
+
+#define dns_reset_correctness_checks() STMT_NIL
+
+#define assert_connection_edge_not_dns_pending(conn) \
+  ((void)(conn))
+#define dump_dns_mem_usage(severity)\
+  ((void)(severity))
+#define dns_cache_handle_oom(now, bytes) \
+  ((void)(now), (void)(bytes), 0)
+
+#define connection_dns_remove(conn) \
+  STMT_BEGIN                        \
+  (void)(conn);                     \
+  tor_assert_nonfatal_unreached();  \
+  STMT_END
+
+static inline int
+dns_reset(void)
+{
+  return 0;
+}
+static inline int
+dns_resolve(edge_connection_t *exitconn)
+{
+  (void)exitconn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #ifdef DNS_PRIVATE
 #include "feature/relay/dns_structs.h"
 
+size_t number_of_configured_nameservers(void);
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+tor_addr_t *configured_nameserver_address(const size_t idx);
+#endif
+
+MOCK_DECL(STATIC void,dns_cancel_pending_resolve,(const char *question));
 MOCK_DECL(STATIC int,dns_resolve_impl,(edge_connection_t *exitconn,
 int is_resolve,or_circuit_t *oncirc, char **hostname_out,
 int *made_connection_pending_out, cached_resolve_t **resolve_out));
@@ -69,4 +102,3 @@ launch_resolve,(cached_resolve_t *resolve));
 #endif /* defined(DNS_PRIVATE) */
 
 #endif /* !defined(TOR_DNS_H) */
-
diff --git a/src/feature/relay/dns_structs.h b/src/feature/relay/dns_structs.h
index e128746f81..27a791b9b3 100644
--- a/src/feature/relay/dns_structs.h
+++ b/src/feature/relay/dns_structs.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,8 @@
 #ifndef TOR_DNS_STRUCTS_H
 #define TOR_DNS_STRUCTS_H
 
+#include "ext/ht.h"
+
 /** Longest hostname we're willing to resolve. */
 #define MAX_ADDRESSLEN 256
 
@@ -99,4 +101,3 @@ typedef struct cached_resolve_t {
 } cached_resolve_t;
 
 #endif /* !defined(TOR_DNS_STRUCTS_H) */
-
diff --git a/src/feature/relay/ext_orport.c b/src/feature/relay/ext_orport.c
index 136aee3084..2cf30262f5 100644
--- a/src/feature/relay/ext_orport.c
+++ b/src/feature/relay/ext_orport.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,7 @@
 #include "core/or/or.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
@@ -90,7 +90,7 @@ connection_ext_or_transition(or_connection_t *conn)
 
   conn->base_.type = CONN_TYPE_OR;
   TO_CONN(conn)->state = 0; // set the state to a neutral value
-  control_event_or_conn_status(conn, OR_CONN_EVENT_NEW, 0);
+  connection_or_event_status(conn, OR_CONN_EVENT_NEW, 0);
   connection_tls_start_handshake(conn, 1);
 }
 
@@ -606,7 +606,7 @@ connection_ext_or_process_inbuf(or_connection_t *or_conn)
                                              command->body, command->len) < 0)
         goto err;
     } else {
-      log_notice(LD_NET,"Got Extended ORPort command we don't regognize (%u).",
+      log_notice(LD_NET,"Got Extended ORPort command we don't recognize (%u).",
                  command->cmd);
     }
 
@@ -656,6 +656,77 @@ connection_ext_or_start_auth(or_connection_t *or_conn)
   return 0;
 }
 
+/** Global map between Extended ORPort identifiers and OR
+ *  connections. */
+static digestmap_t *orconn_ext_or_id_map = NULL;
+
+/** Remove the Extended ORPort identifier of <b>conn</b> from the
+ *  global identifier list. Also, clear the identifier from the
+ *  connection itself. */
+void
+connection_or_remove_from_ext_or_id_map(or_connection_t *conn)
+{
+  or_connection_t *tmp;
+  if (!orconn_ext_or_id_map)
+    return;
+  if (!conn->ext_or_conn_id)
+    return;
+
+  tmp = digestmap_remove(orconn_ext_or_id_map, conn->ext_or_conn_id);
+  if (!tor_digest_is_zero(conn->ext_or_conn_id))
+    tor_assert(tmp == conn);
+
+  memset(conn->ext_or_conn_id, 0, EXT_OR_CONN_ID_LEN);
+}
+
+#ifdef TOR_UNIT_TESTS
+/** Return the connection whose ext_or_id is <b>id</b>. Return NULL if no such
+ * connection is found. */
+or_connection_t *
+connection_or_get_by_ext_or_id(const char *id)
+{
+  if (!orconn_ext_or_id_map)
+    return NULL;
+  return digestmap_get(orconn_ext_or_id_map, id);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** Deallocate the global Extended ORPort identifier list */
+void
+connection_or_clear_ext_or_id_map(void)
+{
+  digestmap_free(orconn_ext_or_id_map, NULL);
+  orconn_ext_or_id_map = NULL;
+}
+
+/** Creates an Extended ORPort identifier for <b>conn</b> and deposits
+ *  it into the global list of identifiers. */
+void
+connection_or_set_ext_or_identifier(or_connection_t *conn)
+{
+  char random_id[EXT_OR_CONN_ID_LEN];
+  or_connection_t *tmp;
+
+  if (!orconn_ext_or_id_map)
+    orconn_ext_or_id_map = digestmap_new();
+
+  /* Remove any previous identifiers: */
+  if (conn->ext_or_conn_id && !tor_digest_is_zero(conn->ext_or_conn_id))
+      connection_or_remove_from_ext_or_id_map(conn);
+
+  do {
+    crypto_rand(random_id, sizeof(random_id));
+  } while (digestmap_get(orconn_ext_or_id_map, random_id));
+
+  if (!conn->ext_or_conn_id)
+    conn->ext_or_conn_id = tor_malloc_zero(EXT_OR_CONN_ID_LEN);
+
+  memcpy(conn->ext_or_conn_id, random_id, EXT_OR_CONN_ID_LEN);
+
+  tmp = digestmap_set(orconn_ext_or_id_map, random_id, conn);
+  tor_assert(!tmp);
+}
+
 /** Free any leftover allocated memory of the ext_orport.c subsystem. */
 void
 ext_orport_free_all(void)
diff --git a/src/feature/relay/ext_orport.h b/src/feature/relay/ext_orport.h
index 7313ebd03d..416c358397 100644
--- a/src/feature/relay/ext_orport.h
+++ b/src/feature/relay/ext_orport.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file ext_orport.h
+ * @brief Header for ext_orport.c
+ **/
+
 #ifndef EXT_ORPORT_H
 #define EXT_ORPORT_H
 
@@ -26,26 +31,56 @@
 #define EXT_OR_CONN_STATE_FLUSHING 5
 #define EXT_OR_CONN_STATE_MAX_ 5
 
-int connection_ext_or_start_auth(or_connection_t *or_conn);
-
-ext_or_cmd_t *ext_or_cmd_new(uint16_t len);
+#ifdef HAVE_MODULE_RELAY
 
-#define ext_or_cmd_free(cmd)                            \
-  FREE_AND_NULL(ext_or_cmd_t, ext_or_cmd_free_, (cmd))
+int connection_ext_or_start_auth(or_connection_t *or_conn);
 
-void ext_or_cmd_free_(ext_or_cmd_t *cmd);
 void connection_or_set_ext_or_identifier(or_connection_t *conn);
 void connection_or_remove_from_ext_or_id_map(or_connection_t *conn);
 void connection_or_clear_ext_or_id_map(void);
-or_connection_t *connection_or_get_by_ext_or_id(const char *id);
-
 int connection_ext_or_finished_flushing(or_connection_t *conn);
 int connection_ext_or_process_inbuf(or_connection_t *or_conn);
+char *get_ext_or_auth_cookie_file_name(void);
 
+/* (No stub needed for these: they are only called within feature/relay.) */
 int init_ext_or_cookie_authentication(int is_enabled);
-char *get_ext_or_auth_cookie_file_name(void);
 void ext_orport_free_all(void);
 
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+static inline int
+connection_ext_or_start_auth(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+static inline int
+connection_ext_or_finished_flushing(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+static inline int
+connection_ext_or_process_inbuf(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+#define connection_or_set_ext_or_identifier(conn) \
+  ((void)(conn))
+#define connection_or_remove_from_ext_or_id_map(conn) \
+  ((void)(conn))
+#define connection_or_clear_ext_or_id_map() \
+  STMT_NIL
+
+#define get_ext_or_auth_cookie_file_name() \
+  (NULL)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #ifdef EXT_ORPORT_PRIVATE
 STATIC int connection_write_ext_or_command(connection_t *conn,
                                            uint16_t command,
@@ -55,9 +90,11 @@ STATIC int handle_client_auth_nonce(const char *client_nonce,
                          size_t client_nonce_len,
                          char **client_hash_out,
                          char **reply_out, size_t *reply_len_out);
+
 #ifdef TOR_UNIT_TESTS
 extern uint8_t *ext_or_auth_cookie;
 extern int ext_or_auth_cookie_is_set;
+or_connection_t *connection_or_get_by_ext_or_id(const char *id);
 #endif
 #endif /* defined(EXT_ORPORT_PRIVATE) */
 
diff --git a/src/feature/relay/feature_relay.md b/src/feature/relay/feature_relay.md
new file mode 100644
index 0000000000..a7f0c2153a
--- /dev/null
+++ b/src/feature/relay/feature_relay.md
@@ -0,0 +1,4 @@
+@dir /feature/relay
+@brief feature/relay: Relay-specific code
+
+(There is also a bunch of relay-specific code in other modules.)
diff --git a/src/feature/relay/include.am b/src/feature/relay/include.am
new file mode 100644
index 0000000000..84bb1ff35e
--- /dev/null
+++ b/src/feature/relay/include.am
@@ -0,0 +1,46 @@
+
+# Legacy shared relay code: migrate to the relay module over time
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/relay/onion_queue.c		\
+	src/feature/relay/relay_find_addr.c	\
+	src/feature/relay/router.c
+
+# The Relay module.
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+MODULE_RELAY_SOURCES = 						\
+	src/feature/relay/circuitbuild_relay.c			\
+	src/feature/relay/dns.c					\
+	src/feature/relay/ext_orport.c		                \
+	src/feature/relay/routermode.c				\
+	src/feature/relay/relay_config.c			\
+	src/feature/relay/relay_handshake.c			\
+	src/feature/relay/relay_periodic.c			\
+	src/feature/relay/relay_sys.c				\
+	src/feature/relay/routerkeys.c				\
+	src/feature/relay/selftest.c				\
+	src/feature/relay/transport_config.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/relay/circuitbuild_relay.h		\
+	src/feature/relay/dns.h				\
+	src/feature/relay/dns_structs.h			\
+	src/feature/relay/ext_orport.h			\
+	src/feature/relay/onion_queue.h			\
+	src/feature/relay/relay_config.h		\
+	src/feature/relay/relay_handshake.h		\
+	src/feature/relay/relay_periodic.h		\
+	src/feature/relay/relay_sys.h			\
+	src/feature/relay/relay_find_addr.h		\
+	src/feature/relay/router.h			\
+	src/feature/relay/routerkeys.h			\
+	src/feature/relay/routermode.h			\
+	src/feature/relay/selftest.h			\
+	src/feature/relay/transport_config.h
+
+if BUILD_MODULE_RELAY
+LIBTOR_APP_A_SOURCES += $(MODULE_RELAY_SOURCES)
+else
+LIBTOR_APP_A_STUB_SOURCES += src/feature/relay/relay_stub.c
+endif
diff --git a/src/feature/relay/onion_queue.c b/src/feature/relay/onion_queue.c
index 696905cf5e..3cbaa65d28 100644
--- a/src/feature/relay/onion_queue.c
+++ b/src/feature/relay/onion_queue.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,10 +49,12 @@ typedef struct onion_queue_t {
 /** 5 seconds on the onion queue til we just send back a destroy */
 #define ONIONQUEUE_WAIT_CUTOFF 5
 
+TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t);
+typedef struct onion_queue_head_t onion_queue_head_t;
+
 /** Array of queues of circuits waiting for CPU workers. An element is NULL
  * if that queue is empty.*/
-static TOR_TAILQ_HEAD(onion_queue_head_t, onion_queue_t)
-              ol_list[MAX_ONION_HANDSHAKE_TYPE+1] =
+static onion_queue_head_t ol_list[MAX_ONION_HANDSHAKE_TYPE+1] =
 { TOR_TAILQ_HEAD_INITIALIZER(ol_list[0]), /* tap */
   TOR_TAILQ_HEAD_INITIALIZER(ol_list[1]), /* fast */
   TOR_TAILQ_HEAD_INITIALIZER(ol_list[2]), /* ntor */
@@ -212,10 +214,12 @@ num_ntors_per_tap(void)
 #define MIN_NUM_NTORS_PER_TAP 1
 #define MAX_NUM_NTORS_PER_TAP 100000
 
-  return networkstatus_get_param(NULL, "NumNTorsPerTAP",
-                                 DEFAULT_NUM_NTORS_PER_TAP,
-                                 MIN_NUM_NTORS_PER_TAP,
-                                 MAX_NUM_NTORS_PER_TAP);
+  int result = networkstatus_get_param(NULL, "NumNTorsPerTAP",
+                                       DEFAULT_NUM_NTORS_PER_TAP,
+                                       MIN_NUM_NTORS_PER_TAP,
+                                       MAX_NUM_NTORS_PER_TAP);
+  tor_assert(result > 0);
+  return result;
 }
 
 /** Choose which onion queue we'll pull from next. If one is empty choose
diff --git a/src/feature/relay/onion_queue.h b/src/feature/relay/onion_queue.h
index 0df921e057..08379b2c00 100644
--- a/src/feature/relay/onion_queue.h
+++ b/src/feature/relay/onion_queue.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ int onion_num_pending(uint16_t handshake_type);
 void onion_pending_remove(or_circuit_t *circ);
 void clear_pending_onions(void);
 
-#endif
+#endif /* !defined(TOR_ONION_QUEUE_H) */
diff --git a/src/feature/relay/relay_config.c b/src/feature/relay/relay_config.c
new file mode 100644
index 0000000000..fac6a2f577
--- /dev/null
+++ b/src/feature/relay/relay_config.c
@@ -0,0 +1,1444 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_config.c
+ * @brief Code to interpret the user's configuration of Tor's relay module.
+ **/
+
+#include "orconfig.h"
+#define RELAY_CONFIG_PRIVATE
+#include "feature/relay/relay_config.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confmgt.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/geoip/geoip.h"
+#include "lib/meminfo/meminfo.h"
+#include "lib/osinfo/uname.h"
+#include "lib/process/setuid.h"
+
+/* Required for dirinfo_type_t in or_options_t */
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "core/mainloop/connection.h"
+#include "core/mainloop/cpuworker.h"
+#include "core/mainloop/mainloop.h"
+#include "core/or/connection_or.h"
+#include "core/or/port_cfg_st.h"
+
+#include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/nickname.h"
+#include "feature/stats/geoip_stats.h"
+#include "feature/stats/predict_ports.h"
+#include "feature/stats/rephist.h"
+
+#include "feature/dirauth/authmode.h"
+
+#include "feature/dircache/consdiffmgr.h"
+#include "feature/relay/dns.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+
+/** Contents of most recently read DirPortFrontPage file. */
+static char *global_dirfrontpagecontents = NULL;
+
+/* Copied from config.c, we will refactor later in 29211. */
+#define REJECT(arg) \
+  STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
+#if defined(__GNUC__) && __GNUC__ <= 3
+#define COMPLAIN(args...) \
+  STMT_BEGIN log_warn(LD_CONFIG, args); STMT_END
+#else
+#define COMPLAIN(args, ...)                                     \
+  STMT_BEGIN log_warn(LD_CONFIG, args, ##__VA_ARGS__); STMT_END
+#endif /* defined(__GNUC__) && __GNUC__ <= 3 */
+
+/* Used in the various options_transition_affects* functions. */
+#define YES_IF_CHANGED_BOOL(opt) \
+  if (!CFG_EQ_BOOL(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_INT(opt) \
+  if (!CFG_EQ_INT(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_STRING(opt) \
+  if (!CFG_EQ_STRING(old_options, new_options, opt)) return 1;
+#define YES_IF_CHANGED_LINELIST(opt) \
+  if (!CFG_EQ_LINELIST(old_options, new_options, opt)) return 1;
+
+/** Return the contents of our frontpage string, or NULL if not configured. */
+MOCK_IMPL(const char*,
+relay_get_dirportfrontpage, (void))
+{
+  return global_dirfrontpagecontents;
+}
+
+/** Release all memory and resources held by global relay configuration
+ * structures.
+ */
+void
+relay_config_free_all(void)
+{
+  tor_free(global_dirfrontpagecontents);
+}
+
+/** Return the bandwidthrate that we are going to report to the authorities
+ * based on the config options. */
+uint32_t
+relay_get_effective_bwrate(const or_options_t *options)
+{
+  uint64_t bw = options->BandwidthRate;
+  if (bw > options->MaxAdvertisedBandwidth)
+    bw = options->MaxAdvertisedBandwidth;
+  if (options->RelayBandwidthRate > 0 && bw > options->RelayBandwidthRate)
+    bw = options->RelayBandwidthRate;
+  /* config_ensure_bandwidth_cap() makes sure that this cast can't overflow. */
+  return (uint32_t)bw;
+}
+
+/** Return the bandwidthburst that we are going to report to the authorities
+ * based on the config options. */
+uint32_t
+relay_get_effective_bwburst(const or_options_t *options)
+{
+  uint64_t bw = options->BandwidthBurst;
+  if (options->RelayBandwidthBurst > 0 && bw > options->RelayBandwidthBurst)
+    bw = options->RelayBandwidthBurst;
+  /* config_ensure_bandwidth_cap() makes sure that this cast can't overflow. */
+  return (uint32_t)bw;
+}
+
+/** Warn for every Extended ORPort port in <b>ports</b> that is on a
+ *  publicly routable address. */
+void
+port_warn_nonlocal_ext_orports(const smartlist_t *ports, const char *portname)
+{
+  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
+    if (port->type != CONN_TYPE_EXT_OR_LISTENER)
+      continue;
+    if (port->is_unix_addr)
+      continue;
+    /* XXX maybe warn even if address is RFC1918? */
+    if (!tor_addr_is_internal(&port->addr, 1)) {
+      log_warn(LD_CONFIG, "You specified a public address '%s' for %sPort. "
+               "This is not advised; this address is supposed to only be "
+               "exposed on localhost so that your pluggable transport "
+               "proxies can connect to it.",
+               fmt_addrport(&port->addr, port->port), portname);
+    }
+  } SMARTLIST_FOREACH_END(port);
+}
+
+/** Given a list of <b>port_cfg_t</b> in <b>ports</b>, check them for internal
+ * consistency and warn as appropriate.  On Unix-based OSes, set
+ * *<b>n_low_ports_out</b> to the number of sub-1024 ports we will be
+ * binding, and warn if we may be unable to re-bind after hibernation. */
+static int
+check_server_ports(const smartlist_t *ports,
+                   const or_options_t *options,
+                   int *n_low_ports_out)
+{
+  if (BUG(!ports))
+    return -1;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!n_low_ports_out))
+    return -1;
+
+  int n_orport_advertised = 0;
+  int n_orport_advertised_ipv4 = 0;
+  int n_orport_listeners = 0;
+  int n_dirport_advertised = 0;
+  int n_dirport_listeners = 0;
+  int n_low_port = 0;
+  int r = 0;
+
+  SMARTLIST_FOREACH_BEGIN(ports, const port_cfg_t *, port) {
+    if (port->type == CONN_TYPE_DIR_LISTENER) {
+      if (! port->server_cfg.no_advertise)
+        ++n_dirport_advertised;
+      if (! port->server_cfg.no_listen)
+        ++n_dirport_listeners;
+    } else if (port->type == CONN_TYPE_OR_LISTENER) {
+      if (! port->server_cfg.no_advertise) {
+        ++n_orport_advertised;
+        if (port_binds_ipv4(port))
+          ++n_orport_advertised_ipv4;
+      }
+      if (! port->server_cfg.no_listen)
+        ++n_orport_listeners;
+    } else {
+      continue;
+    }
+#ifndef _WIN32
+    if (!port->server_cfg.no_listen && port->port < 1024)
+      ++n_low_port;
+#endif
+  } SMARTLIST_FOREACH_END(port);
+
+  if (n_orport_advertised && !n_orport_listeners) {
+    log_warn(LD_CONFIG, "We are advertising an ORPort, but not actually "
+             "listening on one.");
+    r = -1;
+  }
+  if (n_orport_listeners && !n_orport_advertised) {
+    log_warn(LD_CONFIG, "We are listening on an ORPort, but not advertising "
+             "any ORPorts. This will keep us from building a %s "
+             "descriptor, and make us impossible to use.",
+             options->BridgeRelay ? "bridge" : "router");
+    r = -1;
+  }
+  if (n_dirport_advertised && !n_dirport_listeners) {
+    log_warn(LD_CONFIG, "We are advertising a DirPort, but not actually "
+             "listening on one.");
+    r = -1;
+  }
+  if (n_dirport_advertised > 1) {
+    log_warn(LD_CONFIG, "Can't advertise more than one DirPort.");
+    r = -1;
+  }
+  if (n_orport_advertised && !n_orport_advertised_ipv4 &&
+      !options->BridgeRelay) {
+    log_warn(LD_CONFIG, "Configured public relay to listen only on an IPv6 "
+             "address. Tor needs to listen on an IPv4 address too.");
+    r = -1;
+  }
+
+  if (n_low_port && options->AccountingMax &&
+      (!have_capability_support() || options->KeepBindCapabilities == 0)) {
+    const char *extra = "";
+    if (options->KeepBindCapabilities == 0 && have_capability_support())
+      extra = ", and you have disabled KeepBindCapabilities.";
+    log_warn(LD_CONFIG,
+          "You have set AccountingMax to use hibernation. You have also "
+          "chosen a low DirPort or OrPort%s."
+          "This combination can make Tor stop "
+          "working when it tries to re-attach the port after a period of "
+          "hibernation. Please choose a different port or turn off "
+          "hibernation unless you know this combination will work on your "
+          "platform.", extra);
+  }
+
+  if (n_low_ports_out)
+    *n_low_ports_out = n_low_port;
+
+  return r;
+}
+
+/** Parse all relay ports from <b>options</b>. On success, add parsed ports to
+ * <b>ports</b>, and return 0.  On failure, set *<b>msg</b> to a newly
+ * allocated string describing the problem, and return -1.
+ **/
+int
+port_parse_ports_relay(or_options_t *options,
+                  char **msg,
+                  smartlist_t *ports_out,
+                  int *have_low_ports_out)
+{
+  int retval = -1;
+  smartlist_t *ports = smartlist_new();
+  int n_low_ports = 0;
+
+  if (BUG(!options))
+    goto err;
+
+  if (BUG(!msg))
+    goto err;
+
+  if (BUG(!ports_out))
+    goto err;
+
+  if (BUG(!have_low_ports_out))
+    goto err;
+
+  if (options->ClientOnly) {
+    retval = 0;
+    goto err;
+  }
+
+  if (port_parse_config(ports,
+                        options->ORPort_lines,
+                        "OR", CONN_TYPE_OR_LISTENER,
+                        "0.0.0.0", 0,
+                        CL_PORT_SERVER_OPTIONS) < 0) {
+    *msg = tor_strdup("Invalid ORPort configuration");
+    goto err;
+  }
+  if (port_parse_config(ports,
+                        options->ExtORPort_lines,
+                        "ExtOR", CONN_TYPE_EXT_OR_LISTENER,
+                        "127.0.0.1", 0,
+                        CL_PORT_SERVER_OPTIONS|CL_PORT_WARN_NONLOCAL) < 0) {
+    *msg = tor_strdup("Invalid ExtORPort configuration");
+    goto err;
+  }
+  if (port_parse_config(ports,
+                        options->DirPort_lines,
+                        "Dir", CONN_TYPE_DIR_LISTENER,
+                        "0.0.0.0", 0,
+                        CL_PORT_SERVER_OPTIONS) < 0) {
+    *msg = tor_strdup("Invalid DirPort configuration");
+    goto err;
+  }
+
+  if (check_server_ports(ports, options, &n_low_ports) < 0) {
+    *msg = tor_strdup("Misconfigured server ports");
+    goto err;
+  }
+
+  smartlist_add_all(ports_out, ports);
+  smartlist_free(ports);
+  ports = NULL;
+  retval = 0;
+
+ err:
+  if (*have_low_ports_out < 0)
+    *have_low_ports_out = (n_low_ports > 0);
+  if (ports) {
+    SMARTLIST_FOREACH(ports, port_cfg_t *, p, port_cfg_free(p));
+    smartlist_free(ports);
+  }
+  return retval;
+}
+
+/** Update the relay *Port_set values in <b>options</b> from <b>ports</b>. */
+void
+port_update_port_set_relay(or_options_t *options,
+                      const smartlist_t *ports)
+{
+  if (BUG(!options))
+    return;
+
+  if (BUG(!ports))
+    return;
+
+  if (options->ClientOnly)
+    return;
+
+  /* Update the relay *Port_set options.  The !! here is to force a boolean
+   * out of an integer. */
+  options->ORPort_set =
+    !! port_count_real_listeners(ports, CONN_TYPE_OR_LISTENER, 0);
+  options->DirPort_set =
+    !! port_count_real_listeners(ports, CONN_TYPE_DIR_LISTENER, 0);
+  options->ExtORPort_set =
+    !! port_count_real_listeners(ports, CONN_TYPE_EXT_OR_LISTENER, 0);
+}
+
+/**
+ * Legacy validation function, which checks that the current OS is usable in
+ * relay mode, if options is set to a relay mode.
+ *
+ * Warns about OSes with potential issues. Does not set *<b>msg</b>.
+ * Always returns 0.
+ */
+int
+options_validate_relay_os(const or_options_t *old_options,
+                          or_options_t *options,
+                          char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!server_mode(options))
+    return 0;
+
+  const char *uname = get_uname();
+
+  if (!strcmpstart(uname, "Windows 95") ||
+      !strcmpstart(uname, "Windows 98") ||
+      !strcmpstart(uname, "Windows Me")) {
+    log_warn(LD_CONFIG, "Tor is running as a server, but you are "
+        "running %s; this probably won't work. See "
+        "https://www.torproject.org/docs/faq.html#BestOSForRelay "
+        "for details.", uname);
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay info options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_info(const or_options_t *old_options,
+                            or_options_t *options,
+                            char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (options->Nickname == NULL) {
+    if (server_mode(options)) {
+      options->Nickname = tor_strdup(UNNAMED_ROUTER_NICKNAME);
+    }
+  } else {
+    if (!is_legal_nickname(options->Nickname)) {
+      tor_asprintf(msg,
+          "Nickname '%s', nicknames must be between 1 and 19 characters "
+          "inclusive, and must contain only the characters [a-zA-Z0-9].",
+          options->Nickname);
+      return -1;
+    }
+  }
+
+  if (server_mode(options) && !options->ContactInfo) {
+    log_warn(LD_CONFIG,
+             "Your ContactInfo config option is not set. Please strongly "
+             "consider setting it, so we can contact you if your relay is "
+             "misconfigured, end-of-life, or something else goes wrong. "
+             "It is also possible that your relay might get rejected from "
+             "the network due to a missing valid contact address.");
+  }
+
+  const char *ContactInfo = options->ContactInfo;
+  if (ContactInfo && !string_is_utf8(ContactInfo, strlen(ContactInfo)))
+    REJECT("ContactInfo config option must be UTF-8.");
+
+  return 0;
+}
+
+/** Parse an authority type from <b>options</b>-\>PublishServerDescriptor
+ * and write it to <b>options</b>-\>PublishServerDescriptor_. Treat "1"
+ * as "v3" unless BridgeRelay is 1, in which case treat it as "bridge".
+ * Treat "0" as "".
+ * Return 0 on success or -1 if not a recognized authority type (in which
+ * case the value of PublishServerDescriptor_ is undefined). */
+static int
+compute_publishserverdescriptor(or_options_t *options)
+{
+  smartlist_t *list = options->PublishServerDescriptor;
+  dirinfo_type_t *auth = &options->PublishServerDescriptor_;
+  *auth = NO_DIRINFO;
+  if (!list) /* empty list, answer is none */
+    return 0;
+  SMARTLIST_FOREACH_BEGIN(list, const char *, string) {
+    if (!strcasecmp(string, "v1"))
+      log_warn(LD_CONFIG, "PublishServerDescriptor v1 has no effect, because "
+                          "there are no v1 directory authorities anymore.");
+    else if (!strcmp(string, "1"))
+      if (options->BridgeRelay)
+        *auth |= BRIDGE_DIRINFO;
+      else
+        *auth |= V3_DIRINFO;
+    else if (!strcasecmp(string, "v2"))
+      log_warn(LD_CONFIG, "PublishServerDescriptor v2 has no effect, because "
+                          "there are no v2 directory authorities anymore.");
+    else if (!strcasecmp(string, "v3"))
+      *auth |= V3_DIRINFO;
+    else if (!strcasecmp(string, "bridge"))
+      *auth |= BRIDGE_DIRINFO;
+    else if (!strcasecmp(string, "hidserv"))
+      log_warn(LD_CONFIG,
+               "PublishServerDescriptor hidserv is invalid. See "
+               "PublishHidServDescriptors.");
+    else if (!strcasecmp(string, "") || !strcmp(string, "0"))
+      /* no authority */;
+    else
+      return -1;
+  } SMARTLIST_FOREACH_END(string);
+  return 0;
+}
+
+/**
+ * Validate the configured bridge distribution method from a BridgeDistribution
+ * config line.
+ *
+ * The input <b>bd</b>, is a string taken from the BridgeDistribution config
+ * line (if present).  If the option wasn't set, return 0 immediately.  The
+ * BridgeDistribution option is then validated.  Currently valid, recognised
+ * options are:
+ *
+ * - "none"
+ * - "any"
+ * - "https"
+ * - "email"
+ * - "moat"
+ *
+ * If the option string is unrecognised, a warning will be logged and 0 is
+ * returned.  If the option string contains an invalid character, -1 is
+ * returned.
+ **/
+STATIC int
+check_bridge_distribution_setting(const char *bd)
+{
+  if (bd == NULL)
+    return 0;
+
+  const char *RECOGNIZED[] = {
+    "none", "any", "https", "email", "moat"
+  };
+  unsigned i;
+  for (i = 0; i < ARRAY_LENGTH(RECOGNIZED); ++i) {
+    if (!strcasecmp(bd, RECOGNIZED[i]))
+      return 0;
+  }
+
+  const char *cp = bd;
+  //  Method = (KeywordChar | "_") +
+  while (TOR_ISALNUM(*cp) || *cp == '-' || *cp == '_')
+    ++cp;
+
+  if (*cp == 0) {
+    log_warn(LD_CONFIG, "Unrecognized BridgeDistribution value %s. I'll "
+           "assume you know what you are doing...", escaped(bd));
+    return 0; // we reached the end of the string; all is well
+  } else {
+    return -1; // we found a bad character in the string.
+  }
+}
+
+/**
+ * Legacy validation/normalization function for the bridge relay options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_publish_server(const or_options_t *old_options,
+                                or_options_t *options,
+                                char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (compute_publishserverdescriptor(options) < 0) {
+    tor_asprintf(msg, "Unrecognized value in PublishServerDescriptor");
+    return -1;
+  }
+
+  if ((options->BridgeRelay
+        || options->PublishServerDescriptor_ & BRIDGE_DIRINFO)
+      && (options->PublishServerDescriptor_ & V3_DIRINFO)) {
+    REJECT("Bridges are not supposed to publish router descriptors to the "
+           "directory authorities. Please correct your "
+           "PublishServerDescriptor line.");
+  }
+
+  if (options->BridgeDistribution) {
+    if (!options->BridgeRelay) {
+      REJECT("You set BridgeDistribution, but you didn't set BridgeRelay!");
+    }
+    if (check_bridge_distribution_setting(options->BridgeDistribution) < 0) {
+      REJECT("Invalid BridgeDistribution value.");
+    }
+  }
+
+  if (options->PublishServerDescriptor)
+    SMARTLIST_FOREACH(options->PublishServerDescriptor, const char *, pubdes, {
+      if (!strcmp(pubdes, "1") || !strcmp(pubdes, "0"))
+        if (smartlist_len(options->PublishServerDescriptor) > 1) {
+          COMPLAIN("You have passed a list of multiple arguments to the "
+                   "PublishServerDescriptor option that includes 0 or 1. "
+                   "0 or 1 should only be used as the sole argument. "
+                   "This configuration will be rejected in a future release.");
+          break;
+        }
+    });
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay padding options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_padding(const or_options_t *old_options,
+                               or_options_t *options,
+                               char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (!server_mode(options))
+    return 0;
+
+  if (options->ConnectionPadding != -1) {
+    REJECT("Relays must use 'auto' for the ConnectionPadding setting.");
+  }
+
+  if (options->ReducedConnectionPadding != 0) {
+    REJECT("Relays cannot set ReducedConnectionPadding. ");
+  }
+
+  if (options->CircuitPadding == 0) {
+    REJECT("Relays cannot set CircuitPadding to 0. ");
+  }
+
+  if (options->ReducedCircuitPadding == 1) {
+    REJECT("Relays cannot set ReducedCircuitPadding. ");
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay bandwidth options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_bandwidth(const or_options_t *old_options,
+                                 or_options_t *options,
+                                 char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  /* 31851: the tests expect us to validate bandwidths, even when we are not
+  * in relay mode. */
+  if (config_ensure_bandwidth_cap(&options->MaxAdvertisedBandwidth,
+                           "MaxAdvertisedBandwidth", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->RelayBandwidthRate,
+                           "RelayBandwidthRate", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->RelayBandwidthBurst,
+                           "RelayBandwidthBurst", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->PerConnBWRate,
+                           "PerConnBWRate", msg) < 0)
+    return -1;
+  if (config_ensure_bandwidth_cap(&options->PerConnBWBurst,
+                           "PerConnBWBurst", msg) < 0)
+    return -1;
+
+  if (options->RelayBandwidthRate && !options->RelayBandwidthBurst)
+    options->RelayBandwidthBurst = options->RelayBandwidthRate;
+  if (options->RelayBandwidthBurst && !options->RelayBandwidthRate)
+    options->RelayBandwidthRate = options->RelayBandwidthBurst;
+
+  if (server_mode(options)) {
+    const unsigned required_min_bw =
+      public_server_mode(options) ?
+       RELAY_REQUIRED_MIN_BANDWIDTH : BRIDGE_REQUIRED_MIN_BANDWIDTH;
+    const char * const optbridge =
+      public_server_mode(options) ? "" : "bridge ";
+    if (options->BandwidthRate < required_min_bw) {
+      tor_asprintf(msg,
+                       "BandwidthRate is set to %d bytes/second. "
+                       "For %sservers, it must be at least %u.",
+                       (int)options->BandwidthRate, optbridge,
+                       required_min_bw);
+      return -1;
+    } else if (options->MaxAdvertisedBandwidth <
+               required_min_bw/2) {
+      tor_asprintf(msg,
+                       "MaxAdvertisedBandwidth is set to %d bytes/second. "
+                       "For %sservers, it must be at least %u.",
+                       (int)options->MaxAdvertisedBandwidth, optbridge,
+                       required_min_bw/2);
+      return -1;
+    }
+    if (options->RelayBandwidthRate &&
+      options->RelayBandwidthRate < required_min_bw) {
+      tor_asprintf(msg,
+                       "RelayBandwidthRate is set to %d bytes/second. "
+                       "For %sservers, it must be at least %u.",
+                       (int)options->RelayBandwidthRate, optbridge,
+                       required_min_bw);
+      return -1;
+    }
+  }
+
+  /* 31851: the tests expect us to validate bandwidths, even when we are not
+   * in relay mode. */
+  if (options->RelayBandwidthRate > options->RelayBandwidthBurst)
+    REJECT("RelayBandwidthBurst must be at least equal "
+           "to RelayBandwidthRate.");
+
+  /* if they set relaybandwidth* really high but left bandwidth*
+   * at the default, raise the defaults. */
+  if (options->RelayBandwidthRate > options->BandwidthRate)
+    options->BandwidthRate = options->RelayBandwidthRate;
+  if (options->RelayBandwidthBurst > options->BandwidthBurst)
+    options->BandwidthBurst = options->RelayBandwidthBurst;
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay bandwidth accounting
+ * options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_accounting(const or_options_t *old_options,
+                                  or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  /* 31851: the tests expect us to validate accounting, even when we are not
+   * in relay mode. */
+  if (accounting_parse_options(options, 1)<0)
+    REJECT("Failed to parse accounting options. See logs for details.");
+
+  if (options->AccountingMax) {
+    if (options->RendConfigLines && server_mode(options)) {
+      log_warn(LD_CONFIG, "Using accounting with a hidden service and an "
+               "ORPort is risky: your hidden service(s) and your public "
+               "address will all turn off at the same time, which may alert "
+               "observers that they are being run by the same party.");
+    } else if (config_count_key(options->RendConfigLines,
+                                "HiddenServiceDir") > 1) {
+      log_warn(LD_CONFIG, "Using accounting with multiple hidden services is "
+               "risky: they will all turn off at the same time, which may "
+               "alert observers that they are being run by the same party.");
+    }
+  }
+
+  options->AccountingRule = ACCT_MAX;
+  if (options->AccountingRule_option) {
+    if (!strcmp(options->AccountingRule_option, "sum"))
+      options->AccountingRule = ACCT_SUM;
+    else if (!strcmp(options->AccountingRule_option, "max"))
+      options->AccountingRule = ACCT_MAX;
+    else if (!strcmp(options->AccountingRule_option, "in"))
+      options->AccountingRule = ACCT_IN;
+    else if (!strcmp(options->AccountingRule_option, "out"))
+      options->AccountingRule = ACCT_OUT;
+    else
+      REJECT("AccountingRule must be 'sum', 'max', 'in', or 'out'");
+  }
+
+  return 0;
+}
+
+/** Verify whether lst is a list of strings containing valid-looking
+ * comma-separated nicknames, or NULL. Will normalise <b>lst</b> to prefix '$'
+ * to any nickname or fingerprint that needs it. Also splits comma-separated
+ * list elements into multiple elements. Return 0 on success.
+ * Warn and return -1 on failure.
+ */
+static int
+normalize_nickname_list(config_line_t **normalized_out,
+                        const config_line_t *lst, const char *name,
+                        char **msg)
+{
+  if (!lst)
+    return 0;
+
+  config_line_t *new_nicknames = NULL;
+  config_line_t **new_nicknames_next = &new_nicknames;
+
+  const config_line_t *cl;
+  for (cl = lst; cl; cl = cl->next) {
+    const char *line = cl->value;
+    if (!line)
+      continue;
+
+    int valid_line = 1;
+    smartlist_t *sl = smartlist_new();
+    smartlist_split_string(sl, line, ",",
+      SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK|SPLIT_STRIP_SPACE, 0);
+    SMARTLIST_FOREACH_BEGIN(sl, char *, s)
+    {
+      char *normalized = NULL;
+      if (!is_legal_nickname_or_hexdigest(s)) {
+        // check if first char is dollar
+        if (s[0] != '$') {
+          // Try again but with a dollar symbol prepended
+          char *prepended;
+          tor_asprintf(&prepended, "$%s", s);
+
+          if (is_legal_nickname_or_hexdigest(prepended)) {
+            // The nickname is valid when it's prepended, set it as the
+            // normalized version
+            normalized = prepended;
+          } else {
+            // Still not valid, free and fallback to error message
+            tor_free(prepended);
+          }
+        }
+
+        if (!normalized) {
+          tor_asprintf(msg, "Invalid nickname '%s' in %s line", s, name);
+          valid_line = 0;
+          break;
+        }
+      } else {
+        normalized = tor_strdup(s);
+      }
+
+      config_line_t *next = tor_malloc_zero(sizeof(*next));
+      next->key = tor_strdup(cl->key);
+      next->value = normalized;
+      next->next = NULL;
+
+      *new_nicknames_next = next;
+      new_nicknames_next = &next->next;
+    } SMARTLIST_FOREACH_END(s);
+
+    SMARTLIST_FOREACH(sl, char *, s, tor_free(s));
+    smartlist_free(sl);
+
+    if (!valid_line) {
+      config_free_lines(new_nicknames);
+      return -1;
+    }
+  }
+
+  *normalized_out = new_nicknames;
+
+  return 0;
+}
+
+#define ONE_MEGABYTE (UINT64_C(1) << 20)
+
+/* If we have less than 300 MB suggest disabling dircache */
+#define DIRCACHE_MIN_MEM_MB 300
+#define DIRCACHE_MIN_MEM_BYTES (DIRCACHE_MIN_MEM_MB*ONE_MEGABYTE)
+#define STRINGIFY(val) #val
+
+/** Create a warning message for emitting if we are a dircache but may not have
+ * enough system memory, or if we are not a dircache but probably should be.
+ * Return -1 when a message is returned in *msg*, else return 0. */
+STATIC int
+have_enough_mem_for_dircache(const or_options_t *options, size_t total_mem,
+                             char **msg)
+{
+  *msg = NULL;
+  /* XXX We should possibly be looking at MaxMemInQueues here
+   * unconditionally.  Or we should believe total_mem unconditionally. */
+  if (total_mem == 0) {
+    if (get_total_system_memory(&total_mem) < 0) {
+      total_mem = options->MaxMemInQueues >= SIZE_MAX ?
+        SIZE_MAX : (size_t)options->MaxMemInQueues;
+    }
+  }
+  if (options->DirCache) {
+    if (total_mem < DIRCACHE_MIN_MEM_BYTES) {
+      if (options->BridgeRelay) {
+        tor_asprintf(msg, "Running a Bridge with less than %d MB of memory "
+                       "is not recommended.", DIRCACHE_MIN_MEM_MB);
+      } else {
+        tor_asprintf(msg, "Being a directory cache (default) with less than "
+                       "%d MB of memory is not recommended and may consume "
+                       "most of the available resources. Consider disabling "
+                       "this functionality by setting the DirCache option "
+                       "to 0.", DIRCACHE_MIN_MEM_MB);
+      }
+    }
+  } else {
+    if (total_mem >= DIRCACHE_MIN_MEM_BYTES) {
+      *msg = tor_strdup("DirCache is disabled and we are configured as a "
+               "relay. We will not become a Guard.");
+    }
+  }
+  return *msg == NULL ? 0 : -1;
+}
+#undef STRINGIFY
+
+/**
+ * Legacy validation/normalization function for the relay mode options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_mode(const or_options_t *old_options,
+                            or_options_t *options,
+                            char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (server_mode(options) && options->RendConfigLines)
+    log_warn(LD_CONFIG,
+        "Tor is currently configured as a relay and a hidden service. "
+        "That's not very secure: you should probably run your hidden service "
+        "in a separate Tor process, at least -- see "
+        "https://trac.torproject.org/8742");
+
+  if (options->BridgeRelay && options->DirPort_set) {
+    log_warn(LD_CONFIG, "Can't set a DirPort on a bridge relay; disabling "
+             "DirPort");
+    config_free_lines(options->DirPort_lines);
+    options->DirPort_lines = NULL;
+    options->DirPort_set = 0;
+  }
+
+  if (options->DirPort_set && !options->DirCache) {
+    REJECT("DirPort configured but DirCache disabled. DirPort requires "
+           "DirCache.");
+  }
+
+  if (options->BridgeRelay && !options->DirCache) {
+    REJECT("We're a bridge but DirCache is disabled. BridgeRelay requires "
+           "DirCache.");
+  }
+
+  if (options->BridgeRelay == 1 && ! options->ORPort_set)
+    REJECT("BridgeRelay is 1, ORPort is not set. This is an invalid "
+           "combination.");
+
+  if (server_mode(options)) {
+    char *dircache_msg = NULL;
+    if (have_enough_mem_for_dircache(options, 0, &dircache_msg)) {
+      log_warn(LD_CONFIG, "%s", dircache_msg);
+      tor_free(dircache_msg);
+    }
+  }
+
+  if (options->MyFamily_lines && options->BridgeRelay) {
+    log_warn(LD_CONFIG, "Listing a family for a bridge relay is not "
+             "supported: it can reveal bridge fingerprints to censors. "
+             "You should also make sure you aren't listing this bridge's "
+             "fingerprint in any other MyFamily.");
+  }
+  if (options->MyFamily_lines && !options->ContactInfo) {
+    log_warn(LD_CONFIG, "MyFamily is set but ContactInfo is not configured. "
+             "ContactInfo should always be set when MyFamily option is too.");
+  }
+  if (normalize_nickname_list(&options->MyFamily,
+                              options->MyFamily_lines, "MyFamily", msg))
+    return -1;
+
+  if (options->ConstrainedSockets) {
+    if (options->DirPort_set) {
+      /* Providing cached directory entries while system TCP buffers are scarce
+       * will exacerbate the socket errors.  Suggest that this be disabled. */
+      COMPLAIN("You have requested constrained socket buffers while also "
+               "serving directory entries via DirPort.  It is strongly "
+               "suggested that you disable serving directory requests when "
+               "system TCP buffer resources are scarce.");
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Legacy validation/normalization function for the relay testing options
+ * in options. Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_relay_testing(const or_options_t *old_options,
+                               or_options_t *options,
+                               char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  if (options->SigningKeyLifetime < options->TestingSigningKeySlop*2)
+    REJECT("SigningKeyLifetime is too short.");
+  if (options->TestingLinkCertLifetime < options->TestingAuthKeySlop*2)
+    REJECT("LinkCertLifetime is too short.");
+  if (options->TestingAuthKeyLifetime < options->TestingLinkKeySlop*2)
+    REJECT("TestingAuthKeyLifetime is too short.");
+
+  return 0;
+}
+
+/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
+ * will require us to rotate the CPU and DNS workers; else return 0. */
+static int
+options_transition_affects_workers(const or_options_t *old_options,
+                                   const or_options_t *new_options)
+{
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_INT(NumCPUs);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_BOOL(ServerDNSSearchDomains);
+  YES_IF_CHANGED_BOOL(SafeLogging_);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(LogMessageDomains);
+  YES_IF_CHANGED_LINELIST(Logs);
+
+  if (server_mode(old_options) != server_mode(new_options) ||
+      public_server_mode(old_options) != public_server_mode(new_options) ||
+      dir_server_mode(old_options) != dir_server_mode(new_options))
+    return 1;
+
+  /* Nothing that changed matters. */
+  return 0;
+}
+
+/** Return 1 if any change from <b>old_options</b> to <b>new_options</b>
+ * will require us to generate a new descriptor; else return 0. */
+static int
+options_transition_affects_descriptor(const or_options_t *old_options,
+                                      const or_options_t *new_options)
+{
+  /* XXX We can be smarter here. If your DirPort isn't being
+   * published and you just turned it off, no need to republish. Etc. */
+
+  YES_IF_CHANGED_STRING(DataDirectory);
+  YES_IF_CHANGED_STRING(Nickname);
+  YES_IF_CHANGED_STRING(Address);
+  YES_IF_CHANGED_LINELIST(ExitPolicy);
+  YES_IF_CHANGED_BOOL(ExitRelay);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectPrivate);
+  YES_IF_CHANGED_BOOL(ExitPolicyRejectLocalInterfaces);
+  YES_IF_CHANGED_BOOL(IPv6Exit);
+  YES_IF_CHANGED_LINELIST(ORPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_LINELIST(DirPort_lines);
+  YES_IF_CHANGED_BOOL(ClientOnly);
+  YES_IF_CHANGED_BOOL(DisableNetwork);
+  YES_IF_CHANGED_BOOL(PublishServerDescriptor_);
+  YES_IF_CHANGED_STRING(ContactInfo);
+  YES_IF_CHANGED_STRING(BridgeDistribution);
+  YES_IF_CHANGED_LINELIST(MyFamily);
+  YES_IF_CHANGED_STRING(AccountingStart);
+  YES_IF_CHANGED_INT(AccountingMax);
+  YES_IF_CHANGED_INT(AccountingRule);
+  YES_IF_CHANGED_BOOL(DirCache);
+  YES_IF_CHANGED_BOOL(AssumeReachable);
+
+  if (relay_get_effective_bwrate(old_options) !=
+        relay_get_effective_bwrate(new_options) ||
+      relay_get_effective_bwburst(old_options) !=
+        relay_get_effective_bwburst(new_options) ||
+      public_server_mode(old_options) != public_server_mode(new_options))
+    return 1;
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay actions based on it. All of
+ * the things we do should survive being done repeatedly.  If present,
+ * <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  const int transition_affects_workers =
+    old_options && options_transition_affects_workers(old_options, options);
+
+  /* We want to reinit keys as needed before we do much of anything else:
+     keys are important, and other things can depend on them. */
+  if (transition_affects_workers ||
+      (authdir_mode_v3(options) && (!old_options ||
+                                    !authdir_mode_v3(old_options)))) {
+    if (init_keys() < 0) {
+      log_warn(LD_BUG,"Error initializing keys; exiting");
+      return -1;
+    }
+  }
+
+  if (server_mode(options)) {
+    static int cdm_initialized = 0;
+    if (cdm_initialized == 0) {
+      cdm_initialized = 1;
+      consdiffmgr_configure(NULL);
+      consdiffmgr_validate();
+    }
+  }
+
+  /* Check for transitions that need action. */
+  if (old_options) {
+    if (transition_affects_workers) {
+      log_info(LD_GENERAL,
+               "Worker-related options changed. Rotating workers.");
+      const int server_mode_turned_on =
+        server_mode(options) && !server_mode(old_options);
+      const int dir_server_mode_turned_on =
+        dir_server_mode(options) && !dir_server_mode(old_options);
+
+      if (server_mode_turned_on || dir_server_mode_turned_on) {
+        cpu_init();
+      }
+
+      if (server_mode_turned_on) {
+        ip_address_changed(0);
+        if (have_completed_a_circuit() || !any_predicted_circuits(time(NULL)))
+          inform_testing_reachability();
+      }
+      cpuworkers_rotate_keyinfo();
+    }
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay accounting actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_accounting(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  const or_options_t *options = get_options();
+
+  /* Set up accounting */
+  if (accounting_parse_options(options, 0)<0) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG,"Error in previously validated accounting options");
+    return -1;
+    // LCOV_EXCL_STOP
+  }
+  if (accounting_is_enabled(options))
+    configure_accounting(time(NULL));
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay bandwidth actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_bandwidth(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* Check for transitions that need action. */
+  if (old_options) {
+    if (options->PerConnBWRate != old_options->PerConnBWRate ||
+        options->PerConnBWBurst != old_options->PerConnBWBurst)
+      connection_or_update_token_buckets(get_connection_array(), options);
+
+    if (options->RelayBandwidthRate != old_options->RelayBandwidthRate ||
+        options->RelayBandwidthBurst != old_options->RelayBandwidthBurst)
+      connection_bucket_adjust(options);
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take bridge statistics actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_bridge_stats(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+/* How long should we delay counting bridge stats after becoming a bridge?
+ * We use this so we don't count clients who used our bridge thinking it is
+ * a relay. If you change this, don't forget to change the log message
+ * below. It's 4 hours (the time it takes to stop being used by clients)
+ * plus some extra time for clock skew. */
+#define RELAY_BRIDGE_STATS_DELAY (6 * 60 * 60)
+
+  /* Check for transitions that need action. */
+  if (old_options) {
+    if (! bool_eq(options->BridgeRelay, old_options->BridgeRelay)) {
+      int was_relay = 0;
+      if (options->BridgeRelay) {
+        time_t int_start = time(NULL);
+        if (config_lines_eq(old_options->ORPort_lines,options->ORPort_lines)) {
+          int_start += RELAY_BRIDGE_STATS_DELAY;
+          was_relay = 1;
+        }
+        geoip_bridge_stats_init(int_start);
+        log_info(LD_CONFIG, "We are acting as a bridge now.  Starting new "
+                 "GeoIP stats interval%s.", was_relay ? " in 6 "
+                 "hours from now" : "");
+      } else {
+        geoip_bridge_stats_term();
+        log_info(LD_GENERAL, "We are no longer acting as a bridge.  "
+                 "Forgetting GeoIP stats.");
+      }
+    }
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay statistics actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Sets <b>*print_notice_out</b> if we enabled stats, and need to print
+ * a stats log using options_act_relay_stats_msg().
+ *
+ * If loading the GeoIP file failed, sets DirReqStatistics and
+ * EntryStatistics to 0. This breaks the normalization/act ordering
+ * introduced in 29211.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_stats(const or_options_t *old_options,
+                        bool *print_notice_out)
+{
+  if (BUG(!print_notice_out))
+    return -1;
+
+  or_options_t *options = get_options_mutable();
+
+  if (options->CellStatistics || options->DirReqStatistics ||
+      options->EntryStatistics || options->ExitPortStatistics ||
+      options->ConnDirectionStatistics ||
+      options->HiddenServiceStatistics) {
+    time_t now = time(NULL);
+    int print_notice = 0;
+
+    if ((!old_options || !old_options->CellStatistics) &&
+        options->CellStatistics) {
+      rep_hist_buffer_stats_init(now);
+      print_notice = 1;
+    }
+    if ((!old_options || !old_options->DirReqStatistics) &&
+        options->DirReqStatistics) {
+      if (geoip_is_loaded(AF_INET)) {
+        geoip_dirreq_stats_init(now);
+        print_notice = 1;
+      } else {
+        /* disable statistics collection since we have no geoip file */
+        /* 29211: refactor to avoid the normalisation/act inversion */
+        options->DirReqStatistics = 0;
+        if (options->ORPort_set)
+          log_notice(LD_CONFIG, "Configured to measure directory request "
+                                "statistics, but no GeoIP database found. "
+                                "Please specify a GeoIP database using the "
+                                "GeoIPFile option.");
+      }
+    }
+    if ((!old_options || !old_options->EntryStatistics) &&
+        options->EntryStatistics && !should_record_bridge_info(options)) {
+      /* If we get here, we've started recording bridge info when we didn't
+       * do so before.  Note that "should_record_bridge_info()" will
+       * always be false at this point, because of the earlier block
+       * that cleared EntryStatistics when public_server_mode() was false.
+       * We're leaving it in as defensive programming. */
+      if (geoip_is_loaded(AF_INET) || geoip_is_loaded(AF_INET6)) {
+        geoip_entry_stats_init(now);
+        print_notice = 1;
+      } else {
+        options->EntryStatistics = 0;
+        log_notice(LD_CONFIG, "Configured to measure entry node "
+                              "statistics, but no GeoIP database found. "
+                              "Please specify a GeoIP database using the "
+                              "GeoIPFile option.");
+      }
+    }
+    if ((!old_options || !old_options->ExitPortStatistics) &&
+        options->ExitPortStatistics) {
+      rep_hist_exit_stats_init(now);
+      print_notice = 1;
+    }
+    if ((!old_options || !old_options->ConnDirectionStatistics) &&
+        options->ConnDirectionStatistics) {
+      rep_hist_conn_stats_init(now);
+    }
+    if ((!old_options || !old_options->HiddenServiceStatistics) &&
+        options->HiddenServiceStatistics) {
+      log_info(LD_CONFIG, "Configured to measure hidden service statistics.");
+      rep_hist_hs_stats_init(now);
+    }
+    if (print_notice)
+      *print_notice_out = 1;
+  }
+
+  /* If we used to have statistics enabled but we just disabled them,
+     stop gathering them.  */
+  if (old_options && old_options->CellStatistics &&
+      !options->CellStatistics)
+    rep_hist_buffer_stats_term();
+  if (old_options && old_options->DirReqStatistics &&
+      !options->DirReqStatistics)
+    geoip_dirreq_stats_term();
+  if (old_options && old_options->EntryStatistics &&
+      !options->EntryStatistics)
+    geoip_entry_stats_term();
+  if (old_options && old_options->HiddenServiceStatistics &&
+      !options->HiddenServiceStatistics)
+    rep_hist_hs_stats_term();
+  if (old_options && old_options->ExitPortStatistics &&
+      !options->ExitPortStatistics)
+    rep_hist_exit_stats_term();
+  if (old_options && old_options->ConnDirectionStatistics &&
+      !options->ConnDirectionStatistics)
+    rep_hist_conn_stats_term();
+
+  return 0;
+}
+
+/** Print a notice about relay/dirauth stats being enabled. */
+void
+options_act_relay_stats_msg(void)
+{
+  log_notice(LD_CONFIG, "Configured to measure statistics. Look for "
+             "the *-stats files that will first be written to the "
+             "data directory in 24 hours from now.");
+}
+
+/** Fetch the active option list, and take relay descriptor actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_desc(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* Since our options changed, we might need to regenerate and upload our
+   * server descriptor.
+   */
+  if (!old_options ||
+      options_transition_affects_descriptor(old_options, options))
+    mark_my_descriptor_dirty("config change");
+
+  return 0;
+}
+
+/** Fetch the active option list, and take relay DoS actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_dos(const or_options_t *old_options)
+{
+  const or_options_t *options = get_options();
+
+  /* DoS mitigation subsystem only applies to public relay. */
+  if (public_server_mode(options)) {
+    /* If we are configured as a relay, initialize the subsystem. Even on HUP,
+     * this is safe to call as it will load data from the current options
+     * or/and the consensus. */
+    dos_init();
+  } else if (old_options && public_server_mode(old_options)) {
+    /* Going from relay to non relay, clean it up. */
+    dos_free_all();
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take dirport actions based on
+ * it. All of the things we do should survive being done repeatedly. If
+ * present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_relay_dir(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  const or_options_t *options = get_options();
+
+  if (!public_server_mode(options))
+    return 0;
+
+  /* Load the webpage we're going to serve every time someone asks for '/' on
+     our DirPort. */
+  tor_free(global_dirfrontpagecontents);
+  if (options->DirPortFrontPage) {
+    global_dirfrontpagecontents =
+      read_file_to_str(options->DirPortFrontPage, 0, NULL);
+    if (!global_dirfrontpagecontents) {
+      log_warn(LD_CONFIG,
+               "DirPortFrontPage file '%s' not found. Continuing anyway.",
+               options->DirPortFrontPage);
+    }
+  }
+
+  return 0;
+}
diff --git a/src/feature/relay/relay_config.h b/src/feature/relay/relay_config.h
new file mode 100644
index 0000000000..c70c322d88
--- /dev/null
+++ b/src/feature/relay/relay_config.h
@@ -0,0 +1,196 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_config.h
+ * @brief Header for feature/relay/relay_config.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_CONFIG_H
+#define TOR_FEATURE_RELAY_RELAY_CONFIG_H
+
+struct or_options_t;
+
+#ifdef HAVE_MODULE_RELAY
+
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+struct smartlist_t;
+
+int options_validate_relay_mode(const struct or_options_t *old_options,
+                                struct or_options_t *options,
+                                char **msg);
+
+MOCK_DECL(const char*, relay_get_dirportfrontpage, (void));
+void relay_config_free_all(void);
+
+uint32_t relay_get_effective_bwrate(const struct or_options_t *options);
+uint32_t relay_get_effective_bwburst(const struct or_options_t *options);
+
+void port_warn_nonlocal_ext_orports(const struct smartlist_t *ports,
+                               const char *portname);
+
+int port_parse_ports_relay(struct or_options_t *options,
+                      char **msg,
+                      struct smartlist_t *ports_out,
+                      int *have_low_ports_out);
+void port_update_port_set_relay(struct or_options_t *options,
+                           const struct smartlist_t *ports);
+
+int options_validate_relay_os(const struct or_options_t *old_options,
+                              struct or_options_t *options,
+                              char **msg);
+
+int options_validate_relay_info(const struct or_options_t *old_options,
+                                struct or_options_t *options,
+                                char **msg);
+
+int options_validate_publish_server(const struct or_options_t *old_options,
+                                    struct or_options_t *options,
+                                    char **msg);
+
+int options_validate_relay_padding(const struct or_options_t *old_options,
+                                   struct or_options_t *options,
+                                   char **msg);
+
+int options_validate_relay_bandwidth(const struct or_options_t *old_options,
+                                     struct or_options_t *options,
+                                     char **msg);
+
+int options_validate_relay_accounting(const struct or_options_t *old_options,
+                                      struct or_options_t *options,
+                                      char **msg);
+
+int options_validate_relay_testing(const struct or_options_t *old_options,
+                                   struct or_options_t *options,
+                                   char **msg);
+
+int options_act_relay(const struct or_options_t *old_options);
+int options_act_relay_accounting(const struct or_options_t *old_options);
+int options_act_relay_bandwidth(const struct or_options_t *old_options);
+int options_act_bridge_stats(const struct or_options_t *old_options);
+
+int options_act_relay_stats(const struct or_options_t *old_options,
+                            bool *print_notice_out);
+void options_act_relay_stats_msg(void);
+
+int options_act_relay_desc(const struct or_options_t *old_options);
+int options_act_relay_dos(const struct or_options_t *old_options);
+int options_act_relay_dir(const struct or_options_t *old_options);
+
+#ifdef RELAY_CONFIG_PRIVATE
+
+STATIC int check_bridge_distribution_setting(const char *bd);
+STATIC int have_enough_mem_for_dircache(const struct or_options_t *options,
+                                        size_t total_mem, char **msg);
+
+#endif /* defined(RELAY_CONFIG_PRIVATE) */
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#include "lib/cc/compat_compiler.h"
+
+/** When tor is compiled with the relay module disabled, it can't be
+ * configured as a relay or bridge.
+ *
+ * Always sets ClientOnly to 1.
+ *
+ * Returns -1 and sets msg to a newly allocated string, if ORPort, DirPort,
+ * DirCache, or BridgeRelay are set in options. Otherwise returns 0. */
+static inline int
+options_validate_relay_mode(const struct or_options_t *old_options,
+                            struct or_options_t *options,
+                            char **msg)
+{
+  (void)old_options;
+
+  /* Only check the primary options for now, #29211 will disable more
+   * options. These ORPort and DirPort checks are too strict, and will
+   * reject valid configs that disable ports, like "ORPort 0". */
+  if (options->DirCache ||
+      options->BridgeRelay ||
+      options->ORPort_lines ||
+      options->DirPort_lines) {
+    /* REJECT() this configuration */
+    *msg = tor_strdup("This tor was built with relay mode disabled. "
+                      "It can not be configured with an ORPort, a DirPort, "
+                      "DirCache 1, or BridgeRelay 1.");
+    return -1;
+  }
+
+  return 0;
+}
+
+static inline int
+port_parse_ports_relay(or_options_t *options,
+                       char **msg,
+                       smartlist_t *ports_out,
+                       int *have_low_ports_out)
+{
+  (void)options;
+  (void)msg;
+  (void)ports_out;
+  if (*have_low_ports_out < 0)
+    *have_low_ports_out = 0;
+  return 0;
+}
+
+#define relay_get_dirportfrontpage() \
+  (NULL)
+#define relay_config_free_all() \
+  STMT_BEGIN STMT_END
+
+#define relay_get_effective_bwrate(options) \
+  (((void)(options)),0)
+#define relay_get_effective_bwburst(options) \
+  (((void)(options)),0)
+
+#define port_warn_nonlocal_ext_orports(ports, portname) \
+  (((void)(ports)),((void)(portname)))
+
+#define port_update_port_set_relay(options, ports) \
+  (((void)(options)),((void)(ports)))
+
+#define options_validate_relay_os(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_info(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_publish_server(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_padding(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_bandwidth(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_accounting(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_validate_relay_testing(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+
+#define options_act_relay(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_accounting(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_bandwidth(old_options) \
+  (((void)(old_options)),0)
+#define options_act_bridge_stats(old_options) \
+  (((void)(old_options)),0)
+
+#define options_act_relay_stats(old_options, print_notice_out) \
+  (((void)(old_options)),((void)(print_notice_out)),0)
+#define options_act_relay_stats_msg() \
+  STMT_BEGIN STMT_END
+
+#define options_act_relay_desc(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_dos(old_options) \
+  (((void)(old_options)),0)
+#define options_act_relay_dir(old_options) \
+  (((void)(old_options)),0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_CONFIG_H) */
diff --git a/src/feature/relay/relay_find_addr.c b/src/feature/relay/relay_find_addr.c
new file mode 100644
index 0000000000..86cd799d42
--- /dev/null
+++ b/src/feature/relay/relay_find_addr.c
@@ -0,0 +1,133 @@
+/* Copyright (c) 2001-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file relay_find_addr.c
+ * \brief Implement mechanism for a relay to find its address.
+ **/
+
+#include "core/or/or.h"
+
+#include "app/config/config.h"
+#include "app/config/resolve_addr.h"
+
+#include "core/mainloop/mainloop.h"
+
+#include "feature/control/control_events.h"
+#include "feature/dircommon/dir_connection_st.h"
+#include "feature/relay/relay_find_addr.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+
+/** The most recently guessed value of our IP address, based on directory
+ * headers. */
+static tor_addr_t last_guessed_ip = TOR_ADDR_NULL;
+
+/** We failed to resolve our address locally, but we'd like to build
+ * a descriptor and publish / test reachability. If we have a guess
+ * about our address based on directory headers, answer it and return
+ * 0; else return -1. */
+static int
+router_guess_address_from_dir_headers(uint32_t *guess)
+{
+  if (!tor_addr_is_null(&last_guessed_ip)) {
+    *guess = tor_addr_to_ipv4h(&last_guessed_ip);
+    return 0;
+  }
+  return -1;
+}
+
+/** A directory server <b>d_conn</b> told us our IP address is
+ * <b>suggestion</b>.
+ * If this address is different from the one we think we are now, and
+ * if our computer doesn't actually know its IP address, then switch. */
+void
+router_new_address_suggestion(const char *suggestion,
+                              const dir_connection_t *d_conn)
+{
+  tor_addr_t addr;
+  uint32_t cur = 0;             /* Current IPv4 address.  */
+  const or_options_t *options = get_options();
+
+  /* first, learn what the IP address actually is */
+  if (tor_addr_parse(&addr, suggestion) == -1) {
+    log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.",
+              escaped(suggestion));
+    return;
+  }
+
+  log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion);
+
+  if (!server_mode(options)) {
+    tor_addr_copy(&last_guessed_ip, &addr);
+    return;
+  }
+
+  /* XXXX ipv6 */
+  cur = get_last_resolved_addr();
+  if (cur ||
+      resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) {
+    /* We're all set -- we already know our address. Great. */
+    tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we
+                                                   need it later */
+    return;
+  }
+  if (tor_addr_is_internal(&addr, 0)) {
+    /* Don't believe anybody who says our IP is, say, 127.0.0.1. */
+    return;
+  }
+  if (tor_addr_eq(&d_conn->base_.addr, &addr)) {
+    /* Don't believe anybody who says our IP is their IP. */
+    log_debug(LD_DIR, "A directory server told us our IP address is %s, "
+              "but they are just reporting their own IP address. Ignoring.",
+              suggestion);
+    return;
+  }
+
+  /* Okay.  We can't resolve our own address, and X-Your-Address-Is is giving
+   * us an answer different from what we had the last time we managed to
+   * resolve it. */
+  if (!tor_addr_eq(&last_guessed_ip, &addr)) {
+    control_event_server_status(LOG_NOTICE,
+                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV",
+                                suggestion);
+    log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr,
+                         d_conn->base_.address);
+    ip_address_changed(0);
+    tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor()
+                                               will fetch it */
+  }
+}
+
+/** Make a current best guess at our address, either because
+ * it's configured in torrc, or because we've learned it from
+ * dirserver headers. Place the answer in *<b>addr</b> and return
+ * 0 on success, else return -1 if we have no guess.
+ *
+ * If <b>cache_only</b> is true, just return any cached answers, and
+ * don't try to get any new answers.
+ */
+MOCK_IMPL(int,
+router_pick_published_address, (const or_options_t *options, uint32_t *addr,
+                                int cache_only))
+{
+  /* First, check the cached output from resolve_my_address(). */
+  *addr = get_last_resolved_addr();
+  if (*addr)
+    return 0;
+
+  /* Second, consider doing a resolve attempt right here. */
+  if (!cache_only) {
+    if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) {
+      log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr));
+      return 0;
+    }
+  }
+
+  /* Third, check the cached output from router_new_address_suggestion(). */
+  if (router_guess_address_from_dir_headers(addr) >= 0)
+    return 0;
+
+  /* We have no useful cached answers. Return failure. */
+  return -1;
+}
diff --git a/src/feature/relay/relay_find_addr.h b/src/feature/relay/relay_find_addr.h
new file mode 100644
index 0000000000..ac51a977e6
--- /dev/null
+++ b/src/feature/relay/relay_find_addr.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file relay_find_addr.h
+ * \brief Header file for relay_find_addr.c.
+ **/
+
+#ifndef TOR_RELAY_FIND_ADDR_H
+#define TOR_RELAY_FIND_ADDR_H
+
+MOCK_DECL(int, router_pick_published_address,
+          (const or_options_t *options, uint32_t *addr, int cache_only));
+
+void router_new_address_suggestion(const char *suggestion,
+                                   const dir_connection_t *d_conn);
+
+#ifdef RELAY_FIND_ADDR_PRIVATE
+
+#endif /* RELAY_FIND_ADDR_PRIVATE */
+
+#endif /* TOR_RELAY_FIND_ADDR_H */
+
diff --git a/src/feature/relay/relay_handshake.c b/src/feature/relay/relay_handshake.c
new file mode 100644
index 0000000000..030dc94956
--- /dev/null
+++ b/src/feature/relay/relay_handshake.c
@@ -0,0 +1,565 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_handshake.c
+ * @brief Functions to implement the relay-only parts of our
+ *    connection handshake.
+ *
+ * Some parts of our TLS link handshake are only done by relays (including
+ * bridges).  Specifically, only relays need to send CERTS cells; only
+ * relays need to send or receive AUTHCHALLENGE cells, and only relays need to
+ * send or receive AUTHENTICATE cells.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "feature/relay/relay_handshake.h"
+
+#include "app/config/config.h"
+#include "core/or/connection_or.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "trunnel/link_handshake.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/nodelist/torcert.h"
+
+#include "core/or/or_connection_st.h"
+#include "core/or/or_handshake_certs_st.h"
+#include "core/or/or_handshake_state_st.h"
+#include "core/or/var_cell_st.h"
+
+#include "lib/tls/tortls.h"
+#include "lib/tls/x509.h"
+
+/** Helper used to add an encoded certs to a cert cell */
+static void
+add_certs_cell_cert_helper(certs_cell_t *certs_cell,
+                           uint8_t cert_type,
+                           const uint8_t *cert_encoded,
+                           size_t cert_len)
+{
+  tor_assert(cert_len <= UINT16_MAX);
+  certs_cell_cert_t *ccc = certs_cell_cert_new();
+  ccc->cert_type = cert_type;
+  ccc->cert_len = cert_len;
+  certs_cell_cert_setlen_body(ccc, cert_len);
+  memcpy(certs_cell_cert_getarray_body(ccc), cert_encoded, cert_len);
+
+  certs_cell_add_certs(certs_cell, ccc);
+}
+
+/** Add an encoded X509 cert (stored as <b>cert_len</b> bytes at
+ * <b>cert_encoded</b>) to the trunnel certs_cell_t object that we are
+ * building in <b>certs_cell</b>.  Set its type field to <b>cert_type</b>.
+ * (If <b>cert</b> is NULL, take no action.) */
+static void
+add_x509_cert(certs_cell_t *certs_cell,
+              uint8_t cert_type,
+              const tor_x509_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  const uint8_t *cert_encoded = NULL;
+  size_t cert_len;
+  tor_x509_cert_get_der(cert, &cert_encoded, &cert_len);
+
+  add_certs_cell_cert_helper(certs_cell, cert_type, cert_encoded, cert_len);
+}
+
+/** Add an Ed25519 cert from <b>cert</b> to the trunnel certs_cell_t object
+ * that we are building in <b>certs_cell</b>.  Set its type field to
+ * <b>cert_type</b>. (If <b>cert</b> is NULL, take no action.) */
+static void
+add_ed25519_cert(certs_cell_t *certs_cell,
+                 uint8_t cert_type,
+                 const tor_cert_t *cert)
+{
+  if (NULL == cert)
+    return;
+
+  add_certs_cell_cert_helper(certs_cell, cert_type,
+                             cert->encoded, cert->encoded_len);
+}
+
+#ifdef TOR_UNIT_TESTS
+int certs_cell_ed25519_disabled_for_testing = 0;
+#else
+#define certs_cell_ed25519_disabled_for_testing 0
+#endif
+
+/** Send a CERTS cell on the connection <b>conn</b>.  Return 0 on success, -1
+ * on failure. */
+int
+connection_or_send_certs_cell(or_connection_t *conn)
+{
+  const tor_x509_cert_t *global_link_cert = NULL, *id_cert = NULL;
+  tor_x509_cert_t *own_link_cert = NULL;
+  var_cell_t *cell;
+
+  certs_cell_t *certs_cell = NULL;
+
+  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
+
+  if (! conn->handshake_state)
+    return -1;
+
+  const int conn_in_server_mode = ! conn->handshake_state->started_here;
+
+  /* Get the encoded values of the X509 certificates */
+  if (tor_tls_get_my_certs(conn_in_server_mode,
+                           &global_link_cert, &id_cert) < 0)
+    return -1;
+
+  if (conn_in_server_mode) {
+    own_link_cert = tor_tls_get_own_cert(conn->tls);
+  }
+  tor_assert(id_cert);
+
+  certs_cell = certs_cell_new();
+
+  /* Start adding certs.  First the link cert or auth1024 cert. */
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(own_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_TLS_LINK, own_link_cert);
+  } else {
+    tor_assert(global_link_cert);
+    add_x509_cert(certs_cell,
+                  OR_CERT_TYPE_AUTH_1024, global_link_cert);
+  }
+
+  /* Next the RSA->RSA ID cert */
+  add_x509_cert(certs_cell,
+                OR_CERT_TYPE_ID_1024, id_cert);
+
+  /* Next the Ed25519 certs */
+  add_ed25519_cert(certs_cell,
+                   CERTTYPE_ED_ID_SIGN,
+                   get_master_signing_key_cert());
+  if (conn_in_server_mode) {
+    tor_assert_nonfatal(conn->handshake_state->own_link_cert ||
+                        certs_cell_ed25519_disabled_for_testing);
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_LINK,
+                     conn->handshake_state->own_link_cert);
+  } else {
+    add_ed25519_cert(certs_cell,
+                     CERTTYPE_ED_SIGN_AUTH,
+                     get_current_auth_key_cert());
+  }
+
+  /* And finally the crosscert. */
+  {
+    const uint8_t *crosscert=NULL;
+    size_t crosscert_len;
+    get_master_rsa_crosscert(&crosscert, &crosscert_len);
+    if (crosscert) {
+      add_certs_cell_cert_helper(certs_cell,
+                               CERTTYPE_RSA1024_ID_EDID,
+                               crosscert, crosscert_len);
+    }
+  }
+
+  /* We've added all the certs; make the cell. */
+  certs_cell->n_certs = certs_cell_getlen_certs(certs_cell);
+
+  ssize_t alloc_len = certs_cell_encoded_len(certs_cell);
+  tor_assert(alloc_len >= 0 && alloc_len <= UINT16_MAX);
+  cell = var_cell_new(alloc_len);
+  cell->command = CELL_CERTS;
+  ssize_t enc_len = certs_cell_encode(cell->payload, alloc_len, certs_cell);
+  tor_assert(enc_len > 0 && enc_len <= alloc_len);
+  cell->payload_len = enc_len;
+
+  connection_or_write_var_cell_to_buf(cell, conn);
+  var_cell_free(cell);
+  certs_cell_free(certs_cell);
+  tor_x509_cert_free(own_link_cert);
+
+  return 0;
+}
+
+#ifdef TOR_UNIT_TESTS
+int testing__connection_or_pretend_TLSSECRET_is_supported = 0;
+#else
+#define testing__connection_or_pretend_TLSSECRET_is_supported 0
+#endif
+
+/** Return true iff <b>challenge_type</b> is an AUTHCHALLENGE type that
+ * we can send and receive. */
+int
+authchallenge_type_is_supported(uint16_t challenge_type)
+{
+  switch (challenge_type) {
+     case AUTHTYPE_RSA_SHA256_TLSSECRET:
+#ifdef HAVE_WORKING_TOR_TLS_GET_TLSSECRETS
+       return 1;
+#else
+       return testing__connection_or_pretend_TLSSECRET_is_supported;
+#endif
+     case AUTHTYPE_ED25519_SHA256_RFC5705:
+       return 1;
+     case AUTHTYPE_RSA_SHA256_RFC5705:
+     default:
+       return 0;
+  }
+}
+
+/** Return true iff <b>challenge_type_a</b> is one that we would rather
+ * use than <b>challenge_type_b</b>. */
+int
+authchallenge_type_is_better(uint16_t challenge_type_a,
+                             uint16_t challenge_type_b)
+{
+  /* Any supported type is better than an unsupported one;
+   * all unsupported types are equally bad. */
+  if (!authchallenge_type_is_supported(challenge_type_a))
+    return 0;
+  if (!authchallenge_type_is_supported(challenge_type_b))
+    return 1;
+  /* It happens that types are superior in numerically ascending order.
+   * If that ever changes, this must change too. */
+  return (challenge_type_a > challenge_type_b);
+}
+
+/** Send an AUTH_CHALLENGE cell on the connection <b>conn</b>. Return 0
+ * on success, -1 on failure. */
+int
+connection_or_send_auth_challenge_cell(or_connection_t *conn)
+{
+  var_cell_t *cell = NULL;
+  int r = -1;
+  tor_assert(conn->base_.state == OR_CONN_STATE_OR_HANDSHAKING_V3);
+
+  if (! conn->handshake_state)
+    return -1;
+
+  auth_challenge_cell_t *ac = auth_challenge_cell_new();
+
+  tor_assert(sizeof(ac->challenge) == 32);
+  crypto_rand((char*)ac->challenge, sizeof(ac->challenge));
+
+  if (authchallenge_type_is_supported(AUTHTYPE_RSA_SHA256_TLSSECRET))
+    auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_TLSSECRET);
+  /* Disabled, because everything that supports this method also supports
+   * the much-superior ED25519_SHA256_RFC5705 */
+  /* auth_challenge_cell_add_methods(ac, AUTHTYPE_RSA_SHA256_RFC5705); */
+  if (authchallenge_type_is_supported(AUTHTYPE_ED25519_SHA256_RFC5705))
+    auth_challenge_cell_add_methods(ac, AUTHTYPE_ED25519_SHA256_RFC5705);
+  auth_challenge_cell_set_n_methods(ac,
+                                    auth_challenge_cell_getlen_methods(ac));
+
+  cell = var_cell_new(auth_challenge_cell_encoded_len(ac));
+  ssize_t len = auth_challenge_cell_encode(cell->payload, cell->payload_len,
+                                           ac);
+  if (len != cell->payload_len) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Encoded auth challenge cell length not as expected");
+    goto done;
+    /* LCOV_EXCL_STOP */
+  }
+  cell->command = CELL_AUTH_CHALLENGE;
+
+  connection_or_write_var_cell_to_buf(cell, conn);
+  r = 0;
+
+ done:
+  var_cell_free(cell);
+  auth_challenge_cell_free(ac);
+
+  return r;
+}
+
+/** Compute the main body of an AUTHENTICATE cell that a client can use
+ * to authenticate itself on a v3 handshake for <b>conn</b>.  Return it
+ * in a var_cell_t.
+ *
+ * If <b>server</b> is true, only calculate the first
+ * V3_AUTH_FIXED_PART_LEN bytes -- the part of the authenticator that's
+ * determined by the rest of the handshake, and which match the provided value
+ * exactly.
+ *
+ * If <b>server</b> is false and <b>signing_key</b> is NULL, calculate the
+ * first V3_AUTH_BODY_LEN bytes of the authenticator (that is, everything
+ * that should be signed), but don't actually sign it.
+ *
+ * If <b>server</b> is false and <b>signing_key</b> is provided, calculate the
+ * entire authenticator, signed with <b>signing_key</b>.
+ *
+ * Return the length of the cell body on success, and -1 on failure.
+ */
+var_cell_t *
+connection_or_compute_authenticate_cell_body(or_connection_t *conn,
+                                             const int authtype,
+                                             crypto_pk_t *signing_key,
+                                      const ed25519_keypair_t *ed_signing_key,
+                                      int server)
+{
+  auth1_t *auth = NULL;
+  auth_ctx_t *ctx = auth_ctx_new();
+  var_cell_t *result = NULL;
+  int old_tlssecrets_algorithm = 0;
+  const char *authtype_str = NULL;
+
+  int is_ed = 0;
+
+  /* assert state is reasonable XXXX */
+  switch (authtype) {
+  case AUTHTYPE_RSA_SHA256_TLSSECRET:
+    authtype_str = "AUTH0001";
+    old_tlssecrets_algorithm = 1;
+    break;
+  case AUTHTYPE_RSA_SHA256_RFC5705:
+    authtype_str = "AUTH0002";
+    break;
+  case AUTHTYPE_ED25519_SHA256_RFC5705:
+    authtype_str = "AUTH0003";
+    is_ed = 1;
+    break;
+  default:
+    tor_assert(0);
+    break;
+  }
+
+  auth = auth1_new();
+  ctx->is_ed = is_ed;
+
+  /* Type: 8 bytes. */
+  memcpy(auth1_getarray_type(auth), authtype_str, 8);
+
+  {
+    const tor_x509_cert_t *id_cert=NULL;
+    const common_digests_t *my_digests, *their_digests;
+    const uint8_t *my_id, *their_id, *client_id, *server_id;
+    if (tor_tls_get_my_certs(server, NULL, &id_cert))
+      goto err;
+    my_digests = tor_x509_cert_get_id_digests(id_cert);
+    their_digests =
+      tor_x509_cert_get_id_digests(conn->handshake_state->certs->id_cert);
+    tor_assert(my_digests);
+    tor_assert(their_digests);
+    my_id = (uint8_t*)my_digests->d[DIGEST_SHA256];
+    their_id = (uint8_t*)their_digests->d[DIGEST_SHA256];
+
+    client_id = server ? their_id : my_id;
+    server_id = server ? my_id : their_id;
+
+    /* Client ID digest: 32 octets. */
+    memcpy(auth->cid, client_id, 32);
+
+    /* Server ID digest: 32 octets. */
+    memcpy(auth->sid, server_id, 32);
+  }
+
+  if (is_ed) {
+    const ed25519_public_key_t *my_ed_id, *their_ed_id;
+    if (!conn->handshake_state->certs->ed_id_sign) {
+      log_warn(LD_OR, "Ed authenticate without Ed ID cert from peer.");
+      goto err;
+    }
+    my_ed_id = get_master_identity_key();
+    their_ed_id = &conn->handshake_state->certs->ed_id_sign->signing_key;
+
+    const uint8_t *cid_ed = (server ? their_ed_id : my_ed_id)->pubkey;
+    const uint8_t *sid_ed = (server ? my_ed_id : their_ed_id)->pubkey;
+
+    memcpy(auth->u1_cid_ed, cid_ed, ED25519_PUBKEY_LEN);
+    memcpy(auth->u1_sid_ed, sid_ed, ED25519_PUBKEY_LEN);
+  }
+
+  {
+    crypto_digest_t *server_d, *client_d;
+    if (server) {
+      server_d = conn->handshake_state->digest_sent;
+      client_d = conn->handshake_state->digest_received;
+    } else {
+      client_d = conn->handshake_state->digest_sent;
+      server_d = conn->handshake_state->digest_received;
+    }
+
+    /* Server log digest : 32 octets */
+    crypto_digest_get_digest(server_d, (char*)auth->slog, 32);
+
+    /* Client log digest : 32 octets */
+    crypto_digest_get_digest(client_d, (char*)auth->clog, 32);
+  }
+
+  {
+    /* Digest of cert used on TLS link : 32 octets. */
+    tor_x509_cert_t *cert = NULL;
+    if (server) {
+      cert = tor_tls_get_own_cert(conn->tls);
+    } else {
+      cert = tor_tls_get_peer_cert(conn->tls);
+    }
+    if (!cert) {
+      log_warn(LD_OR, "Unable to find cert when making %s data.",
+               authtype_str);
+      goto err;
+    }
+
+    memcpy(auth->scert,
+           tor_x509_cert_get_cert_digests(cert)->d[DIGEST_SHA256], 32);
+
+    tor_x509_cert_free(cert);
+  }
+
+  /* HMAC of clientrandom and serverrandom using master key : 32 octets */
+  if (old_tlssecrets_algorithm) {
+    if (tor_tls_get_tlssecrets(conn->tls, auth->tlssecrets) < 0) {
+      log_fn(LOG_PROTOCOL_WARN, LD_OR, "Somebody asked us for an older TLS "
+         "authentication method (AUTHTYPE_RSA_SHA256_TLSSECRET) "
+         "which we don't support.");
+    }
+  } else {
+    char label[128];
+    tor_snprintf(label, sizeof(label),
+                 "EXPORTER FOR TOR TLS CLIENT BINDING %s", authtype_str);
+    int r = tor_tls_export_key_material(conn->tls, auth->tlssecrets,
+                                        auth->cid, sizeof(auth->cid),
+                                        label);
+    if (r < 0) {
+      if (r != -2)
+        log_warn(LD_BUG, "TLS key export failed for unknown reason.");
+      // If r == -2, this was openssl bug 7712.
+      goto err;
+    }
+  }
+
+  /* 8 octets were reserved for the current time, but we're trying to get out
+   * of the habit of sending time around willynilly.  Fortunately, nothing
+   * checks it.  That's followed by 16 bytes of nonce. */
+  crypto_rand((char*)auth->rand, 24);
+
+  ssize_t maxlen = auth1_encoded_len(auth, ctx);
+  if (ed_signing_key && is_ed) {
+    maxlen += ED25519_SIG_LEN;
+  } else if (signing_key && !is_ed) {
+    maxlen += crypto_pk_keysize(signing_key);
+  }
+
+  const int AUTH_CELL_HEADER_LEN = 4; /* 2 bytes of type, 2 bytes of length */
+  result = var_cell_new(AUTH_CELL_HEADER_LEN + maxlen);
+  uint8_t *const out = result->payload + AUTH_CELL_HEADER_LEN;
+  const size_t outlen = maxlen;
+  ssize_t len;
+
+  result->command = CELL_AUTHENTICATE;
+  set_uint16(result->payload, htons(authtype));
+
+  if ((len = auth1_encode(out, outlen, auth, ctx)) < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed part of AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (server) {
+    auth1_t *tmp = NULL;
+    ssize_t len2 = auth1_parse(&tmp, out, len, ctx);
+    if (!tmp) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to parse signed part of AUTH1 data that "
+               "we just encoded");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    result->payload_len = (tmp->end_of_signed - result->payload);
+
+    auth1_free(tmp);
+    if (len2 != len) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Mismatched length when re-parsing AUTH1 data.");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    goto done;
+  }
+
+  if (ed_signing_key && is_ed) {
+    ed25519_signature_t sig;
+    if (ed25519_sign(&sig, out, len, ed_signing_key) < 0) {
+      /* LCOV_EXCL_START */
+      log_warn(LD_BUG, "Unable to sign ed25519 authentication data");
+      goto err;
+      /* LCOV_EXCL_STOP */
+    }
+    auth1_setlen_sig(auth, ED25519_SIG_LEN);
+    memcpy(auth1_getarray_sig(auth), sig.sig, ED25519_SIG_LEN);
+
+  } else if (signing_key && !is_ed) {
+    auth1_setlen_sig(auth, crypto_pk_keysize(signing_key));
+
+    char d[32];
+    crypto_digest256(d, (char*)out, len, DIGEST_SHA256);
+    int siglen = crypto_pk_private_sign(signing_key,
+                                    (char*)auth1_getarray_sig(auth),
+                                    auth1_getlen_sig(auth),
+                                    d, 32);
+    if (siglen < 0) {
+      log_warn(LD_OR, "Unable to sign AUTH1 data.");
+      goto err;
+    }
+
+    auth1_setlen_sig(auth, siglen);
+  }
+
+  len = auth1_encode(out, outlen, auth, ctx);
+  if (len < 0) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_BUG, "Unable to encode signed AUTH1 data.");
+    goto err;
+    /* LCOV_EXCL_STOP */
+  }
+  tor_assert(len + AUTH_CELL_HEADER_LEN <= result->payload_len);
+  result->payload_len = len + AUTH_CELL_HEADER_LEN;
+  set_uint16(result->payload+2, htons(len));
+
+  goto done;
+
+ err:
+  var_cell_free(result);
+  result = NULL;
+ done:
+  auth1_free(auth);
+  auth_ctx_free(ctx);
+  return result;
+}
+
+/** Send an AUTHENTICATE cell on the connection <b>conn</b>.  Return 0 on
+ * success, -1 on failure */
+MOCK_IMPL(int,
+connection_or_send_authenticate_cell,(or_connection_t *conn, int authtype))
+{
+  var_cell_t *cell;
+  crypto_pk_t *pk = tor_tls_get_my_client_auth_key();
+  /* XXXX make sure we're actually supposed to send this! */
+
+  if (!pk) {
+    log_warn(LD_BUG, "Can't compute authenticate cell: no client auth key");
+    return -1;
+  }
+  if (! authchallenge_type_is_supported(authtype)) {
+    log_warn(LD_BUG, "Tried to send authenticate cell with unknown "
+             "authentication type %d", authtype);
+    return -1;
+  }
+
+  cell = connection_or_compute_authenticate_cell_body(conn,
+                                                 authtype,
+                                                 pk,
+                                                 get_current_auth_keypair(),
+                                                 0 /* not server */);
+  if (! cell) {
+    log_fn(LOG_PROTOCOL_WARN, LD_NET, "Unable to compute authenticate cell!");
+    return -1;
+  }
+  connection_or_write_var_cell_to_buf(cell, conn);
+  var_cell_free(cell);
+
+  return 0;
+}
diff --git a/src/feature/relay/relay_handshake.h b/src/feature/relay/relay_handshake.h
new file mode 100644
index 0000000000..99a658cbcc
--- /dev/null
+++ b/src/feature/relay/relay_handshake.h
@@ -0,0 +1,90 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_handshake.h
+ * @brief Header for feature/relay/relay_handshake.c
+ **/
+
+#ifndef TOR_CORE_OR_RELAY_HANDSHAKE_H
+#define TOR_CORE_OR_RELAY_HANDSHAKE_H
+
+#ifdef HAVE_MODULE_RELAY
+struct ed25519_keypair_t;
+
+int connection_or_send_certs_cell(or_connection_t *conn);
+int connection_or_send_auth_challenge_cell(or_connection_t *conn);
+
+var_cell_t *connection_or_compute_authenticate_cell_body(
+                              or_connection_t *conn,
+                              const int authtype,
+                              crypto_pk_t *signing_key,
+                              const struct ed25519_keypair_t *ed_signing_key,
+                              int server);
+
+int authchallenge_type_is_supported(uint16_t challenge_type);
+int authchallenge_type_is_better(uint16_t challenge_type_a,
+                                 uint16_t challenge_type_b);
+
+MOCK_DECL(int,connection_or_send_authenticate_cell,
+          (or_connection_t *conn, int type));
+
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
+#endif
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+static inline int
+connection_or_send_certs_cell(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+static inline int
+connection_or_send_auth_challenge_cell(or_connection_t *conn)
+{
+  (void)conn;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+static inline var_cell_t *
+connection_or_compute_authenticate_cell_body(
+                              or_connection_t *conn,
+                              const int authtype,
+                              crypto_pk_t *signing_key,
+                              const struct ed25519_keypair_t *ed_signing_key,
+                              int server)
+{
+  (void)conn;
+  (void)authtype;
+  (void)signing_key;
+  (void)ed_signing_key;
+  (void)server;
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+
+#define authchallenge_type_is_supported(t) (0)
+#define authchallenge_type_is_better(a, b) (0)
+
+static inline int
+connection_or_send_authenticate_cell(or_connection_t *conn, int type)
+{
+  (void)conn;
+  (void)type;
+  tor_assert_nonfatal_unreached();
+  return -1;
+}
+
+#ifdef TOR_UNIT_TESTS
+extern int certs_cell_ed25519_disabled_for_testing;
+#endif
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_CORE_OR_RELAY_HANDSHAKE_H) */
diff --git a/src/feature/relay/relay_periodic.c b/src/feature/relay/relay_periodic.c
new file mode 100644
index 0000000000..bfe12cd0b0
--- /dev/null
+++ b/src/feature/relay/relay_periodic.c
@@ -0,0 +1,318 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.c
+ * @brief Periodic functions for the relay subsytem
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "core/mainloop/periodic.h"
+#include "core/mainloop/cpuworker.h" // XXXX use a pubsub event.
+#include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
+#include "core/or/circuituse.h" // XXXX move have_performed_bandwidth_test
+
+#include "feature/relay/dns.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/routermode.h"
+#include "feature/relay/selftest.h"
+#include "feature/stats/predict_ports.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/control/control_events.h"
+
+#ifndef COCCI
+#define DECLARE_EVENT(name, roles, flags)         \
+  static periodic_event_item_t name ## _event =   \
+    PERIODIC_EVENT(name,                          \
+                   PERIODIC_EVENT_ROLE_##roles,   \
+                   flags)
+#endif /* !defined(COCCI) */
+
+#define FL(name) (PERIODIC_EVENT_FLAG_##name)
+
+/**
+ * Periodic callback: If we're a server and initializing dns failed, retry.
+ */
+static int
+retry_dns_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+#define RETRY_DNS_INTERVAL (10*60)
+  if (server_mode(options) && has_dns_init_failed())
+    dns_init();
+  return RETRY_DNS_INTERVAL;
+}
+
+DECLARE_EVENT(retry_dns, ROUTER, 0);
+
+static int dns_honesty_first_time = 1;
+
+/**
+ * Periodic event: if we're an exit, see if our DNS server is telling us
+ * obvious lies.
+ */
+static int
+check_dns_honesty_callback(time_t now, const or_options_t *options)
+{
+  (void)now;
+  /* 9. and if we're an exit node, check whether our DNS is telling stories
+   * to us. */
+  if (net_is_disabled() ||
+      ! public_server_mode(options) ||
+      router_my_exit_policy_is_reject_star())
+    return PERIODIC_EVENT_NO_UPDATE;
+
+  if (dns_honesty_first_time) {
+    /* Don't launch right when we start */
+    dns_honesty_first_time = 0;
+    return crypto_rand_int_range(60, 180);
+  }
+
+  dns_launch_correctness_checks();
+  return 12*3600 + crypto_rand_int(12*3600);
+}
+
+DECLARE_EVENT(check_dns_honesty, RELAY, FL(NEED_NET));
+
+/* Periodic callback: rotate the onion keys after the period defined by the
+ * "onion-key-rotation-days" consensus parameter, shut down and restart all
+ * cpuworkers, and update our descriptor if necessary.
+ */
+static int
+rotate_onion_key_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_lifetime = get_onion_key_lifetime();
+    time_t rotation_time = get_onion_key_set_at()+onion_key_lifetime;
+    if (rotation_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL,"Rotating onion key.");
+    rotate_onion_key();
+    cpuworkers_rotate_keyinfo();
+    if (router_rebuild_descriptor(1)<0) {
+      log_info(LD_CONFIG, "Couldn't rebuild router descriptor");
+    }
+    if (advertised_server_mode() && !net_is_disabled())
+      router_upload_dir_desc_to_dirservers(0);
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(rotate_onion_key, ROUTER, 0);
+
+/** Periodic callback: consider rebuilding or and re-uploading our descriptor
+ * (if we've passed our internal checks). */
+static int
+check_descriptor_callback(time_t now, const or_options_t *options)
+{
+/** How often do we check whether part of our router info has changed in a
+ * way that would require an upload? That includes checking whether our IP
+ * address has changed. */
+#define CHECK_DESCRIPTOR_INTERVAL (60)
+
+  (void)options;
+
+  /* 2b. Once per minute, regenerate and upload the descriptor if the old
+   * one is inaccurate. */
+  if (!net_is_disabled()) {
+    check_descriptor_bandwidth_changed(now);
+    check_descriptor_ipaddress_changed(now);
+    mark_my_descriptor_dirty_if_too_old(now);
+    consider_publishable_server(0);
+  }
+
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_descriptor, ROUTER, FL(NEED_NET));
+
+static int dirport_reachability_count = 0;
+
+/**
+ * Periodic callback: check whether we're reachable (as a relay), and
+ * whether our bandwidth has changed enough that we need to
+ * publish a new descriptor.
+ */
+static int
+check_for_reachability_bw_callback(time_t now, const or_options_t *options)
+{
+  /* XXXX This whole thing was stuck in the middle of what is now
+   * XXXX check_descriptor_callback.  I'm not sure it's right. */
+  /** How often should we consider launching reachability tests in our first
+   * TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT seconds? */
+#define EARLY_CHECK_REACHABILITY_INTERVAL (60)
+
+  /* also, check religiously for reachability, if it's within the first
+   * 20 minutes of our uptime. */
+  if (server_mode(options) &&
+      (have_completed_a_circuit() || !any_predicted_circuits(now)) &&
+      !net_is_disabled()) {
+    if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+      router_do_reachability_checks(1, dirport_reachability_count==0);
+      if (++dirport_reachability_count > 5)
+        dirport_reachability_count = 0;
+      return EARLY_CHECK_REACHABILITY_INTERVAL;
+    } else {
+      /* If we haven't checked for 12 hours and our bandwidth estimate is
+       * low, do another bandwidth test. This is especially important for
+       * bridges, since they might go long periods without much use. */
+      const routerinfo_t *me = router_get_my_routerinfo();
+      static int first_time = 1;
+      if (!first_time && me &&
+          me->bandwidthcapacity < me->bandwidthrate &&
+          me->bandwidthcapacity < 51200) {
+        reset_bandwidth_test();
+      }
+      first_time = 0;
+#define BANDWIDTH_RECHECK_INTERVAL (12*60*60)
+      return BANDWIDTH_RECHECK_INTERVAL;
+    }
+  }
+  return CHECK_DESCRIPTOR_INTERVAL;
+}
+
+DECLARE_EVENT(check_for_reachability_bw, ROUTER, FL(NEED_NET));
+
+/**
+ * Callback: Send warnings if Tor doesn't find its ports reachable.
+ */
+static int
+reachability_warnings_callback(time_t now, const or_options_t *options)
+{
+  (void) now;
+
+  if (get_uptime() < TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT) {
+    return (int)(TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT - get_uptime());
+  }
+
+  if (server_mode(options) &&
+      !net_is_disabled() &&
+      have_completed_a_circuit()) {
+    /* every 20 minutes, check and complain if necessary */
+    const routerinfo_t *me = router_get_my_routerinfo();
+    if (me && !check_whether_orport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      if (address) {
+        log_warn(LD_CONFIG,
+                 "Your server (%s:%d) has not managed to confirm that "
+                 "its ORPort is reachable. Relays do not publish descriptors "
+                 "until their ORPort and DirPort are reachable. Please check "
+                 "your firewalls, ports, address, /etc/hosts file, etc.",
+                 address, me->or_port);
+        control_event_server_status(LOG_WARN,
+                                    "REACHABILITY_FAILED ORADDRESS=%s:%d",
+                                    address, me->or_port);
+        tor_free(address);
+      }
+    }
+
+    if (me && !check_whether_dirport_reachable(options)) {
+      char *address = tor_dup_ip(me->addr);
+      if (address) {
+        log_warn(LD_CONFIG,
+                 "Your server (%s:%d) has not managed to confirm that its "
+                 "DirPort is reachable. Relays do not publish descriptors "
+                 "until their ORPort and DirPort are reachable. Please check "
+                 "your firewalls, ports, address, /etc/hosts file, etc.",
+                 address, me->dir_port);
+        control_event_server_status(LOG_WARN,
+                                    "REACHABILITY_FAILED DIRADDRESS=%s:%d",
+                                    address, me->dir_port);
+        tor_free(address);
+      }
+    }
+  }
+
+  return TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT;
+}
+
+DECLARE_EVENT(reachability_warnings, ROUTER, FL(NEED_NET));
+
+/* Periodic callback: Every 30 seconds, check whether it's time to make new
+ * Ed25519 subkeys.
+ */
+static int
+check_ed_keys_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    if (should_make_new_ed_keys(options, now)) {
+      int new_signing_key = load_ed_keys(options, now);
+      if (new_signing_key < 0 ||
+          generate_ed_link_cert(options, now, new_signing_key > 0)) {
+        log_err(LD_OR, "Unable to update Ed25519 keys!  Exiting.");
+        tor_shutdown_event_loop_and_exit(1);
+      }
+    }
+    return 30;
+  }
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_ed_keys, ROUTER, 0);
+
+/* Period callback: Check if our old onion keys are still valid after the
+ * period of time defined by the consensus parameter
+ * "onion-key-grace-period-days", otherwise expire them by setting them to
+ * NULL.
+ */
+static int
+check_onion_keys_expiry_time_callback(time_t now, const or_options_t *options)
+{
+  if (server_mode(options)) {
+    int onion_key_grace_period = get_onion_key_grace_period();
+    time_t expiry_time = get_onion_key_set_at()+onion_key_grace_period;
+    if (expiry_time > now) {
+      return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+    }
+
+    log_info(LD_GENERAL, "Expiring old onion keys.");
+    expire_old_onion_keys();
+    cpuworkers_rotate_keyinfo();
+    return ONION_KEY_CONSENSUS_CHECK_INTERVAL;
+  }
+
+  return PERIODIC_EVENT_NO_UPDATE;
+}
+
+DECLARE_EVENT(check_onion_keys_expiry_time, ROUTER, 0);
+
+void
+relay_register_periodic_events(void)
+{
+  periodic_events_register(&retry_dns_event);
+  periodic_events_register(&check_dns_honesty_event);
+  periodic_events_register(&rotate_onion_key_event);
+  periodic_events_register(&check_descriptor_event);
+  periodic_events_register(&check_for_reachability_bw_event);
+  periodic_events_register(&reachability_warnings_event);
+  periodic_events_register(&check_ed_keys_event);
+  periodic_events_register(&check_onion_keys_expiry_time_event);
+
+  dns_honesty_first_time = 1;
+  dirport_reachability_count = 0;
+}
+
+/**
+ * Update our schedule so that we'll check whether we need to update our
+ * descriptor immediately, rather than after up to CHECK_DESCRIPTOR_INTERVAL
+ * seconds.
+ */
+void
+reschedule_descriptor_update_check(void)
+{
+  periodic_event_reschedule(&check_descriptor_event);
+}
diff --git a/src/feature/relay/relay_periodic.h b/src/feature/relay/relay_periodic.h
new file mode 100644
index 0000000000..ccda9a440b
--- /dev/null
+++ b/src/feature/relay/relay_periodic.h
@@ -0,0 +1,31 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_periodic.h
+ * @brief Header for feature/relay/relay_periodic.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+#define TOR_FEATURE_RELAY_RELAY_PERIODIC_H
+
+#ifdef HAVE_MODULE_RELAY
+
+void relay_register_periodic_events(void);
+void reschedule_descriptor_update_check(void);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#include "lib/cc/compat_compiler.h"
+
+#define relay_register_periodic_events() \
+  STMT_NIL
+#define reschedule_descriptor_update_check() \
+  STMT_NIL
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_PERIODIC_H) */
diff --git a/src/feature/relay/relay_stub.c b/src/feature/relay/relay_stub.c
new file mode 100644
index 0000000000..283aaf6e49
--- /dev/null
+++ b/src/feature/relay/relay_stub.c
@@ -0,0 +1,21 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_stub.c
+ * @brief Stub declarations for use when relay module is disabled.
+ **/
+
+#include "orconfig.h"
+#include "feature/relay/relay_sys.h"
+#include "lib/subsys/subsys.h"
+
+const struct subsys_fns_t sys_relay = {
+  .name = "relay",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = false,
+  .level = RELAY_SUBSYS_LEVEL,
+};
diff --git a/src/feature/relay/relay_sys.c b/src/feature/relay/relay_sys.c
new file mode 100644
index 0000000000..2e90740925
--- /dev/null
+++ b/src/feature/relay/relay_sys.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.c
+ * @brief Subsystem definitions for the relay module.
+ **/
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "feature/relay/dns.h"
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/onion_queue.h"
+#include "feature/relay/relay_periodic.h"
+#include "feature/relay/relay_sys.h"
+#include "feature/relay/routerkeys.h"
+#include "feature/relay/router.h"
+
+#include "lib/subsys/subsys.h"
+
+static int
+subsys_relay_initialize(void)
+{
+  relay_register_periodic_events();
+  return 0;
+}
+
+static void
+subsys_relay_shutdown(void)
+{
+  dns_free_all();
+  ext_orport_free_all();
+  clear_pending_onions();
+  routerkeys_free_all();
+  router_free_all();
+}
+
+const struct subsys_fns_t sys_relay = {
+  .name = "relay",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = RELAY_SUBSYS_LEVEL,
+  .initialize = subsys_relay_initialize,
+  .shutdown = subsys_relay_shutdown,
+};
diff --git a/src/feature/relay/relay_sys.h b/src/feature/relay/relay_sys.h
new file mode 100644
index 0000000000..9bad93a6c9
--- /dev/null
+++ b/src/feature/relay/relay_sys.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file relay_sys.h
+ * @brief Header for feature/relay/relay_sys.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_RELAY_SYS_H
+#define TOR_FEATURE_RELAY_RELAY_SYS_H
+
+extern const struct subsys_fns_t sys_relay;
+
+/**
+ * Subsystem level for the relay system.
+ *
+ * Defined here so that it can be shared between the real and stub
+ * definitions.
+ **/
+#define RELAY_SUBSYS_LEVEL 50
+
+#endif /* !defined(TOR_FEATURE_RELAY_RELAY_SYS_H) */
diff --git a/src/feature/relay/router.c b/src/feature/relay/router.c
index 283f7c326b..75a3d2f35c 100644
--- a/src/feature/relay/router.c
+++ b/src/feature/relay/router.c
@@ -1,13 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTER_PRIVATE
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 #include "app/config/statefile.h"
 #include "app/main/main.h"
 #include "core/mainloop/connection.h"
@@ -16,7 +17,7 @@
 #include "core/or/policies.h"
 #include "core/or/protover.h"
 #include "feature/client/transports.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirauth/process_descs.h"
 #include "feature/dircache/dirserv.h"
 #include "feature/dirclient/dirclient.h"
@@ -30,10 +31,13 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nickname.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
 #include "feature/relay/dns.h"
+#include "feature/relay/relay_config.h"
+#include "feature/relay/relay_find_addr.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routerkeys.h"
 #include "feature/relay/routermode.h"
@@ -49,6 +53,7 @@
 #include "lib/encoding/confline.h"
 #include "lib/osinfo/uname.h"
 #include "lib/tls/tortls.h"
+#include "lib/version/torversion.h"
 
 #include "feature/dirauth/authmode.h"
 
@@ -58,6 +63,7 @@
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
@@ -149,6 +155,8 @@ routerinfo_err_to_string(int err)
       return "Cannot generate descriptor";
     case TOR_ROUTERINFO_ERROR_DESC_REBUILDING:
       return "Descriptor still rebuilding - not ready yet";
+    case TOR_ROUTERINFO_ERROR_INTERNAL_BUG:
+      return "Internal bug, see logs for details";
   }
 
   log_warn(LD_BUG, "unknown routerinfo error %d - shouldn't happen", err);
@@ -191,8 +199,8 @@ set_onion_key(crypto_pk_t *k)
 
 /** Return the current onion key.  Requires that the onion key has been
  * loaded or generated. */
-crypto_pk_t *
-get_onion_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_onion_key,(void))
 {
   tor_assert(onionkey);
   return onionkey;
@@ -239,7 +247,7 @@ expire_old_onion_keys(void)
     lastonionkey = NULL;
   }
 
-  /* We zero out the keypair. See the tor_mem_is_zero() check made in
+  /* We zero out the keypair. See the fast_mem_is_zero() check made in
    * construct_ntor_key_map() below. */
   memset(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
@@ -266,11 +274,12 @@ expire_old_onion_keys(void)
 
 /** Return the current secret onion key for the ntor handshake. Must only
  * be called from the main thread. */
-static const curve25519_keypair_t *
-get_current_curve25519_keypair(void)
+MOCK_IMPL(STATIC const struct curve25519_keypair_t *,
+get_current_curve25519_keypair,(void))
 {
   return &curve25519_onion_key;
 }
+
 /** Return a map from KEYID (the key itself) to keypairs for use in the ntor
  * handshake. Must only be called from the main thread. */
 di_digest256_map_t *
@@ -281,12 +290,12 @@ construct_ntor_key_map(void)
   const uint8_t *cur_pk = curve25519_onion_key.pubkey.public_key;
   const uint8_t *last_pk = last_curve25519_onion_key.pubkey.public_key;
 
-  if (!tor_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
+  if (!fast_mem_is_zero((const char *)cur_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, cur_pk,
                     tor_memdup(&curve25519_onion_key,
                                sizeof(curve25519_keypair_t)));
   }
-  if (!tor_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
+  if (!fast_mem_is_zero((const char*)last_pk, CURVE25519_PUBKEY_LEN) &&
       tor_memneq(cur_pk, last_pk, CURVE25519_PUBKEY_LEN)) {
     dimap_add_entry(&m, last_pk,
                     tor_memdup(&last_curve25519_onion_key,
@@ -337,6 +346,16 @@ set_server_identity_key(crypto_pk_t *k)
   }
 }
 
+#ifdef TOR_UNIT_TESTS
+/** Testing only -- set the server's RSA identity digest to
+ * be <b>digest</b> */
+void
+set_server_identity_key_digest_testing(const uint8_t *digest)
+{
+  memcpy(server_identitykey_digest, digest, DIGEST_LEN);
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
 /** Make sure that we have set up our identity keys to match or not match as
  * appropriate, and die with an assertion if we have not. */
 static void
@@ -356,11 +375,13 @@ assert_identity_keys_ok(void)
   }
 }
 
+#ifdef HAVE_MODULE_RELAY
+
 /** Returns the current server identity key; requires that the key has
  * been set, and that we are running as a Tor server.
  */
-crypto_pk_t *
-get_server_identity_key(void)
+MOCK_IMPL(crypto_pk_t *,
+get_server_identity_key,(void))
 {
   tor_assert(server_identitykey);
   tor_assert(server_mode(get_options()));
@@ -368,6 +389,8 @@ get_server_identity_key(void)
   return server_identitykey;
 }
 
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 /** Return true iff we are a server and the server identity key
  * has been set. */
 int
@@ -634,7 +657,7 @@ load_authority_keyset(int legacy, crypto_pk_t **key_out,
                fname);
     goto done;
   }
-  parsed = authority_cert_parse_from_string(cert, &eos);
+  parsed = authority_cert_parse_from_string(cert, strlen(cert), &eos);
   if (!parsed) {
     log_warn(LD_DIR, "Unable to parse certificate in %s", fname);
     goto done;
@@ -727,8 +750,8 @@ v3_authority_check_key_expiry(void)
 }
 
 /** Get the lifetime of an onion key in days. This value is defined by the
- * network consesus parameter "onion-key-rotation-days". Always returns a value
- * between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
+ * network consensus parameter "onion-key-rotation-days". Always returns a
+ * value between <b>MIN_ONION_KEY_LIFETIME_DAYS</b> and
  * <b>MAX_ONION_KEY_LIFETIME_DAYS</b>.
  */
 static int
@@ -742,7 +765,7 @@ get_onion_key_rotation_days_(void)
 }
 
 /** Get the current lifetime of an onion key in seconds. This value is defined
- * by the network consesus parameter "onion-key-rotation-days", but the value
+ * by the network consensus parameter "onion-key-rotation-days", but the value
  * is converted to seconds.
  */
 int
@@ -752,7 +775,7 @@ get_onion_key_lifetime(void)
 }
 
 /** Get the grace period of an onion key in seconds. This value is defined by
- * the network consesus parameter "onion-key-grace-period-days", but the value
+ * the network consensus parameter "onion-key-grace-period-days", but the value
  * is converted to seconds.
  */
 int
@@ -866,15 +889,6 @@ init_keys_common(void)
   if (!key_lock)
     key_lock = tor_mutex_new();
 
-  /* There are a couple of paths that put us here before we've asked
-   * openssl to initialize itself. */
-  if (crypto_global_init(get_options()->HardwareAccel,
-                         get_options()->AccelName,
-                         get_options()->AccelDir)) {
-    log_err(LD_BUG, "Unable to initialize OpenSSL. Exiting.");
-    return -1;
-  }
-
   return 0;
 }
 
@@ -1031,7 +1045,7 @@ init_keys(void)
       return -1;
 
     keydir = get_keydir_fname("secret_onion_key_ntor.old");
-    if (tor_mem_is_zero((const char *)
+    if (fast_mem_is_zero((const char *)
                            last_curve25519_onion_key.pubkey.public_key,
                         CURVE25519_PUBKEY_LEN) &&
         file_status(keydir) == FN_FILE) {
@@ -1062,8 +1076,10 @@ init_keys(void)
   if (authdir_mode_v3(options)) {
     const char *m = NULL;
     routerinfo_t *ri;
-    /* We need to add our own fingerprint so it gets recognized. */
-    if (dirserv_add_own_fingerprint(get_server_identity_key())) {
+    /* We need to add our own fingerprint and ed25519 key so it gets
+     * recognized. */
+    if (dirserv_add_own_fingerprint(get_server_identity_key(),
+                                    get_master_identity_key())) {
       log_err(LD_GENERAL,"Error adding own fingerprint to set of relays");
       return -1;
     }
@@ -1202,7 +1218,7 @@ router_should_be_dirserver(const or_options_t *options, int dir_port)
      * much larger effect on output than input so there is no reason to turn it
      * off if using AccountingRule in. */
     int interval_length = accounting_get_interval_length();
-    uint32_t effective_bw = get_effective_bwrate(options);
+    uint32_t effective_bw = relay_get_effective_bwrate(options);
     uint64_t acc_bytes;
     if (!interval_length) {
       log_warn(LD_BUG, "An accounting interval is not allowed to be zero "
@@ -1384,7 +1400,7 @@ consider_publishable_server(int force)
 }
 
 /** Return the port of the first active listener of type
- *  <b>listener_type</b>. */
+ *  <b>listener_type</b>. Returns 0 if no port is found. */
 /** XXX not a very good interface. it's not reliable when there are
     multiple listeners. */
 uint16_t
@@ -1406,8 +1422,7 @@ router_get_active_listener_port_by_type_af(int listener_type,
 
 /** Return the port that we should advertise as our ORPort; this is either
  * the one configured in the ORPort option, or the one we actually bound to
- * if ORPort is "auto".
- */
+ * if ORPort is "auto". Returns 0 if no port is found. */
 uint16_t
 router_get_advertised_or_port(const or_options_t *options)
 {
@@ -1456,7 +1471,7 @@ router_get_advertised_ipv6_or_ap(const or_options_t *options,
                                                       AF_INET6);
 
   if (!addr || port == 0) {
-    log_info(LD_CONFIG, "There is no advertised IPv6 ORPort.");
+    log_debug(LD_CONFIG, "There is no advertised IPv6 ORPort.");
     return;
   }
 
@@ -1477,6 +1492,24 @@ router_get_advertised_ipv6_or_ap(const or_options_t *options,
   ipv6_ap_out->port = port;
 }
 
+/** Returns true if this router has an advertised IPv6 ORPort. */
+bool
+router_has_advertised_ipv6_orport(const or_options_t *options)
+{
+  tor_addr_port_t ipv6_ap;
+  router_get_advertised_ipv6_or_ap(options, &ipv6_ap);
+  return tor_addr_port_is_valid_ap(&ipv6_ap, 0);
+}
+
+/** Returns true if this router has an advertised IPv6 ORPort. */
+MOCK_IMPL(bool,
+router_can_extend_over_ipv6,(const or_options_t *options))
+{
+  /* We might add some extra checks here, such as ExtendAllowIPv6Addresses
+  * from ticket 33818. */
+  return router_has_advertised_ipv6_orport(options);
+}
+
 /** Return the port that we should advertise as our DirPort;
  * this is one of three possibilities:
  * The one that is passed as <b>dirport</b> if the DirPort option is 0, or
@@ -1511,9 +1544,9 @@ static extrainfo_t *desc_extrainfo = NULL;
 static const char *desc_gen_reason = "uninitialized reason";
 /** Since when has our descriptor been "clean"?  0 if we need to regenerate it
  * now. */
-static time_t desc_clean_since = 0;
+STATIC time_t desc_clean_since = 0;
 /** Why did we mark the descriptor dirty? */
-static const char *desc_dirty_reason = "Tor just started";
+STATIC const char *desc_dirty_reason = "Tor just started";
 /** Boolean: do we need to regenerate the above? */
 static int desc_needs_upload = 0;
 
@@ -1732,45 +1765,6 @@ router_get_descriptor_gen_reason(void)
   return desc_gen_reason;
 }
 
-/** A list of nicknames that we've warned about including in our family
- * declaration verbatim rather than as digests. */
-static smartlist_t *warned_nonexistent_family = NULL;
-
-static int router_guess_address_from_dir_headers(uint32_t *guess);
-
-/** Make a current best guess at our address, either because
- * it's configured in torrc, or because we've learned it from
- * dirserver headers. Place the answer in *<b>addr</b> and return
- * 0 on success, else return -1 if we have no guess.
- *
- * If <b>cache_only</b> is true, just return any cached answers, and
- * don't try to get any new answers.
- */
-MOCK_IMPL(int,
-router_pick_published_address,(const or_options_t *options, uint32_t *addr,
-                               int cache_only))
-{
-  /* First, check the cached output from resolve_my_address(). */
-  *addr = get_last_resolved_addr();
-  if (*addr)
-    return 0;
-
-  /* Second, consider doing a resolve attempt right here. */
-  if (!cache_only) {
-    if (resolve_my_address(LOG_INFO, options, addr, NULL, NULL) >= 0) {
-      log_info(LD_CONFIG,"Success: chose address '%s'.", fmt_addr32(*addr));
-      return 0;
-    }
-  }
-
-  /* Third, check the cached output from router_new_address_suggestion(). */
-  if (router_guess_address_from_dir_headers(addr) >= 0)
-    return 0;
-
-  /* We have no useful cached answers. Return failure. */
-  return -1;
-}
-
 /* Like router_check_descriptor_address_consistency, but specifically for the
  * ORPort or DirPort.
  * listener_type is either CONN_TYPE_OR_LISTENER or CONN_TYPE_DIR_LISTENER. */
@@ -1848,26 +1842,159 @@ router_check_descriptor_address_consistency(uint32_t ipv4h_desc_addr)
                                                    CONN_TYPE_DIR_LISTENER);
 }
 
-/** Build a fresh routerinfo, signed server descriptor, and extra-info document
- * for this OR. Set r to the generated routerinfo, e to the generated
- * extra-info document. Return 0 on success, -1 on temporary error. Failure to
- * generate an extra-info document is not an error and is indicated by setting
- * e to NULL. Caller is responsible for freeing generated documents if 0 is
- * returned.
+/** A list of nicknames that we've warned about including in our family,
+ * for one reason or another. */
+static smartlist_t *warned_family = NULL;
+
+/**
+ * Return a new smartlist containing the family members configured in
+ * <b>options</b>.  Warn about invalid or missing entries.  Return NULL
+ * if this relay should not declare a family.
+ **/
+STATIC smartlist_t *
+get_my_declared_family(const or_options_t *options)
+{
+  if (!options->MyFamily)
+    return NULL;
+
+  if (options->BridgeRelay)
+    return NULL;
+
+  if (!warned_family)
+    warned_family = smartlist_new();
+
+  smartlist_t *declared_family = smartlist_new();
+  config_line_t *family;
+
+  /* First we try to get the whole family in the form of hexdigests. */
+  for (family = options->MyFamily; family; family = family->next) {
+    char *name = family->value;
+    const node_t *member;
+    if (options->Nickname && !strcasecmp(name, options->Nickname))
+      continue; /* Don't list ourself by nickname, that's redundant */
+    else
+      member = node_get_by_nickname(name, 0);
+
+    if (!member) {
+      /* This node doesn't seem to exist, so warn about it if it is not
+       * a hexdigest. */
+      int is_legal = is_legal_nickname_or_hexdigest(name);
+      if (!smartlist_contains_string(warned_family, name) &&
+          !is_legal_hexdigest(name)) {
+        if (is_legal)
+          log_warn(LD_CONFIG,
+                   "There is a router named %s in my declared family, but "
+                   "I have no descriptor for it. I'll use the nickname "
+                   "as is, but this may confuse clients. Please list it "
+                   "by identity digest instead.", escaped(name));
+        else
+          log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                   "family, but that isn't a legal digest or nickname. "
+                   "Skipping it.", escaped(name));
+        smartlist_add_strdup(warned_family, name);
+      }
+      if (is_legal) {
+        smartlist_add_strdup(declared_family, name);
+      }
+    } else {
+      /* List the node by digest. */
+      char *fp = tor_malloc(HEX_DIGEST_LEN+2);
+      fp[0] = '$';
+      base16_encode(fp+1,HEX_DIGEST_LEN+1,
+                    member->identity, DIGEST_LEN);
+      smartlist_add(declared_family, fp);
+
+      if (! is_legal_hexdigest(name) &&
+          !smartlist_contains_string(warned_family, name)) {
+        /* Warn if this node was not specified by hexdigest. */
+        log_warn(LD_CONFIG, "There is a router named %s in my declared "
+                 "family, but it wasn't listed by digest. Please consider "
+                 "saying %s instead, if that's what you meant.",
+                 escaped(name), fp);
+        smartlist_add_strdup(warned_family, name);
+      }
+    }
+  }
+
+  /* Now declared_family should have the closest we can come to the
+   * identities that the user wanted.
+   *
+   * Unlike older versions of Tor, we _do_ include our own identity: this
+   * helps microdescriptor compression, and helps in-memory compression
+   * on clients. */
+  nodefamily_t *nf = nodefamily_from_members(declared_family,
+                                     router_get_my_id_digest(),
+                                     NF_WARN_MALFORMED,
+                                     NULL);
+  SMARTLIST_FOREACH(declared_family, char *, s, tor_free(s));
+  smartlist_free(declared_family);
+  if (!nf) {
+    return NULL;
+  }
+
+  char *s = nodefamily_format(nf);
+  nodefamily_free(nf);
+
+  smartlist_t *result = smartlist_new();
+  smartlist_split_string(result, s, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  tor_free(s);
+
+  if (smartlist_len(result) == 1) {
+    /* This is a one-element list containing only ourself; instead return
+     * nothing */
+    const char *singleton = smartlist_get(result, 0);
+    bool is_me = false;
+    if (singleton[0] == '$') {
+      char d[DIGEST_LEN];
+      int n = base16_decode(d, sizeof(d), singleton+1, strlen(singleton+1));
+      if (n == DIGEST_LEN &&
+          fast_memeq(d, router_get_my_id_digest(), DIGEST_LEN)) {
+        is_me = true;
+      }
+    }
+    if (!is_me) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Found a singleton family list with an element "
+               "that wasn't us! Element was %s", escaped(singleton));
+      // LCOV_EXCL_STOP
+    } else {
+      SMARTLIST_FOREACH(result, char *, cp, tor_free(cp));
+      smartlist_free(result);
+      return NULL;
+    }
+  }
+
+  return result;
+}
+
+/** Allocate a fresh, unsigned routerinfo for this OR, without any of the
+ * fields that depend on the corresponding extrainfo.
+ *
+ * On success, set ri_out to the new routerinfo, and return 0.
+ * Caller is responsible for freeing the generated routerinfo.
+ *
+ * Returns a negative value and sets ri_out to NULL on temporary error.
  */
-int
-router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+MOCK_IMPL(STATIC int,
+router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out))
 {
-  routerinfo_t *ri;
-  extrainfo_t *ei;
+  routerinfo_t *ri = NULL;
   uint32_t addr;
   char platform[256];
   int hibernating = we_are_hibernating();
   const or_options_t *options = get_options();
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
+  if (BUG(!ri_out)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
 
   if (router_pick_published_address(options, &addr, 0) < 0) {
     log_warn(LD_CONFIG, "Don't know my address while generating descriptor");
-    return TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    result = TOR_ROUTERINFO_ERROR_NO_EXT_ADDR;
+    goto err;
   }
 
   /* Log a message if the address in the descriptor doesn't match the ORPort
@@ -1894,15 +2021,15 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   /* For now, at most one IPv6 or-address is being advertised. */
   tor_addr_port_t ipv6_orport;
   router_get_advertised_ipv6_or_ap(options, &ipv6_orport);
-  /* If there is no valud IPv6 ORPort, the address and port are null. */
+  /* If there is no valid IPv6 ORPort, the address and port are null. */
   tor_addr_copy(&ri->ipv6_addr, &ipv6_orport.addr);
   ri->ipv6_orport = ipv6_orport.port;
 
   ri->identity_pkey = crypto_pk_dup_key(get_server_identity_key());
   if (BUG(crypto_pk_get_digest(ri->identity_pkey,
                            ri->cache_info.identity_digest) < 0)) {
-    routerinfo_free(ri);
-    return TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    result = TOR_ROUTERINFO_ERROR_DIGEST_FAILED;
+    goto err;
   }
   ri->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
@@ -1913,10 +2040,10 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
   ri->protocol_list = tor_strdup(protover_get_supported_protocols());
 
   /* compute ri->bandwidthrate as the min of various options */
-  ri->bandwidthrate = get_effective_bwrate(options);
+  ri->bandwidthrate = relay_get_effective_bwrate(options);
 
   /* and compute ri->bandwidthburst similarly */
-  ri->bandwidthburst = get_effective_bwburst(options);
+  ri->bandwidthburst = relay_get_effective_bwburst(options);
 
   /* Report bandwidth, unless we're hibernating or shutting down */
   ri->bandwidthcapacity = hibernating ? 0 : rep_hist_bandwidth_assess();
@@ -1939,134 +2066,260 @@ router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
     tor_free(p_tmp);
   }
 
-  if (options->MyFamily && ! options->BridgeRelay) {
-    if (!warned_nonexistent_family)
-      warned_nonexistent_family = smartlist_new();
-    ri->declared_family = smartlist_new();
-    config_line_t *family;
-    for (family = options->MyFamily; family; family = family->next) {
-       char *name = family->value;
-       const node_t *member;
-       if (!strcasecmp(name, options->Nickname))
-         continue; /* Don't list ourself, that's redundant */
-       else
-         member = node_get_by_nickname(name, 0);
-       if (!member) {
-         int is_legal = is_legal_nickname_or_hexdigest(name);
-         if (!smartlist_contains_string(warned_nonexistent_family, name) &&
-             !is_legal_hexdigest(name)) {
-           if (is_legal)
-             log_warn(LD_CONFIG,
-                      "I have no descriptor for the router named \"%s\" in my "
-                      "declared family; I'll use the nickname as is, but "
-                      "this may confuse clients.", name);
-           else
-             log_warn(LD_CONFIG, "There is a router named \"%s\" in my "
-                      "declared family, but that isn't a legal nickname. "
-                      "Skipping it.", escaped(name));
-           smartlist_add_strdup(warned_nonexistent_family, name);
-         }
-         if (is_legal) {
-           smartlist_add_strdup(ri->declared_family, name);
-         }
-       } else if (router_digest_is_me(member->identity)) {
-         /* Don't list ourself in our own family; that's redundant */
-         /* XXX shouldn't be possible */
-       } else {
-         char *fp = tor_malloc(HEX_DIGEST_LEN+2);
-         fp[0] = '$';
-         base16_encode(fp+1,HEX_DIGEST_LEN+1,
-                       member->identity, DIGEST_LEN);
-         smartlist_add(ri->declared_family, fp);
-         if (smartlist_contains_string(warned_nonexistent_family, name))
-           smartlist_string_remove(warned_nonexistent_family, name);
-       }
-    }
+  ri->declared_family = get_my_declared_family(options);
 
-    /* remove duplicates from the list */
-    smartlist_sort_strings(ri->declared_family);
-    smartlist_uniq_strings(ri->declared_family);
+  if (options->BridgeRelay) {
+    ri->purpose = ROUTER_PURPOSE_BRIDGE;
+    /* Bridges shouldn't be able to send their descriptors unencrypted,
+     anyway, since they don't have a DirPort, and always connect to the
+     bridge authority anonymously.  But just in case they somehow think of
+     sending them on an unencrypted connection, don't allow them to try. */
+    ri->cache_info.send_unencrypted = 0;
+  } else {
+    ri->purpose = ROUTER_PURPOSE_GENERAL;
+    ri->cache_info.send_unencrypted = 1;
   }
 
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  *ri_out = NULL;
+  return result;
+
+ done:
+  *ri_out = ri;
+  return 0;
+}
+
+/** Allocate and return a fresh, unsigned extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * Uses options->Nickname to set the nickname, and options->BridgeRelay to set
+ * ei->cache_info.send_unencrypted.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+static extrainfo_t *
+router_build_fresh_unsigned_extrainfo(const routerinfo_t *ri)
+{
+  extrainfo_t *ei = NULL;
+  const or_options_t *options = get_options();
+
+  if (BUG(!ri))
+    return NULL;
+
   /* Now generate the extrainfo. */
   ei = tor_malloc_zero(sizeof(extrainfo_t));
   ei->cache_info.is_extrainfo = 1;
-  strlcpy(ei->nickname, get_options()->Nickname, sizeof(ei->nickname));
+  strlcpy(ei->nickname, options->Nickname, sizeof(ei->nickname));
   ei->cache_info.published_on = ri->cache_info.published_on;
   ei->cache_info.signing_key_cert =
     tor_cert_dup(get_master_signing_key_cert());
 
   memcpy(ei->cache_info.identity_digest, ri->cache_info.identity_digest,
          DIGEST_LEN);
+
+  if (options->BridgeRelay) {
+    /* See note in router_build_fresh_routerinfo(). */
+    ei->cache_info.send_unencrypted = 0;
+  } else {
+    ei->cache_info.send_unencrypted = 1;
+  }
+
+  return ei;
+}
+
+/** Dump the extrainfo descriptor body for ei, sign it, and add the body and
+ * signature to ei->cache_info. Note that the extrainfo body is determined by
+ * ei, and some additional config and statistics state: see
+ * extrainfo_dump_to_string() for details.
+ *
+ * Return 0 on success, -1 on temporary error.
+ * If ei is NULL, logs a BUG() warning and returns -1.
+ * On error, ei->cache_info is not modified.
+ */
+static int
+router_dump_and_sign_extrainfo_descriptor_body(extrainfo_t *ei)
+{
+  if (BUG(!ei))
+    return -1;
+
   if (extrainfo_dump_to_string(&ei->cache_info.signed_descriptor_body,
                                ei, get_server_identity_key(),
                                get_master_signing_keypair()) < 0) {
     log_warn(LD_BUG, "Couldn't generate extra-info descriptor.");
-    extrainfo_free(ei);
-    ei = NULL;
-  } else {
-    ei->cache_info.signed_descriptor_len =
-      strlen(ei->cache_info.signed_descriptor_body);
-    router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
-                              ei->cache_info.signed_descriptor_len,
-                              ei->cache_info.signed_descriptor_digest);
-    crypto_digest256((char*) ei->digest256,
-                     ei->cache_info.signed_descriptor_body,
-                     ei->cache_info.signed_descriptor_len,
-                     DIGEST_SHA256);
+    return -1;
   }
 
-  /* Now finish the router descriptor. */
-  if (ei) {
-    memcpy(ri->cache_info.extra_info_digest,
-           ei->cache_info.signed_descriptor_digest,
-           DIGEST_LEN);
-    memcpy(ri->cache_info.extra_info_digest256,
-           ei->digest256,
-           DIGEST256_LEN);
-  } else {
-    /* ri was allocated with tor_malloc_zero, so there is no need to
-     * zero ri->cache_info.extra_info_digest here. */
+  ei->cache_info.signed_descriptor_len =
+    strlen(ei->cache_info.signed_descriptor_body);
+
+  router_get_extrainfo_hash(ei->cache_info.signed_descriptor_body,
+                            ei->cache_info.signed_descriptor_len,
+                            ei->cache_info.signed_descriptor_digest);
+  crypto_digest256((char*) ei->digest256,
+                   ei->cache_info.signed_descriptor_body,
+                   ei->cache_info.signed_descriptor_len,
+                   DIGEST_SHA256);
+
+  return 0;
+}
+
+/** Allocate and return a fresh, signed extrainfo for this OR, based on the
+ * routerinfo ri.
+ *
+ * If ri is NULL, logs a BUG() warning and returns NULL.
+ * Caller is responsible for freeing the generated extrainfo.
+ */
+STATIC extrainfo_t *
+router_build_fresh_signed_extrainfo(const routerinfo_t *ri)
+{
+  int result = -1;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!ri))
+    return NULL;
+
+  ei = router_build_fresh_unsigned_extrainfo(ri);
+  /* router_build_fresh_unsigned_extrainfo() should not fail. */
+  if (BUG(!ei))
+    goto err;
+
+  result = router_dump_and_sign_extrainfo_descriptor_body(ei);
+  if (result < 0)
+    goto err;
+
+  goto done;
+
+ err:
+  extrainfo_free(ei);
+  return NULL;
+
+ done:
+  return ei;
+}
+
+/** Set the fields in ri that depend on ei.
+ *
+ * If ei is NULL, logs a BUG() warning and zeroes the relevant fields.
+ */
+STATIC void
+router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                        const extrainfo_t *ei)
+{
+  if (BUG(!ei)) {
+    /* Just to be safe, zero ri->cache_info.extra_info_digest here. */
+    memset(ri->cache_info.extra_info_digest, 0, DIGEST_LEN);
+    memset(ri->cache_info.extra_info_digest256, 0, DIGEST256_LEN);
+    return;
   }
+
+  /* Now finish the router descriptor. */
+  memcpy(ri->cache_info.extra_info_digest,
+         ei->cache_info.signed_descriptor_digest,
+         DIGEST_LEN);
+  memcpy(ri->cache_info.extra_info_digest256,
+         ei->digest256,
+         DIGEST256_LEN);
+}
+
+/** Dump the descriptor body for ri, sign it, and add the body and signature to
+ * ri->cache_info. Note that the descriptor body is determined by ri, and some
+ * additional config and state: see router_dump_router_to_string() for details.
+ *
+ * Return 0 on success, and a negative value on temporary error.
+ * If ri is NULL, logs a BUG() warning and returns a negative value.
+ * On error, ri->cache_info is not modified.
+ */
+STATIC int
+router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri)
+{
+  if (BUG(!ri))
+    return TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+
   if (! (ri->cache_info.signed_descriptor_body =
           router_dump_router_to_string(ri, get_server_identity_key(),
                                        get_onion_key(),
                                        get_current_curve25519_keypair(),
                                        get_master_signing_keypair())) ) {
     log_warn(LD_BUG, "Couldn't generate router descriptor.");
-    routerinfo_free(ri);
-    extrainfo_free(ei);
     return TOR_ROUTERINFO_ERROR_CANNOT_GENERATE;
   }
+
   ri->cache_info.signed_descriptor_len =
     strlen(ri->cache_info.signed_descriptor_body);
 
-  ri->purpose =
-    options->BridgeRelay ? ROUTER_PURPOSE_BRIDGE : ROUTER_PURPOSE_GENERAL;
-  if (options->BridgeRelay) {
-    /* Bridges shouldn't be able to send their descriptors unencrypted,
-       anyway, since they don't have a DirPort, and always connect to the
-       bridge authority anonymously.  But just in case they somehow think of
-       sending them on an unencrypted connection, don't allow them to try. */
-    ri->cache_info.send_unencrypted = 0;
-    if (ei)
-      ei->cache_info.send_unencrypted = 0;
-  } else {
-    ri->cache_info.send_unencrypted = 1;
-    if (ei)
-      ei->cache_info.send_unencrypted = 1;
-  }
-
   router_get_router_hash(ri->cache_info.signed_descriptor_body,
                          strlen(ri->cache_info.signed_descriptor_body),
                          ri->cache_info.signed_descriptor_digest);
 
+  return 0;
+}
+
+/** Build a fresh routerinfo, signed server descriptor, and signed extrainfo
+ * document for this OR.
+ *
+ * Set r to the generated routerinfo, e to the generated extrainfo document.
+ * Failure to generate an extra-info document is not an error and is indicated
+ * by setting e to NULL.
+ * Return 0 on success, and a negative value on temporary error.
+ * Caller is responsible for freeing generated documents on success.
+ */
+int
+router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e)
+{
+  int result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+  routerinfo_t *ri = NULL;
+  extrainfo_t *ei = NULL;
+
+  if (BUG(!r))
+    goto err;
+
+  if (BUG(!e))
+    goto err;
+
+  result = router_build_fresh_unsigned_routerinfo(&ri);
+  if (result < 0) {
+    goto err;
+  }
+  /* If ri is NULL, then result should be negative. So this check should be
+   * unreachable. */
+  if (BUG(!ri)) {
+    result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+    goto err;
+  }
+
+  ei = router_build_fresh_signed_extrainfo(ri);
+
+  /* Failing to create an ei is not an error. */
   if (ei) {
-    tor_assert(!
-          routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
-                                                 &ri->cache_info, NULL));
+    router_update_routerinfo_from_extrainfo(ri, ei);
   }
 
+  result = router_dump_and_sign_routerinfo_descriptor_body(ri);
+  if (result < 0)
+    goto err;
+
+  if (ei) {
+     if (BUG(routerinfo_incompatible_with_extrainfo(ri->identity_pkey, ei,
+                                                    &ri->cache_info, NULL))) {
+       result = TOR_ROUTERINFO_ERROR_INTERNAL_BUG;
+       goto err;
+     }
+  }
+
+  goto done;
+
+ err:
+  routerinfo_free(ri);
+  extrainfo_free(ei);
+  *r = NULL;
+  *e = NULL;
+  return result;
+
+ done:
   *r = ri;
   *e = ei;
   return 0;
@@ -2152,7 +2405,9 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
   /* Now we see whether we want to be retrying frequently or no.  The
    * rule here is that we'll retry frequently if we aren't listed in the
    * live consensus we have, or if the publication time of the
-   * descriptor listed for us in the consensus is very old. */
+   * descriptor listed for us in the consensus is very old, or if the
+   * consensus lists us as "stale" and we haven't regenerated since the
+   * consensus was published. */
   ns = networkstatus_get_live_consensus(now);
   if (ns) {
     rs = networkstatus_vote_find_entry(ns, server_identitykey_digest);
@@ -2160,6 +2415,8 @@ mark_my_descriptor_dirty_if_too_old(time_t now)
       retry_fast_reason = "not listed in consensus";
     else if (rs->published_on < slow_cutoff)
       retry_fast_reason = "version listed in consensus is quite old";
+    else if (rs->is_staledesc && ns->valid_after > desc_clean_since)
+      retry_fast_reason = "listed as stale in consensus";
   }
 
   if (retry_fast_reason && desc_clean_since < fast_cutoff)
@@ -2231,9 +2488,13 @@ check_descriptor_bandwidth_changed(time_t now)
   }
 }
 
+// This function can be "noreturn" if relay mode is disabled and
+// ALL_BUGS_ARE_FATAL is set.
+DISABLE_GCC_WARNING("-Wmissing-noreturn")
+
 /** Note at log level severity that our best guess of address has changed from
  * <b>prev</b> to <b>cur</b>. */
-static void
+void
 log_addr_has_changed(int severity,
                      const tor_addr_t *prev,
                      const tor_addr_t *cur,
@@ -2260,6 +2521,7 @@ log_addr_has_changed(int severity,
              "Guessed our IP address as %s (source: %s).",
              addrbuf_cur, source);
 }
+ENABLE_GCC_WARNING("-Wmissing-noreturn")
 
 /** Check whether our own address as defined by the Address configuration
  * has changed. This is for routers that get their address from a service
@@ -2305,86 +2567,6 @@ check_descriptor_ipaddress_changed(time_t now)
   tor_free(hostname);
 }
 
-/** The most recently guessed value of our IP address, based on directory
- * headers. */
-static tor_addr_t last_guessed_ip = TOR_ADDR_NULL;
-
-/** A directory server <b>d_conn</b> told us our IP address is
- * <b>suggestion</b>.
- * If this address is different from the one we think we are now, and
- * if our computer doesn't actually know its IP address, then switch. */
-void
-router_new_address_suggestion(const char *suggestion,
-                              const dir_connection_t *d_conn)
-{
-  tor_addr_t addr;
-  uint32_t cur = 0;             /* Current IPv4 address.  */
-  const or_options_t *options = get_options();
-
-  /* first, learn what the IP address actually is */
-  if (tor_addr_parse(&addr, suggestion) == -1) {
-    log_debug(LD_DIR, "Malformed X-Your-Address-Is header %s. Ignoring.",
-              escaped(suggestion));
-    return;
-  }
-
-  log_debug(LD_DIR, "Got X-Your-Address-Is: %s.", suggestion);
-
-  if (!server_mode(options)) {
-    tor_addr_copy(&last_guessed_ip, &addr);
-    return;
-  }
-
-  /* XXXX ipv6 */
-  cur = get_last_resolved_addr();
-  if (cur ||
-      resolve_my_address(LOG_INFO, options, &cur, NULL, NULL) >= 0) {
-    /* We're all set -- we already know our address. Great. */
-    tor_addr_from_ipv4h(&last_guessed_ip, cur); /* store it in case we
-                                                   need it later */
-    return;
-  }
-  if (tor_addr_is_internal(&addr, 0)) {
-    /* Don't believe anybody who says our IP is, say, 127.0.0.1. */
-    return;
-  }
-  if (tor_addr_eq(&d_conn->base_.addr, &addr)) {
-    /* Don't believe anybody who says our IP is their IP. */
-    log_debug(LD_DIR, "A directory server told us our IP address is %s, "
-              "but they are just reporting their own IP address. Ignoring.",
-              suggestion);
-    return;
-  }
-
-  /* Okay.  We can't resolve our own address, and X-Your-Address-Is is giving
-   * us an answer different from what we had the last time we managed to
-   * resolve it. */
-  if (!tor_addr_eq(&last_guessed_ip, &addr)) {
-    control_event_server_status(LOG_NOTICE,
-                                "EXTERNAL_ADDRESS ADDRESS=%s METHOD=DIRSERV",
-                                suggestion);
-    log_addr_has_changed(LOG_NOTICE, &last_guessed_ip, &addr,
-                         d_conn->base_.address);
-    ip_address_changed(0);
-    tor_addr_copy(&last_guessed_ip, &addr); /* router_rebuild_descriptor()
-                                               will fetch it */
-  }
-}
-
-/** We failed to resolve our address locally, but we'd like to build
- * a descriptor and publish / test reachability. If we have a guess
- * about our address based on directory headers, answer it and return
- * 0; else return -1. */
-static int
-router_guess_address_from_dir_headers(uint32_t *guess)
-{
-  if (!tor_addr_is_null(&last_guessed_ip)) {
-    *guess = tor_addr_to_ipv4h(&last_guessed_ip);
-    return 0;
-  }
-  return -1;
-}
-
 /** Set <b>platform</b> (max length <b>len</b>) to a NUL-terminated short
  * string describing the version of Tor and the operating system we're
  * currently running on.
@@ -2405,6 +2587,10 @@ get_platform_str(char *platform, size_t len)
 /** OR only: Given a routerinfo for this router, and an identity key to sign
  * with, encode the routerinfo as a signed server descriptor and return a new
  * string encoding the result, or NULL on failure.
+ *
+ * In addition to the fields in router, this function calls
+ * onion_key_lifetime(), get_options(), and we_are_hibernating(), and uses the
+ * results to populate some fields in the descriptor.
  */
 char *
 router_dump_router_to_string(routerinfo_t *router,
@@ -2468,11 +2654,8 @@ router_dump_router_to_string(routerinfo_t *router,
       log_err(LD_BUG,"Couldn't base64-encode signing key certificate!");
       goto err;
     }
-    if (ed25519_public_to_base64(ed_fp_base64,
-                       &router->cache_info.signing_key_cert->signing_key)<0) {
-      log_err(LD_BUG,"Couldn't base64-encode identity key\n");
-      goto err;
-    }
+    ed25519_public_to_base64(ed_fp_base64,
+                            &router->cache_info.signing_key_cert->signing_key);
     tor_asprintf(&ed_cert_line, "identity-ed25519\n"
                  "-----BEGIN ED25519 CERT-----\n"
                  "%s"
@@ -2606,6 +2789,9 @@ router_dump_router_to_string(routerinfo_t *router,
   }
 
   address = tor_dup_ip(router->addr);
+  if (!address)
+    goto err;
+
   chunks = smartlist_new();
 
   /* Generate the easy portion of the router descriptor. */
@@ -2722,8 +2908,7 @@ router_dump_router_to_string(routerinfo_t *router,
     if (ed25519_sign(&sig, (const uint8_t*)digest, DIGEST256_LEN,
                      signing_keypair) < 0)
       goto err;
-    if (ed25519_signature_to_base64(buf, &sig) < 0)
-      goto err;
+    ed25519_signature_to_base64(buf, &sig);
 
     smartlist_add_asprintf(chunks, "%s\n", buf);
   }
@@ -2862,34 +3047,26 @@ load_stats_file(const char *filename, const char *end_line, time_t now,
   return r;
 }
 
-/** Write the contents of <b>extrainfo</b> and aggregated statistics to
- * *<b>s_out</b>, signing them with <b>ident_key</b>. Return 0 on
- * success, negative on failure. */
-int
-extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
-                         crypto_pk_t *ident_key,
-                         const ed25519_keypair_t *signing_keypair)
+/** Add header strings to chunks, based on the extrainfo object extrainfo,
+ * and ed25519 keypair signing_keypair, if emit_ed_sigs is true.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_header_helper(
+                                     smartlist_t *chunks,
+                                     const extrainfo_t *extrainfo,
+                                     const ed25519_keypair_t *signing_keypair,
+                                     int emit_ed_sigs)
 {
-  const or_options_t *options = get_options();
   char identity[HEX_DIGEST_LEN+1];
   char published[ISO_TIME_LEN+1];
-  char digest[DIGEST_LEN];
-  char *bandwidth_usage;
-  int result;
-  static int write_stats_to_extrainfo = 1;
-  char sig[DIROBJ_MAX_SIG_LEN+1];
-  char *s = NULL, *pre, *contents, *cp, *s_dup = NULL;
-  time_t now = time(NULL);
-  smartlist_t *chunks = smartlist_new();
-  extrainfo_t *ei_tmp = NULL;
-  const int emit_ed_sigs = signing_keypair &&
-    extrainfo->cache_info.signing_key_cert;
   char *ed_cert_line = NULL;
+  char *pre = NULL;
+  int rv = -1;
 
   base16_encode(identity, sizeof(identity),
                 extrainfo->cache_info.identity_digest, DIGEST_LEN);
   format_iso_time(published, extrainfo->cache_info.published_on);
-  bandwidth_usage = rep_hist_get_bandwidth_lines();
   if (emit_ed_sigs) {
     if (!extrainfo->cache_info.signing_key_cert->signing_key_included ||
         !ed25519_pubkey_eq(&extrainfo->cache_info.signing_key_cert->signed_key,
@@ -2915,21 +3092,64 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     ed_cert_line = tor_strdup("");
   }
 
-  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n%s",
+  /* This is the first chunk in the file. If the file is too big, other chunks
+   * are removed. So we must only add one chunk here. */
+  tor_asprintf(&pre, "extra-info %s %s\n%spublished %s\n",
                extrainfo->nickname, identity,
                ed_cert_line,
-               published, bandwidth_usage);
+               published);
   smartlist_add(chunks, pre);
 
-  if (geoip_is_loaded(AF_INET))
-    smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
-                           geoip_db_digest(AF_INET));
-  if (geoip_is_loaded(AF_INET6))
-    smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
-                           geoip_db_digest(AF_INET6));
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  tor_free(ed_cert_line);
+  return rv;
+}
+
+/** Add pluggable transport and statistics strings to chunks, skipping
+ * statistics if write_stats_to_extrainfo is false.
+ * Helper for extrainfo_dump_to_string().
+ * Can not fail. */
+static void
+extrainfo_dump_to_string_stats_helper(smartlist_t *chunks,
+                                      int write_stats_to_extrainfo)
+{
+  const or_options_t *options = get_options();
+  char *contents = NULL;
+  time_t now = time(NULL);
+
+  /* If the file is too big, these chunks are removed, starting with the last
+   * chunk. So each chunk must be a complete line, and the file must be valid
+   * after each chunk. */
+
+  /* Add information about the pluggable transports we support, even if we
+   * are not publishing statistics. This information is needed by BridgeDB
+   * to distribute bridges. */
+  if (options->ServerTransportPlugin) {
+    char *pluggable_transports = pt_get_extra_info_descriptor_string();
+    if (pluggable_transports)
+      smartlist_add(chunks, pluggable_transports);
+  }
 
   if (options->ExtraInfoStatistics && write_stats_to_extrainfo) {
     log_info(LD_GENERAL, "Adding stats to extra-info descriptor.");
+    /* Bandwidth usage stats don't have their own option */
+    {
+      contents = rep_hist_get_bandwidth_lines();
+      smartlist_add(chunks, contents);
+    }
+    /* geoip hashes aren't useful unless we are publishing other stats */
+    if (geoip_is_loaded(AF_INET))
+      smartlist_add_asprintf(chunks, "geoip-db-digest %s\n",
+                             geoip_db_digest(AF_INET));
+    if (geoip_is_loaded(AF_INET6))
+      smartlist_add_asprintf(chunks, "geoip6-db-digest %s\n",
+                             geoip_db_digest(AF_INET6));
     if (options->DirReqStatistics &&
         load_stats_file("stats"PATH_SEPARATOR"dirreq-stats",
                         "dirreq-stats-end", now, &contents) > 0) {
@@ -2965,50 +3185,140 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
       if (contents)
         smartlist_add(chunks, contents);
     }
+    /* bridge statistics */
+    if (should_record_bridge_info(options)) {
+      const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
+      if (bridge_stats) {
+        smartlist_add_strdup(chunks, bridge_stats);
+      }
+    }
   }
+}
 
-  /* Add information about the pluggable transports we support. */
-  if (options->ServerTransportPlugin) {
-    char *pluggable_transports = pt_get_extra_info_descriptor_string();
-    if (pluggable_transports)
-      smartlist_add(chunks, pluggable_transports);
-  }
+/** Add an ed25519 signature of chunks to chunks, using the ed25519 keypair
+ * signing_keypair.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_ed_sig_helper(
+                                     smartlist_t *chunks,
+                                     const ed25519_keypair_t *signing_keypair)
+{
+  char sha256_digest[DIGEST256_LEN];
+  ed25519_signature_t ed_sig;
+  char buf[ED25519_SIG_BASE64_LEN+1];
+  int rv = -1;
+
+  /* These are two of the three final chunks in the file. If the file is too
+   * big, other chunks are removed. So we must only add two chunks here. */
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
+  crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
+                                 ED_DESC_SIGNATURE_PREFIX,
+                                 chunks, "", DIGEST_SHA256);
+  if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
+                   signing_keypair) < 0)
+    goto err;
+  ed25519_signature_to_base64(buf, &ed_sig);
 
-  if (should_record_bridge_info(options) && write_stats_to_extrainfo) {
-    const char *bridge_stats = geoip_get_bridge_stats_extrainfo(now);
-    if (bridge_stats) {
-      smartlist_add_strdup(chunks, bridge_stats);
-    }
+  smartlist_add_asprintf(chunks, "%s\n", buf);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Add an RSA signature of extrainfo_string to chunks, using the RSA key
+ * ident_key.
+ * Helper for extrainfo_dump_to_string().
+ * Returns 0 on success, negative on failure. */
+static int
+extrainfo_dump_to_string_rsa_sig_helper(smartlist_t *chunks,
+                                        crypto_pk_t *ident_key,
+                                        const char *extrainfo_string)
+{
+  char sig[DIROBJ_MAX_SIG_LEN+1];
+  char digest[DIGEST_LEN];
+  int rv = -1;
+
+  memset(sig, 0, sizeof(sig));
+  if (router_get_extrainfo_hash(extrainfo_string, strlen(extrainfo_string),
+                                digest) < 0 ||
+      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
+                                     ident_key) < 0) {
+    log_warn(LD_BUG, "Could not append signature to extra-info "
+                     "descriptor.");
+    goto err;
   }
+  smartlist_add_strdup(chunks, sig);
+
+  rv = 0;
+  goto done;
+
+ err:
+  rv = -1;
+
+ done:
+  return rv;
+}
+
+/** Write the contents of <b>extrainfo</b>, to * *<b>s_out</b>, signing them
+ * with <b>ident_key</b>.
+ *
+ * If ExtraInfoStatistics is 1, also write aggregated statistics and related
+ * configuration data before signing. Most statistics also have an option that
+ * enables or disables that particular statistic.
+ *
+ * Always write pluggable transport lines.
+ *
+ * Return 0 on success, negative on failure. */
+int
+extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
+                         crypto_pk_t *ident_key,
+                         const ed25519_keypair_t *signing_keypair)
+{
+  int result;
+  static int write_stats_to_extrainfo = 1;
+  char *s = NULL, *cp, *s_dup = NULL;
+  smartlist_t *chunks = smartlist_new();
+  extrainfo_t *ei_tmp = NULL;
+  const int emit_ed_sigs = signing_keypair &&
+    extrainfo->cache_info.signing_key_cert;
+  int rv = 0;
+
+  rv = extrainfo_dump_to_string_header_helper(chunks, extrainfo,
+                                              signing_keypair,
+                                              emit_ed_sigs);
+  if (rv < 0)
+    goto err;
+
+  extrainfo_dump_to_string_stats_helper(chunks, write_stats_to_extrainfo);
 
   if (emit_ed_sigs) {
-    char sha256_digest[DIGEST256_LEN];
-    smartlist_add_strdup(chunks, "router-sig-ed25519 ");
-    crypto_digest_smartlist_prefix(sha256_digest, DIGEST256_LEN,
-                                   ED_DESC_SIGNATURE_PREFIX,
-                                   chunks, "", DIGEST_SHA256);
-    ed25519_signature_t ed_sig;
-    char buf[ED25519_SIG_BASE64_LEN+1];
-    if (ed25519_sign(&ed_sig, (const uint8_t*)sha256_digest, DIGEST256_LEN,
-                     signing_keypair) < 0)
-      goto err;
-    if (ed25519_signature_to_base64(buf, &ed_sig) < 0)
+    rv = extrainfo_dump_to_string_ed_sig_helper(chunks, signing_keypair);
+    if (rv < 0)
       goto err;
-
-    smartlist_add_asprintf(chunks, "%s\n", buf);
   }
 
+  /* This is one of the three final chunks in the file. If the file is too big,
+   * other chunks are removed. So we must only add one chunk here. */
   smartlist_add_strdup(chunks, "router-signature\n");
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
   while (strlen(s) > MAX_EXTRAINFO_UPLOAD_SIZE - DIROBJ_MAX_SIG_LEN) {
     /* So long as there are at least two chunks (one for the initial
      * extra-info line and one for the router-signature), we can keep removing
-     * things. */
-    if (smartlist_len(chunks) > 2) {
-      /* We remove the next-to-last element (remember, len-1 is the last
-         element), since we need to keep the router-signature element. */
-      int idx = smartlist_len(chunks) - 2;
+     * things. If emit_ed_sigs is true, we also keep 2 additional chunks at the
+     * end for the ed25519 signature. */
+    const int required_chunks = emit_ed_sigs ? 4 : 2;
+    if (smartlist_len(chunks) > required_chunks) {
+      /* We remove the next-to-last or 4th-last element (remember, len-1 is the
+       * last element), since we need to keep the router-signature elements. */
+      int idx = smartlist_len(chunks) - required_chunks;
       char *e = smartlist_get(chunks, idx);
       smartlist_del_keeporder(chunks, idx);
       log_warn(LD_GENERAL, "We just generated an extra-info descriptor "
@@ -3025,15 +3335,10 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
     }
   }
 
-  memset(sig, 0, sizeof(sig));
-  if (router_get_extrainfo_hash(s, strlen(s), digest) < 0 ||
-      router_append_dirobj_signature(sig, sizeof(sig), digest, DIGEST_LEN,
-                                     ident_key) < 0) {
-    log_warn(LD_BUG, "Could not append signature to extra-info "
-                     "descriptor.");
+  rv = extrainfo_dump_to_string_rsa_sig_helper(chunks, ident_key, s);
+  if (rv < 0)
     goto err;
-  }
-  smartlist_add_strdup(chunks, sig);
+
   tor_free(s);
   s = smartlist_join_strings(chunks, "", 0, NULL);
 
@@ -3069,9 +3374,7 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
   SMARTLIST_FOREACH(chunks, char *, chunk, tor_free(chunk));
   smartlist_free(chunks);
   tor_free(s_dup);
-  tor_free(ed_cert_line);
   extrainfo_free(ei_tmp);
-  tor_free(bandwidth_usage);
 
   return result;
 }
@@ -3081,9 +3384,9 @@ extrainfo_dump_to_string(char **s_out, extrainfo_t *extrainfo,
 void
 router_reset_warnings(void)
 {
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_clear(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_clear(warned_family);
   }
 }
 
@@ -3096,6 +3399,10 @@ router_free_all(void)
   crypto_pk_free(server_identitykey);
   crypto_pk_free(client_identitykey);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
   tor_mutex_free(key_lock);
   routerinfo_free(desc_routerinfo);
   extrainfo_free(desc_extrainfo);
@@ -3107,11 +3414,12 @@ router_free_all(void)
   memwipe(&curve25519_onion_key, 0, sizeof(curve25519_onion_key));
   memwipe(&last_curve25519_onion_key, 0, sizeof(last_curve25519_onion_key));
 
-  if (warned_nonexistent_family) {
-    SMARTLIST_FOREACH(warned_nonexistent_family, char *, cp, tor_free(cp));
-    smartlist_free(warned_nonexistent_family);
+  if (warned_family) {
+    SMARTLIST_FOREACH(warned_family, char *, cp, tor_free(cp));
+    smartlist_free(warned_family);
   }
 }
+
 /* From the given RSA key object, convert it to ASN-1 encoded format and set
  * the newly allocated object in onion_pkey_out. The length of the key is set
  * in onion_pkey_len_out. */
diff --git a/src/feature/relay/router.h b/src/feature/relay/router.h
index ab1f771017..50790a73dd 100644
--- a/src/feature/relay/router.h
+++ b/src/feature/relay/router.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,11 +23,18 @@ struct ed25519_keypair_t;
 #define TOR_ROUTERINFO_ERROR_DIGEST_FAILED   (-4)
 #define TOR_ROUTERINFO_ERROR_CANNOT_GENERATE (-5)
 #define TOR_ROUTERINFO_ERROR_DESC_REBUILDING (-6)
+#define TOR_ROUTERINFO_ERROR_INTERNAL_BUG    (-7)
 
-crypto_pk_t *get_onion_key(void);
+MOCK_DECL(crypto_pk_t *,get_onion_key,(void));
 time_t get_onion_key_set_at(void);
 void set_server_identity_key(crypto_pk_t *k);
-crypto_pk_t *get_server_identity_key(void);
+/* Some compilers are clever enough to know that when relay mode is disabled,
+ * this function never returns. */
+#ifdef HAVE_MODULE_RELAY
+MOCK_DECL(crypto_pk_t *,get_server_identity_key,(void));
+#else
+#define get_server_identity_key() (tor_abort_(),NULL)
+#endif
 int server_identity_key_is_set(void);
 void set_client_identity_key(crypto_pk_t *k);
 crypto_pk_t *get_tlsclient_identity_key(void);
@@ -61,6 +68,8 @@ uint16_t router_get_active_listener_port_by_type_af(int listener_type,
 uint16_t router_get_advertised_or_port(const or_options_t *options);
 void router_get_advertised_ipv6_or_ap(const or_options_t *options,
                                       tor_addr_port_t *ipv6_ap_out);
+bool router_has_advertised_ipv6_orport(const or_options_t *options);
+MOCK_DECL(bool, router_can_extend_over_ipv6,(const or_options_t *options));
 uint16_t router_get_advertised_or_port_by_af(const or_options_t *options,
                                              sa_family_t family);
 uint16_t router_get_advertised_dir_port(const or_options_t *options,
@@ -78,8 +87,6 @@ void mark_my_descriptor_dirty(const char *reason);
 void check_descriptor_bandwidth_changed(time_t now);
 void check_descriptor_ipaddress_changed(time_t now);
 int router_has_bandwidth_to_be_dirserver(const or_options_t *options);
-void router_new_address_suggestion(const char *suggestion,
-                                   const dir_connection_t *d_conn);
 int router_compare_to_my_exit_policy(const tor_addr_t *addr, uint16_t port);
 MOCK_DECL(int, router_my_exit_policy_is_reject_star,(void));
 MOCK_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
@@ -91,9 +98,6 @@ int router_digest_is_me(const char *digest);
 const uint8_t *router_get_my_id_digest(void);
 int router_extrainfo_digest_is_me(const char *digest);
 int router_is_me(const routerinfo_t *router);
-MOCK_DECL(int,router_pick_published_address,(const or_options_t *options,
-                                             uint32_t *addr,
-                                             int cache_only));
 int router_build_fresh_descriptor(routerinfo_t **r, extrainfo_t **e);
 int router_rebuild_descriptor(int force);
 char *router_dump_router_to_string(routerinfo_t *router,
@@ -111,14 +115,34 @@ int extrainfo_dump_to_string(char **s, extrainfo_t *extrainfo,
 const char *routerinfo_err_to_string(int err);
 int routerinfo_err_is_transient(int err);
 
+void log_addr_has_changed(int severity, const tor_addr_t *prev,
+                          const tor_addr_t *cur, const char *source);
+
 void router_reset_warnings(void);
-void router_reset_reachability(void);
 void router_free_all(void);
 
 #ifdef ROUTER_PRIVATE
-/* Used only by router.c and test.c */
+/* Used only by router.c and the unit tests */
 STATIC void get_platform_str(char *platform, size_t len);
 STATIC int router_write_fingerprint(int hashed);
-#endif
+STATIC smartlist_t *get_my_declared_family(const or_options_t *options);
+
+#ifdef TOR_UNIT_TESTS
+extern time_t desc_clean_since;
+extern const char *desc_dirty_reason;
+void set_server_identity_key_digest_testing(const uint8_t *digest);
+MOCK_DECL(STATIC const struct curve25519_keypair_t *,
+                                       get_current_curve25519_keypair,(void));
+
+MOCK_DECL(STATIC int,
+              router_build_fresh_unsigned_routerinfo,(routerinfo_t **ri_out));
+STATIC extrainfo_t *router_build_fresh_signed_extrainfo(
+                                                      const routerinfo_t *ri);
+STATIC void router_update_routerinfo_from_extrainfo(routerinfo_t *ri,
+                                                    const extrainfo_t *ei);
+STATIC int router_dump_and_sign_routerinfo_descriptor_body(routerinfo_t *ri);
+#endif /* defined(TOR_UNIT_TESTS) */
+
+#endif /* defined(ROUTER_PRIVATE) */
 
 #endif /* !defined(TOR_ROUTER_H) */
diff --git a/src/feature/relay/routerkeys.c b/src/feature/relay/routerkeys.c
index f639fc91e7..d3de83cb86 100644
--- a/src/feature/relay/routerkeys.c
+++ b/src/feature/relay/routerkeys.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -226,7 +226,7 @@ load_ed_keys(const or_options_t *options, time_t now)
         tor_free(fname);
       }
     }
-    if (tor_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
+    if (safe_mem_is_zero((char*)id->seckey.seckey, sizeof(id->seckey)))
       sign_signing_key_with_id = NULL;
     else
       sign_signing_key_with_id = id;
@@ -631,14 +631,14 @@ get_master_identity_keypair(void)
 }
 #endif /* defined(TOR_UNIT_TESTS) */
 
-const ed25519_keypair_t *
-get_master_signing_keypair(void)
+MOCK_IMPL(const ed25519_keypair_t *,
+get_master_signing_keypair,(void))
 {
   return master_signing_key;
 }
 
-const struct tor_cert_st *
-get_master_signing_key_cert(void)
+MOCK_IMPL(const struct tor_cert_st *,
+get_master_signing_key_cert,(void))
 {
   return signing_key_cert;
 }
@@ -706,6 +706,8 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
 
   *len_out = 0;
   if (crypto_pk_get_digest(rsa_id_key, (char*)signed_data) < 0) {
+    log_info(LD_OR, "crypto_pk_get_digest failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
   }
   memcpy(signed_data + DIGEST_LEN, master_id_key->pubkey, ED25519_PUBKEY_LEN);
@@ -713,8 +715,12 @@ make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
   int r = crypto_pk_private_sign(onion_key,
                                (char*)signature, sizeof(signature),
                                (const char*)signed_data, sizeof(signed_data));
-  if (r < 0)
+  if (r < 0) {
+    /* It's probably missing the private key */
+    log_info(LD_OR, "crypto_pk_private_sign failed in "
+                    "make_tap_onion_key_crosscert!");
     return NULL;
+  }
 
   *len_out = r;
 
diff --git a/src/feature/relay/routerkeys.h b/src/feature/relay/routerkeys.h
index 0badd34191..1fb5d724e9 100644
--- a/src/feature/relay/routerkeys.h
+++ b/src/feature/relay/routerkeys.h
@@ -1,14 +1,21 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routerkeys.h
+ * @brief Header for routerkeys.c
+ **/
+
 #ifndef TOR_ROUTERKEYS_H
 #define TOR_ROUTERKEYS_H
 
 #include "lib/crypt_ops/crypto_ed25519.h"
 
+#ifdef HAVE_MODULE_RELAY
+
 const ed25519_public_key_t *get_master_identity_key(void);
-const ed25519_keypair_t *get_master_signing_keypair(void);
-const struct tor_cert_st *get_master_signing_key_cert(void);
+MOCK_DECL(const ed25519_keypair_t *, get_master_signing_keypair,(void));
+MOCK_DECL(const struct tor_cert_st *, get_master_signing_key_cert,(void));
 
 const ed25519_keypair_t *get_current_auth_keypair(void);
 const struct tor_cert_st *get_current_link_cert_cert(void);
@@ -19,6 +26,7 @@ void get_master_rsa_crosscert(const uint8_t **cert_out,
 
 int router_ed25519_id_is_me(const ed25519_public_key_t *id);
 
+/* These are only used by router.c */
 struct tor_cert_st *make_ntor_onion_key_crosscert(
                                   const curve25519_keypair_t *onion_key,
                                   const ed25519_public_key_t *master_id_key,
@@ -37,6 +45,85 @@ int generate_ed_link_cert(const or_options_t *options, time_t now, int force);
 
 void routerkeys_free_all(void);
 
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define router_ed25519_id_is_me(id) \
+  ((void)(id), 0)
+
+static inline void *
+relay_key_is_unavailable_(void)
+{
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+#define relay_key_is_unavailable(type) \
+  ((type)(relay_key_is_unavailable_()))
+
+// Many of these can be removed once relay_handshake.c is relay-only.
+#define get_current_auth_keypair() \
+  relay_key_is_unavailable(const ed25519_keypair_t *)
+#define get_master_signing_keypair() \
+  relay_key_is_unavailable(const ed25519_keypair_t *)
+#define get_current_link_cert_cert() \
+  relay_key_is_unavailable(const struct tor_cert_st *)
+#define get_current_auth_key_cert() \
+  relay_key_is_unavailable(const struct tor_cert_st *)
+#define get_master_signing_key_cert() \
+  relay_key_is_unavailable(const struct tor_cert_st *)
+#define get_master_rsa_crosscert(cert_out, size_out) \
+  STMT_BEGIN                                         \
+  tor_assert_nonfatal_unreached();                   \
+  *(cert_out) = NULL;                                \
+  *(size_out) = 0;                                   \
+  STMT_END
+#define get_master_identity_key() \
+  relay_key_is_unavailable(const ed25519_public_key_t *)
+
+#define generate_ed_link_cert(options, now, force) \
+  ((void)(options), (void)(now), (void)(force), 0)
+#define should_make_new_ed_keys(options, now) \
+  ((void)(options), (void)(now), 0)
+
+// These can get removed once router.c becomes relay-only.
+static inline struct tor_cert_st *
+make_ntor_onion_key_crosscert(const curve25519_keypair_t *onion_key,
+                              const ed25519_public_key_t *master_id_key,
+                              time_t now, time_t lifetime,
+                              int *sign_out)
+{
+  (void)onion_key;
+  (void)master_id_key;
+  (void)now;
+  (void)lifetime;
+  *sign_out = 0;
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+static inline uint8_t *
+make_tap_onion_key_crosscert(const crypto_pk_t *onion_key,
+                             const ed25519_public_key_t *master_id_key,
+                             const crypto_pk_t *rsa_id_key,
+                             int *len_out)
+{
+  (void)onion_key;
+  (void)master_id_key;
+  (void)rsa_id_key;
+  *len_out = 0;
+  tor_assert_nonfatal_unreached();
+  return NULL;
+}
+
+/* This calls is used outside of relay mode, but only to implement
+ * CMD_KEY_EXPIRATION */
+#define log_cert_expiration()                                           \
+  (puts("Not available: Tor has been compiled without relay support"), 0)
+/* This calls is used outside of relay mode, but only to implement
+ * CMD_KEYGEN. */
+#define load_ed_keys(x,y)                                                \
+  (puts("Not available: Tor has been compiled without relay support"), 0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #ifdef TOR_UNIT_TESTS
 const ed25519_keypair_t *get_master_identity_keypair(void);
 void init_mock_ed_keys(const crypto_pk_t *rsa_identity_key);
diff --git a/src/feature/relay/routermode.c b/src/feature/relay/routermode.c
index 2a9ddeac4d..c4d8792b5b 100644
--- a/src/feature/relay/routermode.c
+++ b/src/feature/relay/routermode.c
@@ -1,14 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file routermode.c
+ * @brief Check if we're running as a relay/cache.
+ **/
+
 #include "core/or/or.h"
 
 #include "app/config/config.h"
-#include "core/mainloop/connection.h"
-#include "core/or/port_cfg_st.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
 
@@ -25,21 +28,6 @@ dir_server_mode(const or_options_t *options)
     (server_mode(options) && router_has_bandwidth_to_be_dirserver(options));
 }
 
-/** Return true iff we are trying to proxy client connections. */
-int
-proxy_mode(const or_options_t *options)
-{
-  (void)options;
-  SMARTLIST_FOREACH_BEGIN(get_configured_ports(), const port_cfg_t *, p) {
-    if (p->type == CONN_TYPE_AP_LISTENER ||
-        p->type == CONN_TYPE_AP_TRANS_LISTENER ||
-        p->type == CONN_TYPE_AP_DNS_LISTENER ||
-        p->type == CONN_TYPE_AP_NATD_LISTENER)
-      return 1;
-  } SMARTLIST_FOREACH_END(p);
-  return 0;
-}
-
 /** Return true iff we are trying to be a server.
  */
 MOCK_IMPL(int,
diff --git a/src/feature/relay/routermode.h b/src/feature/relay/routermode.h
index be535af478..6d7404968d 100644
--- a/src/feature/relay/routermode.h
+++ b/src/feature/relay/routermode.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,31 @@
 #ifndef TOR_ROUTERMODE_H
 #define TOR_ROUTERMODE_H
 
+#ifdef HAVE_MODULE_RELAY
+
 int dir_server_mode(const or_options_t *options);
 
 MOCK_DECL(int, server_mode, (const or_options_t *options));
 MOCK_DECL(int, public_server_mode, (const or_options_t *options));
 MOCK_DECL(int, advertised_server_mode, (void));
-int proxy_mode(const or_options_t *options);
 
 void set_server_advertised(int s);
 
+/** Is the relay module enabled? */
+#define have_module_relay() (1)
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define dir_server_mode(options) (((void)(options)),0)
+#define server_mode(options) (((void)(options)),0)
+#define public_server_mode(options) (((void)(options)),0)
+#define advertised_server_mode() (0)
+
+/* We shouldn't be publishing descriptors when relay mode is disabled. */
+#define set_server_advertised(s) tor_assert_nonfatal(!(s))
+
+#define have_module_relay() (0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
 #endif /* !defined(TOR_ROUTERMODE_H) */
diff --git a/src/feature/relay/selftest.c b/src/feature/relay/selftest.c
index 064eea6c46..18fe25b989 100644
--- a/src/feature/relay/selftest.c
+++ b/src/feature/relay/selftest.c
@@ -1,19 +1,17 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
  * \file selftest.c
  * \brief Relay self-testing
  *
- * Relays need to make sure that their own ports are reasonable, and estimate
+ * Relays need to make sure that their own ports are reachable, and estimate
  * their own bandwidth, before publishing.
  */
 
-#define SELFTEST_PRIVATE
-
 #include "core/or/or.h"
 
 #include "app/config/config.h"
@@ -26,7 +24,7 @@
 #include "core/or/crypt_path_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "core/or/relay.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/nodelist/authority_cert_st.h"
@@ -35,6 +33,7 @@
 #include "feature/nodelist/routerlist.h" // but...
 #include "feature/nodelist/routerset.h"
 #include "feature/nodelist/torcert.h"
+#include "feature/relay/relay_periodic.h"
 #include "feature/relay/router.h"
 #include "feature/relay/selftest.h"
 
@@ -214,6 +213,44 @@ router_do_reachability_checks(int test_or, int test_dir)
   }
 }
 
+/** We've decided to start our reachability testing. If all
+ * is set, log this to the user. Return 1 if we did, or 0 if
+ * we chose not to log anything. */
+int
+inform_testing_reachability(void)
+{
+  char dirbuf[128];
+  char *address;
+  const routerinfo_t *me = router_get_my_routerinfo();
+  if (!me)
+    return 0;
+
+  address = tor_dup_ip(me->addr);
+  if (!address)
+    return 0;
+
+  control_event_server_status(LOG_NOTICE,
+                              "CHECKING_REACHABILITY ORADDRESS=%s:%d",
+                              address, me->or_port);
+  if (me->dir_port) {
+    tor_snprintf(dirbuf, sizeof(dirbuf), " and DirPort %s:%d",
+                 address, me->dir_port);
+    control_event_server_status(LOG_NOTICE,
+                                "CHECKING_REACHABILITY DIRADDRESS=%s:%d",
+                                address, me->dir_port);
+  }
+  log_notice(LD_OR, "Now checking whether ORPort %s:%d%s %s reachable... "
+                         "(this may take up to %d minutes -- look for log "
+                         "messages indicating success)",
+      address, me->or_port,
+      me->dir_port ? dirbuf : "",
+      me->dir_port ? "are" : "is",
+      TIMEOUT_UNTIL_UNREACHABILITY_COMPLAINT/60);
+
+  tor_free(address);
+  return 1;
+}
+
 /** Annotate that we found our ORPort reachable. */
 void
 router_orport_found_reachable(void)
@@ -222,6 +259,10 @@ router_orport_found_reachable(void)
   const or_options_t *options = get_options();
   if (!can_reach_or_port && me) {
     char *address = tor_dup_ip(me->addr);
+
+    if (!address)
+      return;
+
     log_notice(LD_OR,"Self-testing indicates your ORPort is reachable from "
                "the outside. Excellent.%s",
                options->PublishServerDescriptor_ != NO_DIRINFO
@@ -249,6 +290,10 @@ router_dirport_found_reachable(void)
   const or_options_t *options = get_options();
   if (!can_reach_dir_port && me) {
     char *address = tor_dup_ip(me->addr);
+
+    if (!address)
+      return;
+
     log_notice(LD_DIRSERV,"Self-testing indicates your DirPort is reachable "
                "from the outside. Excellent.%s",
                options->PublishServerDescriptor_ != NO_DIRINFO
diff --git a/src/feature/relay/selftest.h b/src/feature/relay/selftest.h
index a80ec8936e..f5babc95da 100644
--- a/src/feature/relay/selftest.h
+++ b/src/feature/relay/selftest.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,13 +12,57 @@
 #ifndef TOR_SELFTEST_H
 #define TOR_SELFTEST_H
 
+#ifdef HAVE_MODULE_RELAY
+
 struct or_options_t;
 int check_whether_orport_reachable(const struct or_options_t *options);
 int check_whether_dirport_reachable(const struct or_options_t *options);
 
 void router_do_reachability_checks(int test_or, int test_dir);
+void router_perform_bandwidth_test(int num_circs, time_t now);
+int inform_testing_reachability(void);
+
 void router_orport_found_reachable(void);
 void router_dirport_found_reachable(void);
-void router_perform_bandwidth_test(int num_circs, time_t now);
 
-#endif
+void router_reset_reachability(void);
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+#define check_whether_orport_reachable(opts) \
+  ((void)(opts), 0)
+#define check_whether_dirport_reachable(opts) \
+  ((void)(opts), 0)
+
+static inline void
+router_do_reachability_checks(int test_or, int test_dir)
+{
+  (void)test_or;
+  (void)test_dir;
+  tor_assert_nonfatal_unreached();
+}
+static inline void
+router_perform_bandwidth_test(int num_circs, time_t now)
+{
+  (void)num_circs;
+  (void)now;
+  tor_assert_nonfatal_unreached();
+}
+static inline int
+inform_testing_reachability(void)
+{
+  tor_assert_nonfatal_unreached();
+  return 0;
+}
+
+#define router_orport_found_reachable() \
+  STMT_NIL
+#define router_dirport_found_reachable() \
+  STMT_NIL
+
+#define router_reset_reachability() \
+  STMT_NIL
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_SELFTEST_H) */
diff --git a/src/feature/relay/transport_config.c b/src/feature/relay/transport_config.c
new file mode 100644
index 0000000000..7dcce70e30
--- /dev/null
+++ b/src/feature/relay/transport_config.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file transport_config.c
+ * @brief Code to interpret the user's configuration of Tor's server
+ *        pluggable transports.
+ **/
+
+#include "orconfig.h"
+#define RELAY_TRANSPORT_CONFIG_PRIVATE
+#include "feature/relay/transport_config.h"
+
+#include "lib/encoding/confline.h"
+#include "lib/encoding/keyval.h"
+
+#include "lib/container/smartlist.h"
+
+/* Required for dirinfo_type_t in or_options_t */
+#include "core/or/or.h"
+#include "app/config/config.h"
+
+#include "feature/relay/ext_orport.h"
+#include "feature/relay/routermode.h"
+
+/* Copied from config.c, we will refactor later in 29211. */
+#define REJECT(arg) \
+  STMT_BEGIN *msg = tor_strdup(arg); return -1; STMT_END
+
+/** Given a ServerTransportListenAddr <b>line</b>, return its
+ *  <address:port> string. Return NULL if the line was not
+ *  well-formed.
+ *
+ *  If <b>transport</b> is set, return NULL if the line is not
+ *  referring to <b>transport</b>.
+ *
+ *  The returned string is allocated on the heap and it's the
+ *  responsibility of the caller to free it. */
+static char *
+get_bindaddr_from_transport_listen_line(const char *line,
+                                        const char *transport)
+{
+  smartlist_t *items = NULL;
+  const char *parsed_transport = NULL;
+  char *addrport = NULL;
+  tor_addr_t addr;
+  uint16_t port = 0;
+
+  items = smartlist_new();
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+
+  if (smartlist_len(items) < 2) {
+    log_warn(LD_CONFIG,"Too few arguments on ServerTransportListenAddr line.");
+    goto err;
+  }
+
+  parsed_transport = smartlist_get(items, 0);
+  addrport = tor_strdup(smartlist_get(items, 1));
+
+  /* If 'transport' is given, check if it matches the one on the line */
+  if (transport && strcmp(transport, parsed_transport))
+    goto err;
+
+  /* Validate addrport */
+  if (tor_addr_port_parse(LOG_WARN, addrport, &addr, &port, -1)<0) {
+    log_warn(LD_CONFIG, "Error parsing ServerTransportListenAddr "
+             "address '%s'", addrport);
+    goto err;
+  }
+
+  goto done;
+
+ err:
+  tor_free(addrport);
+  addrport = NULL;
+
+ done:
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+
+  return addrport;
+}
+
+/** Given the name of a pluggable transport in <b>transport</b>, check
+ *  the configuration file to see if the user has explicitly asked for
+ *  it to listen on a specific port. Return a <address:port> string if
+ *  so, otherwise NULL. */
+char *
+pt_get_bindaddr_from_config(const char *transport)
+{
+  config_line_t *cl;
+  const or_options_t *options = get_options();
+
+  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
+    char *bindaddr =
+      get_bindaddr_from_transport_listen_line(cl->value, transport);
+    if (bindaddr)
+      return bindaddr;
+  }
+
+  return NULL;
+}
+
+/** Given a ServerTransportOptions <b>line</b>, return a smartlist
+ *  with the options. Return NULL if the line was not well-formed.
+ *
+ *  If <b>transport</b> is set, return NULL if the line is not
+ *  referring to <b>transport</b>.
+ *
+ *  The returned smartlist and its strings are allocated on the heap
+ *  and it's the responsibility of the caller to free it. */
+STATIC smartlist_t *
+get_options_from_transport_options_line(const char *line,
+                                        const char *transport)
+{
+  smartlist_t *items = smartlist_new();
+  smartlist_t *pt_options = smartlist_new();
+  const char *parsed_transport = NULL;
+
+  smartlist_split_string(items, line, NULL,
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
+
+  if (smartlist_len(items) < 2) {
+    log_warn(LD_CONFIG,"Too few arguments on ServerTransportOptions line.");
+    goto err;
+  }
+
+  parsed_transport = smartlist_get(items, 0);
+  /* If 'transport' is given, check if it matches the one on the line */
+  if (transport && strcmp(transport, parsed_transport))
+    goto err;
+
+  SMARTLIST_FOREACH_BEGIN(items, const char *, option) {
+    if (option_sl_idx == 0) /* skip the transport field (first field)*/
+      continue;
+
+    /* validate that it's a k=v value */
+    if (!string_is_key_value(LOG_WARN, option)) {
+      log_warn(LD_CONFIG, "%s is not a k=v value.", escaped(option));
+      goto err;
+    }
+
+    /* add it to the options smartlist */
+    smartlist_add_strdup(pt_options, option);
+    log_debug(LD_CONFIG, "Added %s to the list of options", escaped(option));
+  } SMARTLIST_FOREACH_END(option);
+
+  goto done;
+
+ err:
+  SMARTLIST_FOREACH(pt_options, char*, s, tor_free(s));
+  smartlist_free(pt_options);
+  pt_options = NULL;
+
+ done:
+  SMARTLIST_FOREACH(items, char*, s, tor_free(s));
+  smartlist_free(items);
+
+  return pt_options;
+}
+
+/** Given the name of a pluggable transport in <b>transport</b>, check
+ *  the configuration file to see if the user has asked us to pass any
+ *  parameters to the pluggable transport. Return a smartlist
+ *  containing the parameters, otherwise NULL. */
+smartlist_t *
+pt_get_options_for_server_transport(const char *transport)
+{
+  config_line_t *cl;
+  const or_options_t *options = get_options();
+
+  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
+    smartlist_t *options_sl =
+      get_options_from_transport_options_line(cl->value, transport);
+    if (options_sl)
+      return options_sl;
+  }
+
+  return NULL;
+}
+
+/**
+ * Legacy validation/normalization function for the server transport options.
+ * Uses old_options as the previous options.
+ *
+ * Returns 0 on success, returns -1 and sets *msg to a newly allocated string
+ * on error.
+ */
+int
+options_validate_server_transport(const or_options_t *old_options,
+                                  or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  if (BUG(!options))
+    return -1;
+
+  if (BUG(!msg))
+    return -1;
+
+  config_line_t *cl;
+
+  if (options->ServerTransportPlugin && !server_mode(options)) {
+    log_notice(LD_GENERAL, "Tor is not configured as a relay but you specified"
+               " a ServerTransportPlugin line (%s). The ServerTransportPlugin "
+               "line will be ignored.",
+               escaped(options->ServerTransportPlugin->value));
+  }
+
+  if (options->ServerTransportListenAddr && !options->ServerTransportPlugin) {
+    log_notice(LD_GENERAL, "You need at least a single managed-proxy to "
+               "specify a transport listen address. The "
+               "ServerTransportListenAddr line will be ignored.");
+  }
+
+  for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
+    if (pt_parse_transport_line(options, cl->value, 1, 1) < 0)
+      REJECT("Invalid server transport line. See logs for details.");
+  }
+
+  for (cl = options->ServerTransportListenAddr; cl; cl = cl->next) {
+    /** If get_bindaddr_from_transport_listen_line() fails with
+        'transport' being NULL, it means that something went wrong
+        while parsing the ServerTransportListenAddr line. */
+    char *bindaddr = get_bindaddr_from_transport_listen_line(cl->value, NULL);
+    if (!bindaddr)
+      REJECT("ServerTransportListenAddr did not parse. See logs for details.");
+    tor_free(bindaddr);
+  }
+
+  for (cl = options->ServerTransportOptions; cl; cl = cl->next) {
+    /** If get_options_from_transport_options_line() fails with
+        'transport' being NULL, it means that something went wrong
+        while parsing the ServerTransportOptions line. */
+    smartlist_t *options_sl =
+      get_options_from_transport_options_line(cl->value, NULL);
+    if (!options_sl)
+      REJECT("ServerTransportOptions did not parse. See logs for details.");
+
+    SMARTLIST_FOREACH(options_sl, char *, cp, tor_free(cp));
+    smartlist_free(options_sl);
+  }
+
+  return 0;
+}
+
+/** Fetch the active option list, and take server pluggable transport actions
+ * based on it. All of the things we do should survive being done repeatedly.
+ * If present, <b>old_options</b> contains the previous value of the options.
+ *
+ * Return 0 if all goes well, return -1 if it's time to die.
+ *
+ * Note: We haven't moved all the "act on new configuration" logic
+ * into the options_act* functions yet.  Some is still in do_hup() and other
+ * places.
+ */
+int
+options_act_server_transport(const or_options_t *old_options)
+{
+  (void)old_options;
+
+  config_line_t *cl;
+  const or_options_t *options = get_options();
+  int running_tor = options->command == CMD_RUN_TOR;
+
+  /* If we are a bridge with a pluggable transport proxy but no
+     Extended ORPort, inform the user that they are missing out. */
+  if (options->ServerTransportPlugin &&
+      !options->ExtORPort_lines) {
+    log_notice(LD_CONFIG, "We use pluggable transports but the Extended "
+               "ORPort is disabled. Tor and your pluggable transports proxy "
+               "communicate with each other via the Extended ORPort so it "
+               "is suggested you enable it: it will also allow your Bridge "
+               "to collect statistics about its clients that use pluggable "
+               "transports. Please enable it using the ExtORPort torrc option "
+               "(e.g. set 'ExtORPort auto').");
+  }
+
+  /* If we have an ExtORPort, initialize its auth cookie. */
+  if (running_tor &&
+      init_ext_or_cookie_authentication(!!options->ExtORPort_lines) < 0) {
+    log_warn(LD_CONFIG,"Error creating Extended ORPort cookie file.");
+    return -1;
+  }
+
+  if (!options->DisableNetwork) {
+    if (options->ServerTransportPlugin) {
+      for (cl = options->ServerTransportPlugin; cl; cl = cl->next) {
+        if (pt_parse_transport_line(options, cl->value, 0, 1) < 0) {
+          // LCOV_EXCL_START
+          log_warn(LD_BUG,
+                   "Previously validated ServerTransportPlugin line "
+                   "could not be added!");
+          return -1;
+          // LCOV_EXCL_STOP
+        }
+      }
+    }
+  }
+
+  return 0;
+}
diff --git a/src/feature/relay/transport_config.h b/src/feature/relay/transport_config.h
new file mode 100644
index 0000000000..6d956d9af1
--- /dev/null
+++ b/src/feature/relay/transport_config.h
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file transport_config.h
+ * @brief Header for feature/relay/transport_config.c
+ **/
+
+#ifndef TOR_FEATURE_RELAY_TRANSPORT_CONFIG_H
+#define TOR_FEATURE_RELAY_TRANSPORT_CONFIG_H
+
+#ifdef HAVE_MODULE_RELAY
+
+#include "lib/testsupport/testsupport.h"
+
+struct or_options_t;
+struct smartlist_t;
+
+int options_validate_server_transport(const struct or_options_t *old_options,
+                                      struct or_options_t *options,
+                                      char **msg);
+
+char *pt_get_bindaddr_from_config(const char *transport);
+struct smartlist_t *pt_get_options_for_server_transport(const char *transport);
+
+int options_act_server_transport(const struct or_options_t *old_options);
+
+#ifdef RELAY_TRANSPORT_CONFIG_PRIVATE
+
+STATIC struct smartlist_t *get_options_from_transport_options_line(
+                      const char *line,
+                      const char *transport);
+
+#endif /* defined(RELAY_TRANSPORT_CONFIG_PRIVATE) */
+
+#else /* !defined(HAVE_MODULE_RELAY) */
+
+/** When tor is compiled with the relay module disabled, it can't be
+ * configured with server pluggable transports.
+ *
+ * Returns -1 and sets msg to a newly allocated string, if ExtORPort,
+ * ServerTransportPlugin, ServerTransportListenAddr, or
+ * ServerTransportOptions are set in options. Otherwise returns 0. */
+static inline int
+options_validate_server_transport(const struct or_options_t *old_options,
+                                  struct or_options_t *options,
+                                  char **msg)
+{
+  (void)old_options;
+
+  /* These ExtORPort checks are too strict, and will reject valid configs
+   * that disable ports, like "ExtORPort 0". */
+  if (options->ServerTransportPlugin ||
+      options->ServerTransportListenAddr ||
+      options->ServerTransportOptions ||
+      options->ExtORPort_lines) {
+    /* REJECT() this configuration */
+    *msg = tor_strdup("This tor was built with relay mode disabled. "
+                      "It can not be configured with an ExtORPort, "
+                      "a ServerTransportPlugin, a ServerTransportListenAddr, "
+                      "or ServerTransportOptions.");
+    return -1;
+  }
+
+  return 0;
+}
+
+#define pt_get_bindaddr_from_config(transport) \
+  (((void)(transport)),NULL)
+
+/* 31851: called from client/transports.c, but only from server code */
+#define pt_get_options_for_server_transport(transport) \
+  (((void)(transport)),NULL)
+
+#define options_validate_server_transport(old_options, options, msg) \
+  (((void)(old_options)),((void)(options)),((void)(msg)),0)
+#define options_act_server_transport(old_options) \
+  (((void)(old_options)),0)
+
+#endif /* defined(HAVE_MODULE_RELAY) */
+
+#endif /* !defined(TOR_FEATURE_RELAY_TRANSPORT_CONFIG_H) */
diff --git a/src/feature/rend/.may_include b/src/feature/rend/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/rend/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/rend/feature_rend.md b/src/feature/rend/feature_rend.md
new file mode 100644
index 0000000000..bfd8ae3dbc
--- /dev/null
+++ b/src/feature/rend/feature_rend.md
@@ -0,0 +1,7 @@
+@dir /feature/rend
+@brief feature/rend: version 2 (old) hidden services
+
+This directory implements the v2 onion service protocol,
+as specified in
+[rend-spec-v2.txt](https://gitweb.torproject.org/torspec.git/tree/rend-spec-v2.txt).
+
diff --git a/src/feature/rend/include.am b/src/feature/rend/include.am
new file mode 100644
index 0000000000..fb12439a90
--- /dev/null
+++ b/src/feature/rend/include.am
@@ -0,0 +1,22 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/rend/rendcache.c		\
+	src/feature/rend/rendclient.c		\
+	src/feature/rend/rendcommon.c		\
+	src/feature/rend/rendmid.c		\
+	src/feature/rend/rendparse.c		\
+	src/feature/rend/rendservice.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/rend/rend_authorized_client_st.h	\
+	src/feature/rend/rend_encoded_v2_service_descriptor_st.h	\
+	src/feature/rend/rend_intro_point_st.h		\
+	src/feature/rend/rend_service_descriptor_st.h	\
+	src/feature/rend/rendcache.h			\
+	src/feature/rend/rendclient.h			\
+	src/feature/rend/rendcommon.h			\
+	src/feature/rend/rendmid.h			\
+	src/feature/rend/rendparse.h			\
+	src/feature/rend/rendservice.h
diff --git a/src/feature/rend/rend_authorized_client_st.h b/src/feature/rend/rend_authorized_client_st.h
index 7bd4f2fe8c..c6a6676da9 100644
--- a/src/feature/rend/rend_authorized_client_st.h
+++ b/src/feature/rend/rend_authorized_client_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_authorized_client_st.h
+ * @brief Hidden-service authorized client structure.
+ **/
+
 #ifndef REND_AUTHORIZED_CLIENT_ST_H
 #define REND_AUTHORIZED_CLIENT_ST_H
 
@@ -14,5 +19,4 @@ struct rend_authorized_client_t {
   crypto_pk_t *client_key;
 };
 
-#endif
-
+#endif /* !defined(REND_AUTHORIZED_CLIENT_ST_H) */
diff --git a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
index 05ff145d53..fea91b876a 100644
--- a/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
+++ b/src/feature/rend/rend_encoded_v2_service_descriptor_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_encoded_v2_service_descriptor_st.h
+ * @brief Encoded v2 HS descriptor structure.
+ **/
+
 #ifndef REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
 #define REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H
 
@@ -13,5 +18,4 @@ struct rend_encoded_v2_service_descriptor_t {
   char *desc_str; /**< Descriptor string. */
 };
 
-#endif
-
+#endif /* !defined(REND_ENCODED_V2_SERVICE_DESCRIPTOR_ST_H) */
diff --git a/src/feature/rend/rend_intro_point_st.h b/src/feature/rend/rend_intro_point_st.h
index de6987e569..4f0aa01523 100644
--- a/src/feature/rend/rend_intro_point_st.h
+++ b/src/feature/rend/rend_intro_point_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_intro_point_st.h
+ * @brief v2 hidden service introduction point structure.
+ **/
+
 #ifndef REND_INTRO_POINT_ST_H
 #define REND_INTRO_POINT_ST_H
 
@@ -73,4 +78,4 @@ struct rend_intro_point_t {
   unsigned int circuit_established:1;
 };
 
-#endif
+#endif /* !defined(REND_INTRO_POINT_ST_H) */
diff --git a/src/feature/rend/rend_service_descriptor_st.h b/src/feature/rend/rend_service_descriptor_st.h
index aeb3178064..80c8034f46 100644
--- a/src/feature/rend/rend_service_descriptor_st.h
+++ b/src/feature/rend/rend_service_descriptor_st.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file rend_service_descriptor_st.h
+ * @brief Parsed v2 HS descriptor structure.
+ **/
+
 #ifndef REND_SERVICE_DESCRIPTOR_ST_H
 #define REND_SERVICE_DESCRIPTOR_ST_H
 
@@ -30,5 +35,4 @@ struct rend_service_descriptor_t {
   smartlist_t *successful_uploads;
 };
 
-#endif
-
+#endif /* !defined(REND_SERVICE_DESCRIPTOR_ST_H) */
diff --git a/src/feature/rend/rendcache.c b/src/feature/rend/rendcache.c
index 1c3badaff3..0890a81d8f 100644
--- a/src/feature/rend/rendcache.c
+++ b/src/feature/rend/rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,8 @@
 #include "feature/rend/rend_intro_point_st.h"
 #include "feature/rend/rend_service_descriptor_st.h"
 
+#include "lib/ctime/di_ops.h"
+
 /** Map from service id (as generated by rend_get_service_id) to
  * rend_cache_entry_t. */
 STATIC strmap_t *rend_cache = NULL;
@@ -45,7 +47,7 @@ STATIC digestmap_t *rend_cache_v2_dir = NULL;
  * looked up in this cache and if present, it is discarded from the fetched
  * descriptor. At the end, all IP(s) in the cache, for a specific service
  * ID, that were NOT present in the descriptor are removed from this cache.
- * Which means that if at least one IP was not in this cache, thus usuable,
+ * Which means that if at least one IP was not in this cache, thus usable,
  * it's considered a new descriptor so we keep it. Else, if all IPs were in
  * this cache, we discard the descriptor as it's considered unusable.
  *
@@ -226,6 +228,17 @@ rend_cache_entry_free_void(void *p)
   rend_cache_entry_free_(p);
 }
 
+/** Check if a failure cache entry exists for the given intro point. */
+bool
+rend_cache_intro_failure_exists(const char *service_id,
+                                const uint8_t *intro_identity)
+{
+  tor_assert(service_id);
+  tor_assert(intro_identity);
+
+  return cache_failure_intro_lookup(intro_identity, service_id, NULL);
+}
+
 /** Free all storage held by the service descriptor cache. */
 void
 rend_cache_free_all(void)
@@ -513,9 +526,16 @@ rend_cache_lookup_entry(const char *query, int version, rend_cache_entry_t **e)
   rend_cache_entry_t *entry = NULL;
   static const int default_version = 2;
 
-  tor_assert(rend_cache);
   tor_assert(query);
 
+  /* This is possible if we are in the shutdown process and the cache was
+   * freed while some other subsystem might do a lookup to the cache for
+   * cleanup reasons such HS circuit cleanup for instance. */
+  if (!rend_cache) {
+    ret = -ENOENT;
+    goto end;
+  }
+
   if (!rend_valid_v2_service_id(query)) {
     ret = -EINVAL;
     goto end;
@@ -593,10 +613,10 @@ rend_cache_lookup_v2_desc_as_dir(const char *desc_id, const char **desc)
   char desc_id_digest[DIGEST_LEN];
   tor_assert(rend_cache_v2_dir);
   if (base32_decode(desc_id_digest, DIGEST_LEN,
-                    desc_id, REND_DESC_ID_V2_LEN_BASE32) < 0) {
+                    desc_id, REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
     log_fn(LOG_PROTOCOL_WARN, LD_REND,
            "Rejecting v2 rendezvous descriptor request -- descriptor ID "
-           "contains illegal characters: %s",
+           "has wrong length or illegal characters: %s",
            safe_str(desc_id));
     return -1;
   }
@@ -854,7 +874,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
     *entry = NULL;
   }
   if (base32_decode(want_desc_id, sizeof(want_desc_id),
-                    desc_id_base32, strlen(desc_id_base32)) != 0) {
+                    desc_id_base32, strlen(desc_id_base32)) !=
+      sizeof(want_desc_id)) {
     log_warn(LD_BUG, "Couldn't decode base32 %s for descriptor id.",
              escaped_safe_str_client(desc_id_base32));
     goto err;
@@ -888,8 +909,8 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   if (intro_content && intro_size > 0) {
     int n_intro_points;
     if (rend_data->auth_type != REND_NO_AUTH &&
-        !tor_mem_is_zero(rend_data->descriptor_cookie,
-                         sizeof(rend_data->descriptor_cookie))) {
+        !safe_mem_is_zero(rend_data->descriptor_cookie,
+                          sizeof(rend_data->descriptor_cookie))) {
       char *ipos_decrypted = NULL;
       size_t ipos_decrypted_size;
       if (rend_decrypt_introduction_points(&ipos_decrypted,
@@ -1005,4 +1026,3 @@ rend_cache_store_v2_desc_as_client(const char *desc,
   tor_free(intro_content);
   return retval;
 }
-
diff --git a/src/feature/rend/rendcache.h b/src/feature/rend/rendcache.h
index aec97eabb8..45410610b4 100644
--- a/src/feature/rend/rendcache.h
+++ b/src/feature/rend/rendcache.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -80,6 +80,8 @@ int rend_cache_store_v2_desc_as_client(const char *desc,
                                        rend_cache_entry_t **entry);
 size_t rend_cache_get_total_allocation(void);
 
+bool rend_cache_intro_failure_exists(const char *service_id,
+                                     const uint8_t *intro_identity);
 void rend_cache_intro_failure_note(rend_intro_point_failure_t failure,
                                    const uint8_t *identity,
                                    const char *service_id);
diff --git a/src/feature/rend/rendclient.c b/src/feature/rend/rendclient.c
index 6e95142c0b..427491e3a8 100644
--- a/src/feature/rend/rendclient.c
+++ b/src/feature/rend/rendclient.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,7 +17,7 @@
 #include "core/or/connection_edge.h"
 #include "core/or/relay.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_circuit.h"
@@ -119,7 +119,7 @@ rend_client_send_introduction(origin_circuit_t *introcirc,
   char tmp[RELAY_PAYLOAD_SIZE];
   rend_cache_entry_t *entry = NULL;
   crypt_path_t *cpath;
-  off_t dh_offset;
+  ptrdiff_t dh_offset;
   crypto_pk_t *intro_key = NULL;
   int status = 0;
   const char *onion_address;
@@ -403,14 +403,23 @@ rend_client_introduction_acked(origin_circuit_t *circ,
     } else {
       log_info(LD_REND,"...Found no rend circ. Dropping on the floor.");
     }
+    /* Save the rend data digest to a temporary object so that we don't access
+     * it after we mark the circuit for close. */
+    const uint8_t *rend_digest_tmp = NULL;
+    size_t digest_len;
+    uint8_t *cached_rend_digest = NULL;
+    rend_digest_tmp = rend_data_get_pk_digest(circ->rend_data, &digest_len);
+    cached_rend_digest = tor_malloc_zero(digest_len);
+    memcpy(cached_rend_digest, rend_digest_tmp, digest_len);
+
     /* close the circuit: we won't need it anymore. */
     circuit_change_purpose(TO_CIRCUIT(circ),
                            CIRCUIT_PURPOSE_C_INTRODUCE_ACKED);
     circuit_mark_for_close(TO_CIRCUIT(circ), END_CIRC_REASON_FINISHED);
 
     /* close any other intros launched in parallel */
-    rend_client_close_other_intros(rend_data_get_pk_digest(circ->rend_data,
-                                                           NULL));
+    rend_client_close_other_intros(cached_rend_digest);
+    tor_free(cached_rend_digest); /* free the temporary digest */
   } else {
     /* It's a NAK; the introduction point didn't relay our request. */
     circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_C_INTRODUCING);
@@ -469,16 +478,19 @@ directory_get_from_hs_dir(const char *desc_id,
 
   /* Automatically pick an hs dir if none given. */
   if (!rs_hsdir) {
+    bool rate_limited = false;
+
     /* Determine responsible dirs. Even if we can't get all we want, work with
      * the ones we have. If it's empty, we'll notice in hs_pick_hsdir(). */
     smartlist_t *responsible_dirs = smartlist_new();
     hid_serv_get_responsible_directories(responsible_dirs, desc_id);
 
-    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32);
+    hs_dir = hs_pick_hsdir(responsible_dirs, desc_id_base32, &rate_limited);
     if (!hs_dir) {
       /* No suitable hs dir can be found, stop right now. */
-      control_event_hsv2_descriptor_failed(rend_query, NULL,
-                                           "QUERY_NO_HSDIR");
+      const char *query_response = (rate_limited) ? "QUERY_RATE_LIMITED" :
+                                                    "QUERY_NO_HSDIR";
+      control_event_hsv2_descriptor_failed(rend_query, NULL, query_response);
       control_event_hs_descriptor_content(rend_data_get_address(rend_query),
                                           desc_id_base32, NULL, NULL);
       return 0;
@@ -1036,18 +1048,30 @@ rend_client_get_random_intro_impl(const rend_cache_entry_t *entry,
   const or_options_t *options = get_options();
   smartlist_t *usable_nodes;
   int n_excluded = 0;
+  char service_id[REND_SERVICE_ID_LEN_BASE32 + 1];
 
   /* We'll keep a separate list of the usable nodes.  If this becomes empty,
    * no nodes are usable.  */
   usable_nodes = smartlist_new();
   smartlist_add_all(usable_nodes, entry->parsed->intro_nodes);
 
+  /* Get service ID so we can use it to query the failure cache. If we fail to
+   * parse it, this cache entry is no good. */
+  if (BUG(rend_get_service_id(entry->parsed->pk, service_id) < 0)) {
+    smartlist_free(usable_nodes);
+    return NULL;
+  }
+
   /* Remove the intro points that have timed out during this HS
    * connection attempt from our list of usable nodes. */
-  SMARTLIST_FOREACH(usable_nodes, rend_intro_point_t *, ip,
-                    if (ip->timed_out) {
-                      SMARTLIST_DEL_CURRENT(usable_nodes, ip);
-                    });
+  SMARTLIST_FOREACH_BEGIN(usable_nodes, const rend_intro_point_t *, ip) {
+    bool failed_intro =
+      rend_cache_intro_failure_exists(service_id,
+                       (const uint8_t *) ip->extend_info->identity_digest);
+    if (ip->timed_out || failed_intro) {
+      SMARTLIST_DEL_CURRENT(usable_nodes, ip);
+    };
+  } SMARTLIST_FOREACH_END(ip);
 
  again:
   if (smartlist_len(usable_nodes) == 0) {
@@ -1226,3 +1250,66 @@ rend_parse_service_authorization(const or_options_t *options,
   }
   return res;
 }
+
+/** The given circuit is being freed. Take appropriate action if it is of
+ * interest to the client subsystem. */
+void
+rend_client_circuit_cleanup_on_free(const circuit_t *circ)
+{
+  int reason, orig_reason;
+  bool has_timed_out, ip_is_redundant;
+  const origin_circuit_t *ocirc = NULL;
+
+  tor_assert(circ);
+  tor_assert(CIRCUIT_IS_ORIGIN(circ));
+
+  reason = circ->marked_for_close_reason;
+  orig_reason = circ->marked_for_close_orig_reason;
+  ocirc = CONST_TO_ORIGIN_CIRCUIT(circ);
+  tor_assert(ocirc->rend_data);
+
+  has_timed_out = (reason == END_CIRC_REASON_TIMEOUT);
+  ip_is_redundant = (orig_reason == END_CIRC_REASON_IP_NOW_REDUNDANT);
+
+  switch (circ->purpose) {
+  case CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT:
+  {
+    if (ip_is_redundant) {
+      break;
+    }
+    tor_assert(circ->state == CIRCUIT_STATE_OPEN);
+    tor_assert(ocirc->build_state->chosen_exit);
+    /* Treat this like getting a nack from it */
+    log_info(LD_REND, "Failed intro circ %s to %s (awaiting ack). %s",
+        safe_str_client(rend_data_get_address(ocirc->rend_data)),
+        safe_str_client(build_state_get_exit_nickname(ocirc->build_state)),
+        has_timed_out ? "Recording timeout." : "Removing from descriptor.");
+    rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
+                                           ocirc->rend_data,
+                                           has_timed_out ?
+                                           INTRO_POINT_FAILURE_TIMEOUT :
+                                           INTRO_POINT_FAILURE_GENERIC);
+    break;
+  }
+  case CIRCUIT_PURPOSE_C_INTRODUCING:
+  {
+    /* Ignore if we were introducing and it timed out, we didn't pick an exit
+     * point yet (IP) or the reason indicate that it was a redundant IP. */
+    if (has_timed_out || !ocirc->build_state->chosen_exit || ip_is_redundant) {
+      break;
+    }
+    log_info(LD_REND, "Failed intro circ %s to %s "
+             "(building circuit to intro point). "
+             "Marking intro point as possibly unreachable.",
+             safe_str_client(rend_data_get_address(ocirc->rend_data)),
+             safe_str_client(build_state_get_exit_nickname(
+                                                  ocirc->build_state)));
+    rend_client_report_intro_point_failure(ocirc->build_state->chosen_exit,
+                                           ocirc->rend_data,
+                                           INTRO_POINT_FAILURE_UNREACHABLE);
+    break;
+  }
+  default:
+    break;
+  }
+}
diff --git a/src/feature/rend/rendclient.h b/src/feature/rend/rendclient.h
index e5f333238e..b7aa212487 100644
--- a/src/feature/rend/rendclient.h
+++ b/src/feature/rend/rendclient.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #ifndef TOR_RENDCLIENT_H
 #define TOR_RENDCLIENT_H
 
+#include "feature/hs/hs_circuit.h"
 #include "feature/rend/rendcache.h"
 
 void rend_client_purge_state(void);
@@ -47,5 +48,7 @@ rend_service_authorization_t *rend_client_lookup_service_authorization(
                                                 const char *onion_address);
 void rend_service_authorization_free_all(void);
 
+void rend_client_circuit_cleanup_on_free(const circuit_t *circ);
+
 #endif /* !defined(TOR_RENDCLIENT_H) */
 
diff --git a/src/feature/rend/rendcommon.c b/src/feature/rend/rendcommon.c
index de48af795f..5d04755819 100644
--- a/src/feature/rend/rendcommon.c
+++ b/src/feature/rend/rendcommon.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "feature/hs/hs_client.h"
@@ -171,9 +171,10 @@ rend_compute_v2_desc_id(char *desc_id_out, const char *service_id,
   }
   /* Convert service ID to binary. */
   if (base32_decode(service_id_binary, REND_SERVICE_ID_LEN,
-                    service_id, REND_SERVICE_ID_LEN_BASE32) < 0) {
+                    service_id, REND_SERVICE_ID_LEN_BASE32) !=
+      REND_SERVICE_ID_LEN) {
     log_warn(LD_REND, "Could not compute v2 descriptor ID: "
-                      "Illegal characters in service ID: %s",
+                      "Illegal characters or wrong length for service ID: %s",
              safe_str_client(service_id));
     return -1;
   }
@@ -785,39 +786,39 @@ rend_process_relay_cell(circuit_t *circ, const crypt_path_t *layer_hint,
   switch (command) {
     case RELAY_COMMAND_ESTABLISH_INTRO:
       if (or_circ)
-        r = hs_intro_received_establish_intro(or_circ,payload,length);
+        r = hs_intro_received_establish_intro(or_circ, payload, length);
       break;
     case RELAY_COMMAND_ESTABLISH_RENDEZVOUS:
       if (or_circ)
-        r = rend_mid_establish_rendezvous(or_circ,payload,length);
+        r = rend_mid_establish_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE1:
       if (or_circ)
-        r = hs_intro_received_introduce1(or_circ,payload,length);
+        r = hs_intro_received_introduce1(or_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE2:
       if (origin_circ)
-        r = hs_service_receive_introduce2(origin_circ,payload,length);
+        r = hs_service_receive_introduce2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRODUCE_ACK:
       if (origin_circ)
-        r = hs_client_receive_introduce_ack(origin_circ,payload,length);
+        r = hs_client_receive_introduce_ack(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS1:
       if (or_circ)
-        r = rend_mid_rendezvous(or_circ,payload,length);
+        r = rend_mid_rendezvous(or_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS2:
       if (origin_circ)
-        r = hs_client_receive_rendezvous2(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous2(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_INTRO_ESTABLISHED:
       if (origin_circ)
-        r = hs_service_receive_intro_established(origin_circ,payload,length);
+        r = hs_service_receive_intro_established(origin_circ, payload, length);
       break;
     case RELAY_COMMAND_RENDEZVOUS_ESTABLISHED:
       if (origin_circ)
-        r = hs_client_receive_rendezvous_acked(origin_circ,payload,length);
+        r = hs_client_receive_rendezvous_acked(origin_circ, payload, length);
       break;
     default:
       tor_fragile_assert();
diff --git a/src/feature/rend/rendcommon.h b/src/feature/rend/rendcommon.h
index f136863c7a..d8281e0578 100644
--- a/src/feature/rend/rendcommon.h
+++ b/src/feature/rend/rendcommon.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/rend/rendmid.c b/src/feature/rend/rendmid.c
index af02b34e6b..b497362857 100644
--- a/src/feature/rend/rendmid.c
+++ b/src/feature/rend/rendmid.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,7 @@
 #include "feature/rend/rendmid.h"
 #include "feature/stats/rephist.h"
 #include "feature/hs/hs_circuitmap.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 
 #include "core/or/or_circuit_st.h"
@@ -58,7 +59,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
   pk = crypto_pk_asn1_decode((char*)(request+2), asn1len);
   if (!pk) {
     reason = END_CIRC_REASON_TORPROTOCOL;
-    log_warn(LD_PROTOCOL, "Couldn't decode public key.");
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL, "Couldn't decode public key.");
     goto err;
   }
 
@@ -80,7 +81,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
                                        (char*)request, 2+asn1len+DIGEST_LEN,
                                        (char*)(request+2+DIGEST_LEN+asn1len),
                                        request_len-(2+DIGEST_LEN+asn1len))<0) {
-    log_warn(LD_PROTOCOL,
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
              "Incorrect signature on ESTABLISH_INTRO cell; rejecting.");
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
@@ -117,6 +118,7 @@ rend_mid_establish_intro_legacy(or_circuit_t *circ, const uint8_t *request,
   /* Now, set up this circuit. */
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_INTRO_POINT);
   hs_circuitmap_register_intro_circ_v2_relay_side(circ, (uint8_t *)pk_digest);
+  hs_dos_setup_default_intro2_defenses(circ);
 
   log_info(LD_REND,
            "Established introduction point on circuit %u for service %s",
@@ -160,9 +162,9 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
   if (request_len < (DIGEST_LEN+(MAX_NICKNAME_LEN+1)+REND_COOKIE_LEN+
                      DH1024_KEY_LEN+CIPHER_KEY_LEN+
                      PKCS1_OAEP_PADDING_OVERHEAD)) {
-    log_warn(LD_PROTOCOL, "Impossibly short INTRODUCE1 cell on circuit %u; "
-             "responding with nack.",
-             (unsigned)circ->p_circ_id);
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Impossibly short INTRODUCE1 cell on circuit %u; "
+           "responding with nack.", (unsigned)circ->p_circ_id);
     goto err;
   }
 
@@ -181,6 +183,14 @@ rend_mid_introduce_legacy(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
+  /* Before sending, lets make sure this cell can be sent on the service
+   * circuit asking the DoS defenses. */
+  if (!hs_dos_can_send_intro2(intro_circ)) {
+    log_info(LD_PROTOCOL, "Can't relay INTRODUCE1 v2 cell due to DoS "
+                          "limitations. Sending NACK to client.");
+    goto err;
+  }
+
   log_info(LD_REND,
            "Sending introduction request for service %s "
            "from circ %u to circ %u",
@@ -237,8 +247,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
     goto err;
   }
 
-  /* Check if we are configured to accept established rendezvous cells from
-   * client or in other words Tor2Web clients. */
+  /* Check if we are configured to defend ourselves from clients that
+   * attempt to establish rendezvous points directly to us. */
   if (channel_is_client(circ->p_chan) &&
       dos_should_refuse_single_hop_client()) {
     /* Note it down for the heartbeat log purposes. */
@@ -248,7 +258,7 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
   }
 
   if (circ->base_.n_chan) {
-    log_warn(LD_PROTOCOL,
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
              "Tried to establish rendezvous on non-edge circuit");
     goto err;
   }
@@ -260,8 +270,8 @@ rend_mid_establish_rendezvous(or_circuit_t *circ, const uint8_t *request,
   }
 
   if (hs_circuitmap_get_rend_circ_relay_side(request)) {
-    log_warn(LD_PROTOCOL,
-             "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Duplicate rendezvous cookie in ESTABLISH_RENDEZVOUS.");
     goto err;
   }
 
@@ -303,9 +313,9 @@ rend_mid_rendezvous(or_circuit_t *circ, const uint8_t *request,
   int reason = END_CIRC_REASON_INTERNAL;
 
   if (circ->base_.purpose != CIRCUIT_PURPOSE_OR || circ->base_.n_chan) {
-    log_info(LD_REND,
-             "Tried to complete rendezvous on non-OR or non-edge circuit %u.",
-             (unsigned)circ->p_circ_id);
+    log_fn(LOG_PROTOCOL_WARN, LD_PROTOCOL,
+           "Tried to complete rendezvous on non-OR or non-edge circuit %u.",
+           (unsigned)circ->p_circ_id);
     reason = END_CIRC_REASON_TORPROTOCOL;
     goto err;
   }
diff --git a/src/feature/rend/rendmid.h b/src/feature/rend/rendmid.h
index 8ae1fa16b8..789596d855 100644
--- a/src/feature/rend/rendmid.h
+++ b/src/feature/rend/rendmid.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/rend/rendparse.c b/src/feature/rend/rendparse.c
index abd0feb448..0979d767a7 100644
--- a/src/feature/rend/rendparse.c
+++ b/src/feature/rend/rendparse.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -143,8 +143,9 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     goto err;
   }
   if (base32_decode(desc_id_out, DIGEST_LEN,
-                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) < 0) {
-    log_warn(LD_REND, "Descriptor ID contains illegal characters: %s",
+                    tok->args[0], REND_DESC_ID_V2_LEN_BASE32) != DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Descriptor ID has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -174,8 +175,10 @@ rend_parse_v2_service_descriptor(rend_service_descriptor_t **parsed_out,
     log_warn(LD_REND, "Invalid secret ID part: '%s'", tok->args[0]);
     goto err;
   }
-  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) < 0) {
-    log_warn(LD_REND, "Secret ID part contains illegal characters: %s",
+  if (base32_decode(secret_id_part, DIGEST_LEN, tok->args[0], 32) !=
+      DIGEST_LEN) {
+    log_warn(LD_REND,
+             "Secret ID part has wrong length or illegal characters: %s",
              tok->args[0]);
     goto err;
   }
@@ -429,8 +432,10 @@ rend_parse_introduction_points(rend_service_descriptor_t *parsed,
     /* Parse identifier. */
     tok = find_by_keyword(tokens, R_IPO_IDENTIFIER);
     if (base32_decode(info->identity_digest, DIGEST_LEN,
-                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) < 0) {
-      log_warn(LD_REND, "Identity digest contains illegal characters: %s",
+                      tok->args[0], REND_INTRO_POINT_ID_LEN_BASE32) !=
+        DIGEST_LEN) {
+      log_warn(LD_REND,
+               "Identity digest has wrong length or illegal characters: %s",
                tok->args[0]);
       rend_intro_point_free(intro);
       goto err;
diff --git a/src/feature/rend/rendparse.h b/src/feature/rend/rendparse.h
index 0cef931e90..75109c204d 100644
--- a/src/feature/rend/rendparse.h
+++ b/src/feature/rend/rendparse.h
@@ -1,12 +1,12 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file rend_parse.h
- * \brief Header file for rend_parse.c.
+ * \file rendparse.h
+ * \brief Header file for rendparse.c.
  **/
 
 #ifndef TOR_REND_PARSE_H
@@ -29,4 +29,4 @@ int rend_parse_introduction_points(rend_service_descriptor_t *parsed,
                                    size_t intro_points_encoded_size);
 int rend_parse_client_keys(strmap_t *parsed_clients, const char *str);
 
-#endif
+#endif /* !defined(TOR_REND_PARSE_H) */
diff --git a/src/feature/rend/rendservice.c b/src/feature/rend/rendservice.c
index eaf0432a7d..83388a72eb 100644
--- a/src/feature/rend/rendservice.c
+++ b/src/feature/rend/rendservice.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,8 +18,9 @@
 #include "core/or/circuituse.h"
 #include "core/or/policies.h"
 #include "core/or/relay.h"
+#include "core/or/crypt_path.h"
 #include "feature/client/circpathbias.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/dirclient/dirclient.h"
 #include "feature/dircommon/directory.h"
 #include "feature/hs/hs_common.h"
@@ -48,6 +49,7 @@
 #include "core/or/crypt_path_reference_st.h"
 #include "core/or/edge_connection_st.h"
 #include "core/or/extend_info_st.h"
+#include "feature/hs/hs_opts_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "feature/rend/rend_authorized_client_st.h"
@@ -729,22 +731,20 @@ service_config_shadow_copy(rend_service_t *service,
   config->ports = NULL;
 }
 
-/* Parse the hidden service configuration starting at <b>line_</b> using the
+/* Parse the hidden service configuration from <b>hs_opts</b> using the
  * already configured generic service configuration in <b>config</b>. This
  * function will translate the config object to a rend_service_t and add it to
  * the temporary list if valid. If <b>validate_only</b> is set, parse, warn
  * and return as normal but don't actually add the service to the list. */
 int
-rend_config_service(const config_line_t *line_,
+rend_config_service(const hs_opts_t *hs_opts,
                     const or_options_t *options,
                     hs_service_config_t *config)
 {
-  const config_line_t *line;
   rend_service_t *service = NULL;
 
-  /* line_ can be NULL which would mean that the service configuration only
-   * have one line that is the directory directive. */
   tor_assert(options);
+  tor_assert(hs_opts);
   tor_assert(config);
 
   /* We are about to configure a version 2 service. Warn of deprecation. */
@@ -764,126 +764,109 @@ rend_config_service(const config_line_t *line_,
    * options, we'll copy over the useful data to the rend_service_t object. */
   service_config_shadow_copy(service, config);
 
-  for (line = line_; line; line = line->next) {
-    if (!strcasecmp(line->key, "HiddenServiceDir")) {
-      /* We just hit the next hidden service, stop right now. */
-      break;
+  /* Number of introduction points. */
+  if (hs_opts->HiddenServiceNumIntroductionPoints > NUM_INTRO_POINTS_MAX) {
+    log_warn(LD_CONFIG, "HiddenServiceNumIntroductionPoints must be "
+             "between 0 and %d, not %d.",
+             NUM_INTRO_POINTS_MAX,
+             hs_opts->HiddenServiceNumIntroductionPoints);
+    goto err;
+  }
+  service->n_intro_points_wanted = hs_opts->HiddenServiceNumIntroductionPoints;
+  log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
+           service->n_intro_points_wanted, escaped(service->directory));
+
+  /* Client authorization */
+  if (hs_opts->HiddenServiceAuthorizeClient) {
+    /* Parse auth type and comma-separated list of client names and add a
+     * rend_authorized_client_t for each client to the service's list
+     * of authorized clients. */
+    smartlist_t *type_names_split, *clients;
+    const char *authname;
+    type_names_split = smartlist_new();
+    smartlist_split_string(type_names_split,
+                           hs_opts->HiddenServiceAuthorizeClient, " ", 0, 2);
+    if (smartlist_len(type_names_split) < 1) {
+      log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This "
+               "should have been prevented when parsing the "
+               "configuration.");
+      smartlist_free(type_names_split);
+      goto err;
     }
-    /* Number of introduction points. */
-    if (!strcasecmp(line->key, "HiddenServiceNumIntroductionPoints")) {
-      int ok = 0;
-      /* Those are specific defaults for version 2. */
-      service->n_intro_points_wanted =
-        (unsigned int) tor_parse_long(line->value, 10,
-                                      0, NUM_INTRO_POINTS_MAX, &ok, NULL);
-      if (!ok) {
-        log_warn(LD_CONFIG,
-                 "HiddenServiceNumIntroductionPoints "
-                 "should be between %d and %d, not %s",
-                 0, NUM_INTRO_POINTS_MAX, line->value);
-        goto err;
-      }
-      log_info(LD_CONFIG, "HiddenServiceNumIntroductionPoints=%d for %s",
-               service->n_intro_points_wanted, escaped(service->directory));
-      continue;
+    authname = smartlist_get(type_names_split, 0);
+    if (!strcasecmp(authname, "basic")) {
+      service->auth_type = REND_BASIC_AUTH;
+    } else if (!strcasecmp(authname, "stealth")) {
+      service->auth_type = REND_STEALTH_AUTH;
+    } else {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
+               "unrecognized auth-type '%s'. Only 'basic' or 'stealth' "
+               "are recognized.",
+               (char *) smartlist_get(type_names_split, 0));
+      SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+      smartlist_free(type_names_split);
+      goto err;
     }
-    if (!strcasecmp(line->key, "HiddenServiceAuthorizeClient")) {
-      /* Parse auth type and comma-separated list of client names and add a
-       * rend_authorized_client_t for each client to the service's list
-       * of authorized clients. */
-      smartlist_t *type_names_split, *clients;
-      const char *authname;
-      if (service->auth_type != REND_NO_AUTH) {
-        log_warn(LD_CONFIG, "Got multiple HiddenServiceAuthorizeClient "
-                 "lines for a single service.");
-        goto err;
-      }
-      type_names_split = smartlist_new();
-      smartlist_split_string(type_names_split, line->value, " ", 0, 2);
-      if (smartlist_len(type_names_split) < 1) {
-        log_warn(LD_BUG, "HiddenServiceAuthorizeClient has no value. This "
-                         "should have been prevented when parsing the "
-                         "configuration.");
-        smartlist_free(type_names_split);
-        goto err;
-      }
-      authname = smartlist_get(type_names_split, 0);
-      if (!strcasecmp(authname, "basic")) {
-        service->auth_type = REND_BASIC_AUTH;
-      } else if (!strcasecmp(authname, "stealth")) {
-        service->auth_type = REND_STEALTH_AUTH;
-      } else {
-        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
-                 "unrecognized auth-type '%s'. Only 'basic' or 'stealth' "
-                 "are recognized.",
-                 (char *) smartlist_get(type_names_split, 0));
-        SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
-        smartlist_free(type_names_split);
-        goto err;
-      }
-      service->clients = smartlist_new();
-      if (smartlist_len(type_names_split) < 2) {
-        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
-                            "auth-type '%s', but no client names.",
-                 service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
-        SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
-        smartlist_free(type_names_split);
-        continue;
-      }
-      clients = smartlist_new();
-      smartlist_split_string(clients, smartlist_get(type_names_split, 1),
-                             ",", SPLIT_SKIP_SPACE, 0);
+    service->clients = smartlist_new();
+    if (smartlist_len(type_names_split) < 2) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains "
+               "auth-type '%s', but no client names.",
+               service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
       SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
       smartlist_free(type_names_split);
-      /* Remove duplicate client names. */
-      {
-        int num_clients = smartlist_len(clients);
-        smartlist_sort_strings(clients);
-        smartlist_uniq_strings(clients);
-        if (smartlist_len(clients) < num_clients) {
-          log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
-                   "duplicate client name(s); removing.",
-                   num_clients - smartlist_len(clients));
-        }
-      }
-      SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name)
-      {
-        rend_authorized_client_t *client;
-        if (!rend_valid_client_name(client_name)) {
-          log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an "
-                              "illegal client name: '%s'. Names must be "
-                              "between 1 and %d characters and contain "
-                              "only [A-Za-z0-9+_-].",
-                   client_name, REND_CLIENTNAME_MAX_LEN);
-          SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
-          smartlist_free(clients);
-          goto err;
-        }
-        client = tor_malloc_zero(sizeof(rend_authorized_client_t));
-        client->client_name = tor_strdup(client_name);
-        smartlist_add(service->clients, client);
-        log_debug(LD_REND, "Adding client name '%s'", client_name);
+      goto err;
+    }
+    clients = smartlist_new();
+    smartlist_split_string(clients, smartlist_get(type_names_split, 1),
+                             ",", SPLIT_SKIP_SPACE, 0);
+    SMARTLIST_FOREACH(type_names_split, char *, cp, tor_free(cp));
+    smartlist_free(type_names_split);
+    /* Remove duplicate client names. */
+    {
+      int num_clients = smartlist_len(clients);
+      smartlist_sort_strings(clients);
+      smartlist_uniq_strings(clients);
+      if (smartlist_len(clients) < num_clients) {
+        log_info(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                 "duplicate client name(s); removing.",
+                 num_clients - smartlist_len(clients));
       }
-      SMARTLIST_FOREACH_END(client_name);
-      SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
-      smartlist_free(clients);
-      /* Ensure maximum number of clients. */
-      if ((service->auth_type == REND_BASIC_AUTH &&
-            smartlist_len(service->clients) > 512) ||
-          (service->auth_type == REND_STEALTH_AUTH &&
-            smartlist_len(service->clients) > 16)) {
-        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
-                            "client authorization entries, but only a "
-                            "maximum of %d entries is allowed for "
-                            "authorization type '%s'.",
-                 smartlist_len(service->clients),
-                 service->auth_type == REND_BASIC_AUTH ? 512 : 16,
-                 service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
+    }
+    SMARTLIST_FOREACH_BEGIN(clients, const char *, client_name) {
+      rend_authorized_client_t *client;
+      if (!rend_valid_client_name(client_name)) {
+        log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains an "
+                            "illegal client name: '%s'. Names must be "
+                            "between 1 and %d characters and contain "
+                            "only [A-Za-z0-9+_-].",
+                 client_name, REND_CLIENTNAME_MAX_LEN);
+        SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
+        smartlist_free(clients);
         goto err;
       }
-      continue;
+      client = tor_malloc_zero(sizeof(rend_authorized_client_t));
+      client->client_name = tor_strdup(client_name);
+      smartlist_add(service->clients, client);
+      log_debug(LD_REND, "Adding client name '%s'", client_name);
+    } SMARTLIST_FOREACH_END(client_name);
+    SMARTLIST_FOREACH(clients, char *, cp, tor_free(cp));
+    smartlist_free(clients);
+    /* Ensure maximum number of clients. */
+    if ((service->auth_type == REND_BASIC_AUTH &&
+         smartlist_len(service->clients) > 512) ||
+        (service->auth_type == REND_STEALTH_AUTH &&
+         smartlist_len(service->clients) > 16)) {
+      log_warn(LD_CONFIG, "HiddenServiceAuthorizeClient contains %d "
+                          "client authorization entries, but only a "
+                          "maximum of %d entries is allowed for "
+                          "authorization type '%s'.",
+               smartlist_len(service->clients),
+               service->auth_type == REND_BASIC_AUTH ? 512 : 16,
+               service->auth_type == REND_BASIC_AUTH ? "basic" : "stealth");
+      goto err;
     }
   }
+
   /* Validate the service just parsed. */
   if (rend_validate_service(rend_service_staging_list, service) < 0) {
     /* Service is in the staging list so don't try to free it. */
@@ -2145,7 +2128,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
      *
      * We only use a one-hop path on the first attempt. If the first attempt
      * fails, we use a 3-hop path for reachability / reliability.
-     * See the comment in rend_service_relauch_rendezvous() for details. */
+     * See the comment in rend_service_relaunch_rendezvous() for details. */
     if (rend_service_use_direct_connection(options, rp) && i == 0) {
       flags = flags | CIRCLAUNCH_ONEHOP_TUNNEL;
     }
@@ -2186,7 +2169,7 @@ rend_service_receive_introduction(origin_circuit_t *circuit,
 
   cpath->rend_dh_handshake_state = dh;
   dh = NULL;
-  if (circuit_init_cpath_crypto(cpath,
+  if (cpath_init_circuit_crypto(cpath,
                                 keys+DIGEST_LEN, sizeof(keys)-DIGEST_LEN,
                                 1, 0)<0)
     goto err;
@@ -3035,6 +3018,10 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
 {
   origin_circuit_t *newcirc;
   cpath_build_state_t *newstate, *oldstate;
+  const char *rend_pk_digest;
+  rend_service_t *service = NULL;
+
+  int flags = CIRCLAUNCH_NEED_CAPACITY | CIRCLAUNCH_IS_INTERNAL;
 
   tor_assert(oldcirc->base_.purpose == CIRCUIT_PURPOSE_S_CONNECT_REND);
   oldstate = oldcirc->build_state;
@@ -3049,13 +3036,31 @@ rend_service_relaunch_rendezvous(origin_circuit_t *oldcirc)
   log_info(LD_REND,"Reattempting rendezvous circuit to '%s'",
            safe_str(extend_info_describe(oldstate->chosen_exit)));
 
+  /* Look up the service. */
+  rend_pk_digest = (char *) rend_data_get_pk_digest(oldcirc->rend_data, NULL);
+  service = rend_service_get_by_pk_digest(rend_pk_digest);
+
+  if (!service) {
+    char serviceid[REND_SERVICE_ID_LEN_BASE32+1];
+    base32_encode(serviceid, REND_SERVICE_ID_LEN_BASE32+1,
+                  rend_pk_digest, REND_SERVICE_ID_LEN);
+
+    log_warn(LD_BUG, "Internal error: Trying to relaunch a rendezvous circ "
+                     "for an unrecognized service %s.",
+                     safe_str_client(serviceid));
+    return;
+  }
+
+  if (hs_service_requires_uptime_circ(service->ports)) {
+    flags |= CIRCLAUNCH_NEED_UPTIME;
+  }
+
   /* You'd think Single Onion Services would want to retry the rendezvous
    * using a direct connection. But if it's blocked by a firewall, or the
    * service is IPv6-only, or the rend point avoiding becoming a one-hop
    * proxy, we need a 3-hop connection. */
   newcirc = circuit_launch_by_extend_info(CIRCUIT_PURPOSE_S_CONNECT_REND,
-                            oldstate->chosen_exit,
-                            CIRCLAUNCH_NEED_CAPACITY|CIRCLAUNCH_IS_INTERNAL);
+                            oldstate->chosen_exit, flags);
 
   if (!newcirc) {
     log_warn(LD_REND,"Couldn't relaunch rendezvous circuit to '%s'.",
@@ -3555,7 +3560,7 @@ rend_service_rendezvous_has_opened(origin_circuit_t *circuit)
   hop->package_window = circuit_initial_package_window();
   hop->deliver_window = CIRCWINDOW_START;
 
-  onion_append_to_cpath(&circuit->cpath, hop);
+  cpath_extend_linked_list(&circuit->cpath, hop);
   circuit->build_state->pending_final_cpath = NULL; /* prevent double-free */
 
   /* Change the circuit purpose. */
@@ -3729,20 +3734,23 @@ directory_post_to_hs_dir(rend_service_descriptor_t *renddesc,
       base32_encode(desc_id_base32, sizeof(desc_id_base32),
                     desc->desc_id, DIGEST_LEN);
       hs_dir_ip = tor_dup_ip(hs_dir->addr);
-      log_info(LD_REND, "Launching upload for v2 descriptor for "
-                        "service '%s' with descriptor ID '%s' with validity "
-                        "of %d seconds to hidden service directory '%s' on "
-                        "%s:%d.",
-               safe_str_client(service_id),
-               safe_str_client(desc_id_base32),
-               seconds_valid,
-               hs_dir->nickname,
-               hs_dir_ip,
-               hs_dir->or_port);
+      if (hs_dir_ip) {
+        log_info(LD_REND, "Launching upload for v2 descriptor for "
+                          "service '%s' with descriptor ID '%s' with validity "
+                          "of %d seconds to hidden service directory '%s' on "
+                          "%s:%d.",
+                 safe_str_client(service_id),
+                 safe_str_client(desc_id_base32),
+                 seconds_valid,
+                 hs_dir->nickname,
+                 hs_dir_ip,
+                 hs_dir->or_port);
+        tor_free(hs_dir_ip);
+      }
+
       control_event_hs_descriptor_upload(service_id,
                                          hs_dir->identity_digest,
                                          desc_id_base32, NULL);
-      tor_free(hs_dir_ip);
       /* Remember successful upload to this router for next time. */
       if (!smartlist_contains_digest(successful_uploads,
                                      hs_dir->identity_digest))
@@ -3995,7 +4003,7 @@ remove_invalid_intro_points(rend_service_t *service,
        * accounted for when considiring uploading a descriptor. */
       intro->circuit_established = 0;
 
-      /* Node is gone or we've reached our maximum circuit creationg retry
+      /* Node is gone or we've reached our maximum circuit creation retry
        * count, clean up everything, we'll find a new one. */
       if (node == NULL ||
           intro->circuit_retries >= MAX_INTRO_POINT_CIRCUIT_RETRIES) {
@@ -4235,6 +4243,7 @@ rend_consider_services_intro_points(time_t now)
        * directly ourselves. */
       intro->extend_info = extend_info_from_node(node, 0);
       if (BUG(intro->extend_info == NULL)) {
+        tor_free(intro);
         break;
       }
       intro->intro_key = crypto_pk_new();
@@ -4364,17 +4373,16 @@ rend_consider_descriptor_republication(void)
 void
 rend_service_dump_stats(int severity)
 {
-  int i,j;
   rend_service_t *service;
   rend_intro_point_t *intro;
   const char *safe_name;
   origin_circuit_t *circ;
 
-  for (i=0; i < smartlist_len(rend_service_list); ++i) {
+  for (int i = 0; i < smartlist_len(rend_service_list); ++i) {
     service = smartlist_get(rend_service_list, i);
     tor_log(severity, LD_GENERAL, "Service configured in %s:",
             rend_service_escaped_dir(service));
-    for (j=0; j < smartlist_len(service->intro_nodes); ++j) {
+    for (int j = 0; j < smartlist_len(service->intro_nodes); ++j) {
       intro = smartlist_get(service->intro_nodes, j);
       safe_name = safe_str_client(intro->extend_info->nickname);
 
diff --git a/src/feature/rend/rendservice.h b/src/feature/rend/rendservice.h
index a8eb28bee2..012afc0f9f 100644
--- a/src/feature/rend/rendservice.h
+++ b/src/feature/rend/rendservice.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -139,7 +139,8 @@ STATIC void rend_service_prune_list_impl_(void);
 #endif /* defined(RENDSERVICE_PRIVATE) */
 
 int rend_num_services(void);
-int rend_config_service(const struct config_line_t *line_,
+struct hs_opts_t;
+int rend_config_service(const struct hs_opts_t *hs_opts,
                         const or_options_t *options,
                         hs_service_config_t *config);
 void rend_service_prune_list(void);
diff --git a/src/feature/stats/.may_include b/src/feature/stats/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/feature/stats/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/feature/stats/feature_stats.md b/src/feature/stats/feature_stats.md
new file mode 100644
index 0000000000..d205fe5571
--- /dev/null
+++ b/src/feature/stats/feature_stats.md
@@ -0,0 +1,10 @@
+@dir /feature/stats
+@brief feature/stats: Relay statistics. Also, port prediction.
+
+This module collects anonymized relay statistics in order to publish them in
+relays' routerinfo and extrainfo documents.
+
+Additionally, it contains predict_ports.c, which remembers which ports we've
+visited recently as a client, so we can make sure we have open circuits that
+support them.
+
diff --git a/src/feature/stats/geoip_stats.c b/src/feature/stats/geoip_stats.c
index a54b589eb6..f9a2f19d2e 100644
--- a/src/feature/stats/geoip_stats.c
+++ b/src/feature/stats/geoip_stats.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -30,9 +30,9 @@
 #include "core/or/or.h"
 
 #include "ht.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/dnsserv.h"
 #include "core/or/dos.h"
 #include "lib/geoip/geoip.h"
@@ -146,9 +146,9 @@ clientmap_entries_eq(const clientmap_entry_t *a, const clientmap_entry_t *b)
 }
 
 HT_PROTOTYPE(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
-             clientmap_entries_eq)
+             clientmap_entries_eq);
 HT_GENERATE2(clientmap, clientmap_entry_t, node, clientmap_entry_hash,
-             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             clientmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 #define clientmap_entry_free(ent) \
   FREE_AND_NULL(clientmap_entry_t, clientmap_entry_free_, ent)
@@ -484,9 +484,9 @@ dirreq_map_ent_hash(const dirreq_map_entry_t *entry)
 }
 
 HT_PROTOTYPE(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
-             dirreq_map_ent_eq)
+             dirreq_map_ent_eq);
 HT_GENERATE2(dirreqmap, dirreq_map_entry_t, node, dirreq_map_ent_hash,
-             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+             dirreq_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /** Helper: Put <b>entry</b> into map of directory requests using
  * <b>type</b> and <b>dirreq_id</b> as key parts. If there is
diff --git a/src/feature/stats/geoip_stats.h b/src/feature/stats/geoip_stats.h
index 2fc62b5466..fcfe7a31f0 100644
--- a/src/feature/stats/geoip_stats.h
+++ b/src/feature/stats/geoip_stats.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_GEOIP_STATS_H
 
 #include "core/or/dos.h"
+#include "ext/ht.h"
 
 /** Indicates an action that we might be noting geoip statistics on.
  * Note that if we're noticing CONNECT, we're a bridge, and if we're noticing
diff --git a/src/feature/stats/include.am b/src/feature/stats/include.am
new file mode 100644
index 0000000000..8789bc3d96
--- /dev/null
+++ b/src/feature/stats/include.am
@@ -0,0 +1,12 @@
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+LIBTOR_APP_A_SOURCES += 			\
+	src/feature/stats/geoip_stats.c		\
+	src/feature/stats/rephist.c		\
+	src/feature/stats/predict_ports.c
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/feature/stats/geoip_stats.h			\
+	src/feature/stats/rephist.h			\
+	src/feature/stats/predict_ports.h
diff --git a/src/feature/stats/predict_ports.c b/src/feature/stats/predict_ports.c
index 3cbba2c831..d728f106a2 100644
--- a/src/feature/stats/predict_ports.c
+++ b/src/feature/stats/predict_ports.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/feature/stats/predict_ports.h b/src/feature/stats/predict_ports.h
index 272344da2f..ed067b6ced 100644
--- a/src/feature/stats/predict_ports.h
+++ b/src/feature/stats/predict_ports.h
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file predict_portst.h
+ * \file predict_ports.h
  * \brief Header file for predict_ports.c.
  **/
 
@@ -27,4 +27,4 @@ int rep_hist_circbuilding_dormant(time_t now);
 int predicted_ports_prediction_time_remaining(time_t now);
 void predicted_ports_free_all(void);
 
-#endif
+#endif /* !defined(TOR_PREDICT_PORTS_H) */
diff --git a/src/feature/stats/rephist.c b/src/feature/stats/rephist.c
index 3f560fbce7..71e2e00086 100644
--- a/src/feature/stats/rephist.c
+++ b/src/feature/stats/rephist.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -1122,7 +1122,7 @@ static bw_array_t *dir_read_array = NULL;
     directory protocol. */
 static bw_array_t *dir_write_array = NULL;
 
-/** Set up [dir-]read_array and [dir-]write_array, freeing them if they
+/** Set up [dir_]read_array and [dir_]write_array, freeing them if they
  * already exist. */
 static void
 bw_arrays_init(void)
@@ -2285,9 +2285,9 @@ bidi_map_ent_hash(const bidi_map_entry_t *entry)
 }
 
 HT_PROTOTYPE(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
-             bidi_map_ent_eq)
+             bidi_map_ent_eq);
 HT_GENERATE2(bidimap, bidi_map_entry_t, node, bidi_map_ent_hash,
-             bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_)
+             bidi_map_ent_eq, 0.6, tor_reallocarray_, tor_free_);
 
 /* DOCDOC bidi_map_free */
 static void
diff --git a/src/feature/stats/rephist.h b/src/feature/stats/rephist.h
index 3accc8c610..92c3d2a5a5 100644
--- a/src/feature/stats/rephist.h
+++ b/src/feature/stats/rephist.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -103,7 +103,7 @@ typedef struct bw_array_t bw_array_t;
 STATIC uint64_t find_largest_max(bw_array_t *b);
 STATIC void commit_max(bw_array_t *b);
 STATIC void advance_obs(bw_array_t *b);
-#endif
+#endif /* defined(REPHIST_PRIVATE) */
 
 /**
  * Represents the type of a cell for padding accounting
diff --git a/src/include.am b/src/include.am
index d2f83da814..657f6e823a 100644
--- a/src/include.am
+++ b/src/include.am
@@ -1,12 +1,17 @@
 include src/ext/include.am
+
 include src/lib/arch/include.am
+include src/lib/buf/include.am
 include src/lib/err/include.am
 include src/lib/cc/include.am
 include src/lib/ctime/include.am
 include src/lib/compress/include.am
+include src/lib/conf/include.am
+include src/lib/confmgt/include.am
 include src/lib/container/include.am
 include src/lib/crypt_ops/include.am
 include src/lib/defs/include.am
+include src/lib/dispatch/include.am
 include src/lib/encoding/include.am
 include src/lib/evloop/include.am
 include src/lib/fdio/include.am
@@ -14,6 +19,7 @@ include src/lib/fs/include.am
 include src/lib/geoip/include.am
 include src/lib/include.libdonna.am
 include src/lib/intmath/include.am
+include src/lib/llharden/include.am
 include src/lib/lock/include.am
 include src/lib/log/include.am
 include src/lib/math/include.am
@@ -23,8 +29,10 @@ include src/lib/malloc/include.am
 include src/lib/net/include.am
 include src/lib/osinfo/include.am
 include src/lib/process/include.am
+include src/lib/pubsub/include.am
 include src/lib/sandbox/include.am
 include src/lib/string/include.am
+include src/lib/subsys/include.am
 include src/lib/smartlist_core/include.am
 include src/lib/term/include.am
 include src/lib/testsupport/include.am
@@ -32,9 +40,46 @@ include src/lib/thread/include.am
 include src/lib/time/include.am
 include src/lib/tls/include.am
 include src/lib/trace/include.am
+include src/lib/version/include.am
 include src/lib/wallclock/include.am
 include src/trunnel/include.am
 
+noinst_LIBRARIES += src/core/libtor-app.a
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/core/libtor-app-testing.a
+endif
+
+LIBTOR_APP_A_SOURCES =
+#
+# Sources that we only add for the real libtor_a, and not for testing.
+#
+LIBTOR_APP_A_STUB_SOURCES =
+
+include src/core/crypto/include.am
+include src/core/mainloop/include.am
+include src/core/or/include.am
+include src/core/proto/include.am
+
+include src/feature/api/include.am
+include src/feature/client/include.am
+include src/feature/control/include.am
+include src/feature/dirauth/include.am
+include src/feature/dircache/include.am
+include src/feature/dirclient/include.am
+include src/feature/dircommon/include.am
+include src/feature/dirparse/include.am
+include src/feature/hibernate/include.am
+include src/feature/hs_common/include.am
+include src/feature/hs/include.am
+include src/feature/keymgt/include.am
+include src/feature/nodelist/include.am
+include src/feature/relay/include.am
+include src/feature/rend/include.am
+include src/feature/stats/include.am
+
+include src/app/config/include.am
+include src/app/main/include.am
+
 include src/core/include.am
 include src/app/include.am
 
diff --git a/src/lib/arch/bytes.h b/src/lib/arch/bytes.h
index fa82241b28..c72ac3eb8e 100644
--- a/src/lib/arch/bytes.h
+++ b/src/lib/arch/bytes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BYTES_H
@@ -16,12 +16,17 @@
 #include <string.h>
 #include "lib/cc/torint.h"
 
-/* The uint8 variants are defined to make the code more uniform. */
+/**
+ * Read an 8-bit from <b>cp</b>.
+ */
 static inline uint8_t
 get_uint8(const void *cp)
 {
   return *(const uint8_t*)(cp);
 }
+/**
+ * Store an 8-bit value from <b>v</b> to <b>cp</b>.
+ */
 static inline void
 set_uint8(void *cp, uint8_t v)
 {
@@ -93,7 +98,7 @@ set_uint64(void *cp, uint64_t v)
   memcpy(cp,&v,8);
 }
 
-#ifdef WORDS_BIGENDIAN
+#if defined(WORDS_BIGENDIAN)
 static inline uint16_t
 tor_htons(uint32_t a)
 {
@@ -129,7 +134,10 @@ tor_ntohll(uint64_t a)
 {
   return a;
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
+/**
+ * Convert a 16-bit value from host order to network order (big-endian).
+ **/
 static inline uint16_t
 tor_htons(uint16_t a)
 {
@@ -139,12 +147,18 @@ tor_htons(uint16_t a)
     ((a & 0xff00) >> 8);
 }
 
+/**
+ * Convert a 16-bit value from network order (big-endian) to host order.
+ **/
 static inline uint16_t
 tor_ntohs(uint16_t a)
 {
   return tor_htons(a);
 }
 
+/**
+ * Convert a 32-bit value from host order to network order (big-endian).
+ **/
 static inline uint32_t
 tor_htonl(uint32_t a)
 {
@@ -156,6 +170,9 @@ tor_htonl(uint32_t a)
     ((a & 0xff000000) >>24);
 }
 
+/**
+ * Convert a 32-bit value from network order (big-endian) to host order.
+ **/
 static inline uint32_t
 tor_ntohl(uint32_t a)
 {
@@ -177,6 +194,6 @@ tor_ntohll(uint64_t a)
 {
   return tor_htonll(a);
 }
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
-#endif
+#endif /* !defined(TOR_BYTES_H) */
diff --git a/src/lib/arch/include.am b/src/lib/arch/include.am
index f92ee9222f..c5926c6330 100644
--- a/src/lib/arch/include.am
+++ b/src/lib/arch/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/arch/bytes.h
diff --git a/src/lib/arch/lib_arch.md b/src/lib/arch/lib_arch.md
new file mode 100644
index 0000000000..9b8bccdf16
--- /dev/null
+++ b/src/lib/arch/lib_arch.md
@@ -0,0 +1,2 @@
+@dir /lib/arch
+@brief lib/arch: Compatibility code for handling different CPU architectures.
diff --git a/src/lib/buf/.may_include b/src/lib/buf/.may_include
new file mode 100644
index 0000000000..c4be73bce2
--- /dev/null
+++ b/src/lib/buf/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/buf/*.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
+lib/log/*.h
+lib/string/*.h
+lib/time/*.h
diff --git a/src/lib/container/buffers.c b/src/lib/buf/buffers.c
index fe4cf7c385..a5031a47a6 100644
--- a/src/lib/container/buffers.c
+++ b/src/lib/buf/buffers.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -99,6 +99,7 @@
 #define DBG_S(s) (void)0
 #endif
 
+#ifndef COCCI
 #ifdef DISABLE_MEMORY_SENTINELS
 #define CHUNK_SET_SENTINEL(chunk, alloclen) STMT_NIL
 #else
@@ -109,6 +110,7 @@
     memset(a,0,SENTINEL_LEN);                                           \
   } while (0)
 #endif /* defined(DISABLE_MEMORY_SENTINELS) */
+#endif /* !defined(COCCI) */
 
 /** Move all bytes stored in <b>chunk</b> to the front of <b>chunk</b>->mem,
  * to free up space at the end. */
@@ -158,7 +160,7 @@ chunk_new_with_alloc_size(size_t alloc)
 static inline chunk_t *
 chunk_grow(chunk_t *chunk, size_t sz)
 {
-  off_t offset;
+  ptrdiff_t offset;
   const size_t memlen_orig = chunk->memlen;
   const size_t orig_alloc = CHUNK_ALLOC_SIZE(memlen_orig);
   const size_t new_alloc = CHUNK_ALLOC_SIZE(sz);
@@ -283,7 +285,7 @@ buf_t *
 buf_new_with_data(const char *cp, size_t sz)
 {
   /* Validate arguments */
-  if (!cp || sz <= 0 || sz >= INT_MAX) {
+  if (!cp || sz <= 0 || sz > BUF_MAX_LEN) {
     return NULL;
   }
 
@@ -440,7 +442,7 @@ chunk_copy(const chunk_t *in_chunk)
 #endif
   newch->next = NULL;
   if (in_chunk->data) {
-    off_t offset = in_chunk->data - in_chunk->mem;
+    ptrdiff_t offset = in_chunk->data - in_chunk->mem;
     newch->data = newch->mem + offset;
   }
   return newch;
@@ -528,9 +530,9 @@ buf_add(buf_t *buf, const char *string, size_t string_len)
     return (int)buf->datalen;
   check();
 
-  if (BUG(buf->datalen >= INT_MAX))
+  if (BUG(buf->datalen > BUF_MAX_LEN))
     return -1;
-  if (BUG(buf->datalen >= INT_MAX - string_len))
+  if (BUG(buf->datalen > BUF_MAX_LEN - string_len))
     return -1;
 
   while (string_len) {
@@ -549,7 +551,7 @@ buf_add(buf_t *buf, const char *string, size_t string_len)
   }
 
   check();
-  tor_assert(buf->datalen < INT_MAX);
+  tor_assert(buf->datalen <= BUF_MAX_LEN);
   return (int)buf->datalen;
 }
 
@@ -578,6 +580,7 @@ buf_add_vprintf(buf_t *buf, const char *format, va_list args)
   /* XXXX Faster implementations are easy enough, but let's optimize later */
   char *tmp;
   tor_vasprintf(&tmp, format, args);
+  tor_assert(tmp != NULL);
   buf_add(buf, tmp, strlen(tmp));
   tor_free(tmp);
 }
@@ -642,7 +645,7 @@ buf_get_bytes(buf_t *buf, char *string, size_t string_len)
   buf_peek(buf, string, string_len);
   buf_drain(buf, string_len);
   check();
-  tor_assert(buf->datalen < INT_MAX);
+  tor_assert(buf->datalen <= BUF_MAX_LEN);
   return (int)buf->datalen;
 }
 
@@ -657,9 +660,9 @@ buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
   char b[4096];
   size_t cp, len;
 
-  if (BUG(buf_out->datalen >= INT_MAX || *buf_flushlen >= INT_MAX))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN || *buf_flushlen > BUF_MAX_LEN))
     return -1;
-  if (BUG(buf_out->datalen >= INT_MAX - *buf_flushlen))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN - *buf_flushlen))
     return -1;
 
   len = *buf_flushlen;
@@ -667,7 +670,7 @@ buf_move_to_buf(buf_t *buf_out, buf_t *buf_in, size_t *buf_flushlen)
     len = buf_in->datalen;
 
   cp = len; /* Remember the number of bytes we intend to copy. */
-  tor_assert(cp < INT_MAX);
+  tor_assert(cp <= BUF_MAX_LEN);
   while (len) {
     /* This isn't the most efficient implementation one could imagine, since
      * it does two copies instead of 1, but I kinda doubt that this will be
@@ -691,9 +694,9 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in)
     return;
   if (buf_datalen(buf_in) == 0)
     return;
-  if (BUG(buf_out->datalen >= INT_MAX || buf_in->datalen >= INT_MAX))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN || buf_in->datalen > BUF_MAX_LEN))
     return;
-  if (BUG(buf_out->datalen >= INT_MAX - buf_in->datalen))
+  if (BUG(buf_out->datalen > BUF_MAX_LEN - buf_in->datalen))
     return;
 
   if (buf_out->head == NULL) {
@@ -712,7 +715,8 @@ buf_move_all(buf_t *buf_out, buf_t *buf_in)
 /** Internal structure: represents a position in a buffer. */
 typedef struct buf_pos_t {
   const chunk_t *chunk; /**< Which chunk are we pointing to? */
-  int pos;/**< Which character inside the chunk's data are we pointing to? */
+  ptrdiff_t pos;/**< Which character inside the chunk's data are we pointing
+                 * to? */
   size_t chunk_pos; /**< Total length of all previous chunks. */
 } buf_pos_t;
 
@@ -728,15 +732,15 @@ buf_pos_init(const buf_t *buf, buf_pos_t *out)
 /** Advance <b>out</b> to the first appearance of <b>ch</b> at the current
  * position of <b>out</b>, or later.  Return -1 if no instances are found;
  * otherwise returns the absolute position of the character. */
-static off_t
+static ptrdiff_t
 buf_find_pos_of_char(char ch, buf_pos_t *out)
 {
   const chunk_t *chunk;
-  int pos;
+  ptrdiff_t pos;
   tor_assert(out);
   if (out->chunk) {
     if (out->chunk->datalen) {
-      tor_assert(out->pos < (off_t)out->chunk->datalen);
+      tor_assert(out->pos < (ptrdiff_t)out->chunk->datalen);
     } else {
       tor_assert(out->pos == 0);
     }
@@ -746,7 +750,7 @@ buf_find_pos_of_char(char ch, buf_pos_t *out)
     char *cp = memchr(chunk->data+pos, ch, chunk->datalen - pos);
     if (cp) {
       out->chunk = chunk;
-      tor_assert(cp - chunk->data < INT_MAX);
+      tor_assert(cp - chunk->data <= BUF_MAX_LEN);
       out->pos = (int)(cp - chunk->data);
       return out->chunk_pos + out->pos;
     } else {
@@ -762,9 +766,9 @@ buf_find_pos_of_char(char ch, buf_pos_t *out)
 static inline int
 buf_pos_inc(buf_pos_t *pos)
 {
-  tor_assert(pos->pos < INT_MAX - 1);
+  tor_assert(pos->pos < BUF_MAX_LEN);
   ++pos->pos;
-  if (pos->pos == (off_t)pos->chunk->datalen) {
+  if (pos->pos == (ptrdiff_t)pos->chunk->datalen) {
     if (!pos->chunk->next)
       return -1;
     pos->chunk_pos += pos->chunk->datalen;
@@ -809,7 +813,7 @@ buf_find_string_offset(const buf_t *buf, const char *s, size_t n)
   buf_pos_init(buf, &pos);
   while (buf_find_pos_of_char(*s, &pos) >= 0) {
     if (buf_matches_at_pos(&pos, s, n)) {
-      tor_assert(pos.chunk_pos + pos.pos < INT_MAX);
+      tor_assert(pos.chunk_pos + pos.pos <= BUF_MAX_LEN);
       return (int)(pos.chunk_pos + pos.pos);
     } else {
       if (buf_pos_inc(&pos)<0)
@@ -838,12 +842,12 @@ buf_peek_startswith(const buf_t *buf, const char *cmd)
 
 /** Return the index within <b>buf</b> at which <b>ch</b> first appears,
  * or -1 if <b>ch</b> does not appear on buf. */
-static off_t
+static ptrdiff_t
 buf_find_offset_of_char(buf_t *buf, char ch)
 {
   chunk_t *chunk;
-  off_t offset = 0;
-  tor_assert(buf->datalen < INT_MAX);
+  ptrdiff_t offset = 0;
+  tor_assert(buf->datalen <= BUF_MAX_LEN);
   for (chunk = buf->head; chunk; chunk = chunk->next) {
     char *cp = memchr(chunk->data, ch, chunk->datalen);
     if (cp)
@@ -865,7 +869,7 @@ int
 buf_get_line(buf_t *buf, char *data_out, size_t *data_len)
 {
   size_t sz;
-  off_t offset;
+  ptrdiff_t offset;
 
   if (!buf->head)
     return 0;
@@ -913,7 +917,7 @@ buf_assert_ok(buf_t *buf)
     for (ch = buf->head; ch; ch = ch->next) {
       total += ch->datalen;
       tor_assert(ch->datalen <= ch->memlen);
-      tor_assert(ch->datalen < INT_MAX);
+      tor_assert(ch->datalen <= BUF_MAX_LEN);
       tor_assert(ch->data >= &ch->mem[0]);
       tor_assert(ch->data <= &ch->mem[0]+ch->memlen);
       if (ch->data == &ch->mem[0]+ch->memlen) {
diff --git a/src/lib/container/buffers.h b/src/lib/buf/buffers.h
index c103b93a82..d8a77feb72 100644
--- a/src/lib/container/buffers.h
+++ b/src/lib/buf/buffers.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,9 @@ void buf_free_(buf_t *buf);
 void buf_clear(buf_t *buf);
 buf_t *buf_copy(const buf_t *buf);
 
+/** Maximum bytes in a buffer, inclusive. */
+#define BUF_MAX_LEN (INT_MAX - 1)
+
 MOCK_DECL(size_t, buf_datalen, (const buf_t *buf));
 size_t buf_allocation(const buf_t *buf);
 size_t buf_slack(const buf_t *buf);
diff --git a/src/lib/buf/include.am b/src/lib/buf/include.am
new file mode 100644
index 0000000000..27430d1d38
--- /dev/null
+++ b/src/lib/buf/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-buf.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-buf-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_buf_a_SOURCES =			\
+	src/lib/buf/buffers.c
+
+src_lib_libtor_buf_testing_a_SOURCES = \
+	$(src_lib_libtor_buf_a_SOURCES)
+src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/buf/buffers.h
diff --git a/src/lib/buf/lib_buf.md b/src/lib/buf/lib_buf.md
new file mode 100644
index 0000000000..519ab50a2d
--- /dev/null
+++ b/src/lib/buf/lib_buf.md
@@ -0,0 +1,13 @@
+@dir /lib/buf
+@brief lib/buf: An efficient byte queue.
+
+This module defines the buf_t type, which is used throughout our networking
+code.  The implementation is a singly-linked queue of buffer chunks, similar
+to the BSD kernel's
+["mbuf"](https://www.freebsd.org/cgi/man.cgi?query=mbuf&sektion=9) structure.
+
+The buf_t type is also reasonable for use in constructing long strings.
+
+See \refdir{lib/net} for networking code that uses buf_t, and
+\refdir{lib/tls} for cryptographic code that uses buf_t.
+
diff --git a/src/lib/cc/.may_include b/src/lib/cc/.may_include
index 2b06e8519c..fa1478ce46 100644
--- a/src/lib/cc/.may_include
+++ b/src/lib/cc/.may_include
@@ -1 +1,2 @@
 orconfig.h
+lib/cc/*.h
\ No newline at end of file
diff --git a/src/lib/cc/compat_compiler.h b/src/lib/cc/compat_compiler.h
index fbe6a38f1f..96aa912652 100644
--- a/src/lib/cc/compat_compiler.h
+++ b/src/lib/cc/compat_compiler.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,11 +25,11 @@
 #endif /* defined(__has_feature) */
 
 #ifndef NULL_REP_IS_ZERO_BYTES
-#error "It seems your platform does not represent NULL as zero. We can't cope."
+#error "Your platform does not represent NULL as zero. We can't cope."
 #endif
 
 #ifndef DOUBLE_0_REP_IS_ZERO_BYTES
-#error "It seems your platform does not represent 0.0 as zeros. We can't cope."
+#error "Your platform does not represent 0.0 as zeros. We can't cope."
 #endif
 
 #if 'a'!=97 || 'z'!=122 || 'A'!=65 || ' '!=32
@@ -65,8 +65,6 @@
 
 /* Temporarily enable and disable warnings. */
 #ifdef __GNUC__
-#  define PRAGMA_STRINGIFY_(s) #s
-#  define PRAGMA_JOIN_STRINGIFY_(a,b) PRAGMA_STRINGIFY_(a ## b)
 /* Support for macro-generated pragmas (c99) */
 #  define PRAGMA_(x) _Pragma (#x)
 #  ifdef __clang__
@@ -78,17 +76,17 @@
 /* we have push/pop support */
 #    define DISABLE_GCC_WARNING(warningopt) \
           PRAGMA_DIAGNOSTIC_(push) \
-          PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+          PRAGMA_DIAGNOSTIC_(ignored warningopt)
 #    define ENABLE_GCC_WARNING(warningopt) \
           PRAGMA_DIAGNOSTIC_(pop)
 #else /* !(defined(__clang__) || GCC_VERSION >= 406) */
 /* older version of gcc: no push/pop support. */
 #    define DISABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(ignored PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+         PRAGMA_DIAGNOSTIC_(ignored warningopt)
 #    define ENABLE_GCC_WARNING(warningopt) \
-         PRAGMA_DIAGNOSTIC_(warning PRAGMA_JOIN_STRINGIFY_(-W,warningopt))
+         PRAGMA_DIAGNOSTIC_(warning warningopt)
 #endif /* defined(__clang__) || GCC_VERSION >= 406 */
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 /* not gcc at all */
 # define DISABLE_GCC_WARNING(warning)
 # define ENABLE_GCC_WARNING(warning)
@@ -200,8 +198,8 @@
 /** Macro: yield a pointer to the field at position <b>off</b> within the
  * structure <b>st</b>.  Example:
  * <pre>
- *   struct a { int foo; int bar; } x;
- *   off_t bar_offset = offsetof(struct a, bar);
+ *   struct a_t { int foo; int bar; } x;
+ *   ptrdiff_t bar_offset = offsetof(struct a_t, bar);
  *   int *bar_p = STRUCT_VAR_P(&x, bar_offset);
  *   *bar_p = 3;
  * </pre>
@@ -211,10 +209,10 @@
 /** Macro: yield a pointer to an enclosing structure given a pointer to
  * a substructure at offset <b>off</b>. Example:
  * <pre>
- *   struct base { ... };
- *   struct subtype { int x; struct base b; } x;
- *   struct base *bp = &x.base;
- *   struct *sp = SUBTYPE_P(bp, struct subtype, b);
+ *   struct base_t { ... };
+ *   struct subtype_t { int x; struct base_t b; } x;
+ *   struct base_t *bp = &x.base;
+ *   struct *sp = SUBTYPE_P(bp, struct subtype_t, b);
  * </pre>
  */
 #define SUBTYPE_P(p, subtype, basemember) \
@@ -223,4 +221,29 @@
 /** Macro: Yields the number of elements in array x. */
 #define ARRAY_LENGTH(x) ((sizeof(x)) / sizeof(x[0]))
 
-#endif /* !defined(TOR_COMPAT_H) */
+/**
+ * "Eat" a semicolon that somebody puts at the end of a top-level macro.
+ *
+ * Frequently, we want to declare a macro that people will use at file scope,
+ * and we want to allow people to put a semicolon after the macro.
+ *
+ * This declaration of a struct can be repeated any number of times, and takes
+ * a trailing semicolon afterwards.
+ **/
+#define EAT_SEMICOLON                                   \
+  struct dummy_semicolon_eater__
+
+/**
+ * Tell our static analysis tool to believe that (clang's scan-build or
+ * coverity scan) that an expression might be true.  We use this to suppress
+ * dead-code warnings.
+ **/
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* By calling getenv, we force the analyzer not to conclude that 'expr' is
+ * false. */
+#define POSSIBLE(expr) ((expr) || getenv("STATIC_ANALYZER_DEADCODE_DUMMY_"))
+#else
+#define POSSIBLE(expr) (expr)
+#endif /* defined(__COVERITY__) || defined(__clang_analyzer__) */
+
+#endif /* !defined(TOR_COMPAT_COMPILER_H) */
diff --git a/src/lib/cc/ctassert.h b/src/lib/cc/ctassert.h
new file mode 100644
index 0000000000..d9d3aa40b0
--- /dev/null
+++ b/src/lib/cc/ctassert.h
@@ -0,0 +1,53 @@
+/* Copyright (c) 2018 The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file ctassert.h
+ *
+ * \brief Compile-time assertions: CTASSERT(expression).
+ */
+
+#ifndef TOR_CTASSERT_H
+#define TOR_CTASSERT_H
+
+#include "lib/cc/compat_compiler.h"
+
+/**
+ * CTASSERT(expression)
+ *
+ *       Trigger a compiler error if expression is false.
+ */
+#if __STDC_VERSION__ >= 201112L
+
+/* If C11 is available, just use _Static_assert.  */
+#define CTASSERT(x) _Static_assert((x), #x)
+
+#else /* !(__STDC_VERSION__ >= 201112L) */
+
+/*
+ * If C11 is not available, expand __COUNTER__, or __INCLUDE_LEVEL__
+ * and __LINE__, or just __LINE__, with an intermediate preprocessor
+ * macro CTASSERT_EXPN, and then use CTASSERT_DECL to paste the
+ * expansions together into a unique name.
+ *
+ * We use this name as a typedef of an array type with a positive
+ * length if the assertion is true, and a negative length of the
+ * assertion is false, which is invalid and hence triggers a compiler
+ * error.
+ */
+#if defined(__COUNTER__)
+#define CTASSERT(x) CTASSERT_EXPN((x), c, __COUNTER__)
+#elif defined(__INCLUDE_LEVEL__)
+#define CTASSERT(x) CTASSERT_EXPN((x), __INCLUDE_LEVEL__, __LINE__)
+#else
+/* hope it's unique enough */
+#define CTASSERT(x) CTASSERT_EXPN((x), l, __LINE__)
+#endif /* defined(__COUNTER__) || ... */
+
+#define CTASSERT_EXPN(x, a, b) CTASSERT_DECL(x, a, b)
+#define CTASSERT_DECL(x, a, b) \
+  typedef char tor_ctassert_##a##_##b[(x) ? 1 : -1] ATTR_UNUSED; EAT_SEMICOLON
+
+#endif /* __STDC_VERSION__ >= 201112L */
+
+#endif /* !defined(TOR_CTASSERT_H) */
diff --git a/src/lib/cc/include.am b/src/lib/cc/include.am
index 2ae90f97dd..d2a415e956 100644
--- a/src/lib/cc/include.am
+++ b/src/lib/cc/include.am
@@ -1,4 +1,7 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/cc/compat_compiler.h \
+	src/lib/cc/ctassert.h \
+	src/lib/cc/tokpaste.h \
 	src/lib/cc/torint.h
diff --git a/src/lib/cc/lib_cc.md b/src/lib/cc/lib_cc.md
new file mode 100644
index 0000000000..bd49005ba2
--- /dev/null
+++ b/src/lib/cc/lib_cc.md
@@ -0,0 +1,2 @@
+@dir /lib/cc
+@brief lib/cc: Macros for managing the C compiler and language.
diff --git a/src/lib/cc/tokpaste.h b/src/lib/cc/tokpaste.h
new file mode 100644
index 0000000000..068621b5bd
--- /dev/null
+++ b/src/lib/cc/tokpaste.h
@@ -0,0 +1,30 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file tokpaste.h
+ * @brief Token-pasting macros.
+ **/
+
+#ifndef TOR_LIB_CC_TOKPASTE_H
+#define TOR_LIB_CC_TOKPASTE_H
+
+/**
+ * Concatenate `a` and `b` in a way that allows their result itself to be
+ * expanded by the preprocessor.
+ *
+ * Ordinarily you could just say `a ## b` in a macro definition.  But doing so
+ * results in a symbol which the preprocessor will not then expand.  If you
+ * wanted to use `a ## b` to create the name of a macro and have the
+ * preprocessor expand _that_ macro, you need to have another level of
+ * indirection, as this macro provides.
+ **/
+#define PASTE(a,b) PASTE__(a,b)
+
+/** Helper for PASTE(). */
+#define PASTE__(a,b) a ## b
+
+#endif /* !defined(TOR_LIB_CC_TOKPASTE_H) */
diff --git a/src/lib/cc/torint.h b/src/lib/cc/torint.h
index c9b2d329f2..af7a90431c 100644
--- a/src/lib/cc/torint.h
+++ b/src/lib/cc/torint.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -49,7 +49,7 @@ typedef int32_t ssize_t;
  * aren't 2's complement, and you don't define LONG_MAX, then you're so
  * bizarre that I want nothing to do with you. */
 #ifndef USING_TWOS_COMPLEMENT
-#error "Seems that your platform doesn't use 2's complement arithmetic. Argh."
+#error "Your platform doesn't use 2's complement arithmetic."
 #endif
 
 #ifndef TIME_MAX
@@ -96,9 +96,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIuSZ PRIu32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIuSZ "zu"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef _WIN32
 #  ifdef _WIN64
@@ -106,9 +106,9 @@ typedef int32_t ssize_t;
 #  else
 #    define TOR_PRIdSZ PRId32
 #  endif
-#else
+#else /* !defined(_WIN32) */
 #  define TOR_PRIdSZ "zd"
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef SSIZE_MAX
 #if (SIZEOF_SIZE_T == 4)
@@ -125,4 +125,12 @@ typedef int32_t ssize_t;
 /** Any size_t larger than this amount is likely to be an underflow. */
 #define SIZE_T_CEILING  ((size_t)(SSIZE_MAX-16))
 
+#if SIZEOF_INT > SIZEOF_VOID_P
+#error "sizeof(int) > sizeof(void *) - Can't build Tor here."
+#endif
+
+#if SIZEOF_UNSIGNED_INT > SIZEOF_VOID_P
+#error "sizeof(unsigned int) > sizeof(void *) - Can't build Tor here."
+#endif
+
 #endif /* !defined(TOR_TORINT_H) */
diff --git a/src/lib/compress/.may_include b/src/lib/compress/.may_include
index 68fe9f1c54..6cd80086e6 100644
--- a/src/lib/compress/.may_include
+++ b/src/lib/compress/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/compress/*.h
 lib/container/*.h
@@ -8,5 +9,6 @@ lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
diff --git a/src/lib/compress/compress.c b/src/lib/compress/compress.c
index 95fd73bb32..7ce3910d84 100644
--- a/src/lib/compress/compress.c
+++ b/src/lib/compress/compress.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,10 +29,12 @@
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_none.h"
+#include "lib/compress/compress_sys.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
 #include "lib/intmath/cmp.h"
 #include "lib/malloc/malloc.h"
+#include "lib/subsys/subsys.h"
 #include "lib/thread/threads.h"
 
 /** Total number of bytes allocated for compression state overhead. */
@@ -660,7 +662,7 @@ tor_compress_state_size(const tor_compress_state_t *state)
 }
 
 /** Initialize all compression modules. */
-void
+int
 tor_compress_init(void)
 {
   atomic_counter_init(&total_compress_allocation);
@@ -668,6 +670,8 @@ tor_compress_init(void)
   tor_zlib_init();
   tor_lzma_init();
   tor_zstd_init();
+
+  return 0;
 }
 
 /** Warn if we had any problems while setting up our compression libraries.
@@ -677,5 +681,21 @@ tor_compress_init(void)
 void
 tor_compress_log_init_warnings(void)
 {
+  // XXXX can we move this into tor_compress_init() after all?  log.c queues
+  // XXXX log messages at startup.
   tor_zstd_warn_if_version_mismatched();
 }
+
+static int
+subsys_compress_initialize(void)
+{
+  return tor_compress_init();
+}
+
+const subsys_fns_t sys_compress = {
+  .name = "compress",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -55,
+  .initialize = subsys_compress_initialize,
+};
diff --git a/src/lib/compress/compress.h b/src/lib/compress/compress.h
index 5f16a2ab27..f36cdb82aa 100644
--- a/src/lib/compress/compress.h
+++ b/src/lib/compress/compress.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -89,7 +89,7 @@ void tor_compress_free_(tor_compress_state_t *state);
 
 size_t tor_compress_state_size(const tor_compress_state_t *state);
 
-void tor_compress_init(void);
+int tor_compress_init(void);
 void tor_compress_log_init_warnings(void);
 
 struct buf_t;
diff --git a/src/lib/compress/compress_buf.c b/src/lib/compress/compress_buf.c
index 198128b261..d1941c9da6 100644
--- a/src/lib/compress/compress_buf.c
+++ b/src/lib/compress/compress_buf.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/cc/compat_compiler.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/compress/compress.h"
 #include "lib/log/util_bug.h"
 
diff --git a/src/lib/compress/compress_lzma.c b/src/lib/compress/compress_lzma.c
index 2dab37e433..8884b020e8 100644
--- a/src/lib/compress/compress_lzma.c
+++ b/src/lib/compress/compress_lzma.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -221,7 +221,7 @@ tor_lzma_compress_new(int compress,
   tor_free(result);
   return NULL;
  /* LCOV_EXCL_STOP */
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)compress;
   (void)method;
   (void)level;
@@ -312,7 +312,7 @@ tor_lzma_compress_process(tor_lzma_compress_state_t *state,
                lzma_error_str(retval));
       return TOR_COMPRESS_ERROR;
   }
-#else /* !(defined(HAVE_LZMA)) */
+#else /* !defined(HAVE_LZMA) */
   (void)state;
   (void)out;
   (void)out_len;
diff --git a/src/lib/compress/compress_lzma.h b/src/lib/compress/compress_lzma.h
index 556ab437dc..de03cda91c 100644
--- a/src/lib/compress/compress_lzma.h
+++ b/src/lib/compress/compress_lzma.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_none.c b/src/lib/compress/compress_none.c
index 0b5760773a..43c021c788 100644
--- a/src/lib/compress/compress_none.c
+++ b/src/lib/compress/compress_none.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_none.h b/src/lib/compress/compress_none.h
index 2bb9c3d66c..df696a11aa 100644
--- a/src/lib/compress/compress_none.h
+++ b/src/lib/compress/compress_none.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_sys.h b/src/lib/compress/compress_sys.h
new file mode 100644
index 0000000000..dce0549924
--- /dev/null
+++ b/src/lib/compress/compress_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file compress_sys.h
+ * \brief Declare subsystem object for the compress module
+ **/
+
+#ifndef TOR_COMPRESS_SYS_H
+#define TOR_COMPRESS_SYS_H
+
+extern const struct subsys_fns_t sys_compress;
+
+#endif /* !defined(TOR_COMPRESS_SYS_H) */
diff --git a/src/lib/compress/compress_zlib.c b/src/lib/compress/compress_zlib.c
index df0d1bff5f..3711e46222 100644
--- a/src/lib/compress/compress_zlib.c
+++ b/src/lib/compress/compress_zlib.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_zlib.h b/src/lib/compress/compress_zlib.h
index e4f248cd9b..7328ce899b 100644
--- a/src/lib/compress/compress_zlib.h
+++ b/src/lib/compress/compress_zlib.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/compress_zstd.c b/src/lib/compress/compress_zstd.c
index 45d0d4d602..5913d823e1 100644
--- a/src/lib/compress/compress_zstd.c
+++ b/src/lib/compress/compress_zstd.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,17 +25,17 @@
  * all invocations of zstd's static-only functions in a check to make sure
  * that the compile-time version matches the run-time version. */
 #define ZSTD_STATIC_LINKING_ONLY
-#endif
+#endif /* defined(ENABLE_ZSTD_ADVANCED_APIS) */
 
 #ifdef HAVE_ZSTD
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
-DISABLE_GCC_WARNING(unused-const-variable)
+DISABLE_GCC_WARNING("-Wunused-const-variable")
 #endif
 #include <zstd.h>
 #ifdef HAVE_CFLAG_WUNUSED_CONST_VARIABLE
-ENABLE_GCC_WARNING(unused-const-variable)
-#endif
+ENABLE_GCC_WARNING("-Wunused-const-variable")
 #endif
+#endif /* defined(HAVE_ZSTD) */
 
 /** Total number of bytes allocated for Zstandard state. */
 static atomic_counter_t total_zstd_allocation;
@@ -77,7 +77,7 @@ tor_zstd_format_version(char *buf, size_t buflen, unsigned version_number)
                version_number / 100 % 100,
                version_number % 100);
 }
-#endif
+#endif /* defined(HAVE_ZSTD) */
 
 #define VERSION_STR_MAX_LEN 16 /* more than enough space for 99.99.99 */
 
@@ -93,7 +93,7 @@ tor_zstd_get_version_str(void)
                           ZSTD_versionNumber());
 
   return version_str;
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   return NULL;
 #endif /* defined(HAVE_ZSTD) */
 }
@@ -125,9 +125,9 @@ tor_zstd_can_use_static_apis(void)
   }
 #endif
   return (ZSTD_VERSION_NUMBER == ZSTD_versionNumber());
-#else
+#else /* !(defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD)) */
   return 0;
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) && defined(HAVE_ZSTD) */
 }
 
 /** Internal Zstandard state for incremental compression/decompression.
@@ -237,7 +237,7 @@ tor_zstd_state_size_precalc(int compress, int preset)
 #endif
     }
   }
-#endif
+#endif /* defined(ZSTD_STATIC_LINKING_ONLY) */
   return tor_zstd_state_size_precalc_fake(compress, preset);
 }
 #endif /* defined(HAVE_ZSTD) */
@@ -317,7 +317,7 @@ tor_zstd_compress_new(int compress,
   tor_free(result);
   return NULL;
   // LCOV_EXCL_STOP
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)compress;
   (void)method;
   (void)level;
@@ -454,7 +454,7 @@ tor_zstd_compress_process(tor_zstd_compress_state_t *state,
       return TOR_COMPRESS_OK;
   }
 
-#else /* !(defined(HAVE_ZSTD)) */
+#else /* !defined(HAVE_ZSTD) */
   (void)state;
   (void)out;
   (void)out_len;
@@ -527,7 +527,7 @@ tor_zstd_warn_if_version_mismatched(void)
              "For safety, we'll avoid using advanced zstd functionality.",
              header_version, runtime_version);
   }
-#endif
+#endif /* defined(HAVE_ZSTD) && defined(ENABLE_ZSTD_ADVANCED_APIS) */
 }
 
 #ifdef TOR_UNIT_TESTS
@@ -538,4 +538,4 @@ tor_zstd_set_static_apis_disabled_for_testing(int disabled)
 {
   static_apis_disable_for_testing = disabled;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/compress/compress_zstd.h b/src/lib/compress/compress_zstd.h
index 47f950b9e0..0fc71db749 100644
--- a/src/lib/compress/compress_zstd.h
+++ b/src/lib/compress/compress_zstd.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/compress/include.am b/src/lib/compress/include.am
index 75c9032bd2..60dd447d4e 100644
--- a/src/lib/compress/include.am
+++ b/src/lib/compress/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-compress-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_compress_a_SOURCES =		\
 	src/lib/compress/compress.c		\
 	src/lib/compress/compress_buf.c		\
@@ -18,9 +19,11 @@ src_lib_libtor_compress_testing_a_SOURCES = \
 src_lib_libtor_compress_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_compress_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS	+=				\
 	src/lib/compress/compress.h		\
 	src/lib/compress/compress_lzma.h	\
 	src/lib/compress/compress_none.h	\
+	src/lib/compress/compress_sys.h		\
 	src/lib/compress/compress_zlib.h	\
 	src/lib/compress/compress_zstd.h
diff --git a/src/lib/compress/lib_compress.md b/src/lib/compress/lib_compress.md
new file mode 100644
index 0000000000..c43f223fe7
--- /dev/null
+++ b/src/lib/compress/lib_compress.md
@@ -0,0 +1,6 @@
+@dir /lib/compress
+@brief lib/compress: Wraps several compression libraries
+
+Currently supported are zlib (mandatory), zstd (optional), and lzma
+(optional).
+
diff --git a/src/lib/conf/.may_include b/src/lib/conf/.may_include
new file mode 100644
index 0000000000..629e2f897d
--- /dev/null
+++ b/src/lib/conf/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
diff --git a/src/lib/conf/confdecl.h b/src/lib/conf/confdecl.h
new file mode 100644
index 0000000000..c2d3fb335d
--- /dev/null
+++ b/src/lib/conf/confdecl.h
@@ -0,0 +1,221 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file confdecl.h
+ * @brief Macros for generating a configuration struct from a list
+ *   of its individual fields.
+ *
+ * This header defines three important macros: BEGIN_CONF_STRUCT(),
+ * END_CONF_STRUCT(), and CONF_VAR().  They're meant to be used together to
+ * define a configuration structure and the means for encoding and decoding
+ * it.
+ *
+ * To use them, make a new header with a name like `MOD_options.inc`.  Start
+ * it with a BEGIN_CONF_STRUCT(), then define your variables with CONF_VAR(),
+ * then end the header with END_CONF_STRUCT(), as in:
+ *
+ *     BEGIN_CONF_STRUCT(module_options_t)
+ *     CONF_VAR(ModuleIsActive, BOOLEAN, 0, "1")
+ *     END_CONF_STRUCT(module_options_t)
+ *
+ * Once you've done that, you can use that header to define a configuration
+ * structure by saying:
+ *
+ *     typedef struct module_options_t module_options_t;
+ *     #define CONF_CONTEXT STRUCT
+ *     #include "MOD_options.inc"
+ *     #undef CONF_CONTEXT
+ *
+ * And you can define your field definition table by saying:
+ *
+ *     #define CONF_CONTEXT TABLE
+ *     #include "MOD_options.inc"
+ *     #undef CONF_CONTEXT
+ *
+ * The two above snippets will define a structure called `module_options_t`
+ * with appropriate members, and a table of config_var_t objects called
+ * `module_options_t_vars[]`.
+ *
+ * For lower-level modules, you can say <tt>\#define CONF_TABLE LL_TABLE</tt>,
+ * and get a table definition suitable for use in modules that are at a lower
+ * level than lib/confmgt.  Note that the types for these tables cannot
+ * include any extended types.
+ **/
+
+#ifndef TOR_LIB_CONF_CONFDECL_H
+#define TOR_LIB_CONF_CONFDECL_H
+
+#undef CONF_CONTEXT
+#include "lib/cc/tokpaste.h"
+#include "lib/cc/torint.h"
+
+/**
+ * Begin the definition of a configuration object called `name`.
+ **/
+#define BEGIN_CONF_STRUCT(name) \
+  PASTE(BEGIN_CONF_STRUCT__, CONF_CONTEXT)(name)
+/**
+ * End the definition of a configuration object called `name`.
+ **/
+#define END_CONF_STRUCT(name) \
+  PASTE(END_CONF_STRUCT__, CONF_CONTEXT)(name)
+/**
+ * Declare a single configuration field with name `varname`, type `vartype`,
+ * flags `varflags`, and initial value `initval`.
+ **/
+#define CONF_VAR(varname, vartype, varflags, initval)   \
+  PASTE(CONF_VAR__, CONF_CONTEXT)(varname, vartype, varflags, initval)
+
+#ifndef COCCI
+/**
+ * @defgroup STRUCT_MACROS Internal macros: struct definitions.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to STRUCT.  Don't use them directly.
+ * @{*/
+#define BEGIN_CONF_STRUCT__STRUCT(name)         \
+  struct name {                                 \
+  uint32_t magic;
+#define END_CONF_STRUCT__STRUCT(name)           \
+  };
+#define CONF_VAR__STRUCT(varname, vartype, varflags, initval)   \
+  config_decl_ ## vartype varname;
+/** @} */
+
+/**
+ * @defgroup TABLE_MACROS Internal macros: table definitions.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to TABLE.  Don't use them directly.
+ * @{*/
+#define BEGIN_CONF_STRUCT__TABLE(structname)                            \
+  /* We use this typedef so we can refer to the config type */          \
+  /* without having its name as a macro argument to CONF_VAR. */        \
+  typedef struct structname config_var_reference__obj;  \
+  static const config_var_t structname##_vars[] = {
+#define END_CONF_STRUCT__TABLE(structname)      \
+  { .member = { .name = NULL } }                \
+    };
+#define CONF_VAR__TABLE(varname, vartype, varflags, initval)    \
+  {                                                             \
+   .member =                                                    \
+   { .name = #varname,                                          \
+     .type = CONFIG_TYPE_EXTENDED,                              \
+     .type_def = &vartype ## _type_defn,                        \
+     .offset=offsetof(config_var_reference__obj, varname),      \
+   },                                                           \
+   .flags = varflags,                                           \
+   .initvalue = initval                                         \
+  },
+/**@}*/
+
+/**
+ * @defgroup LL_TABLE_MACROS Internal macros: low-level table definitions.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to LL_TABLE.  Don't use them directly.
+ * @{*/
+#define BEGIN_CONF_STRUCT__LL_TABLE(structname)                         \
+  /* We use this typedef so we can refer to the config type */          \
+  /* without having its name as a macro argument to CONF_VAR. */        \
+  typedef struct structname config_var_reference__obj;  \
+  static const config_var_t structname##_vars[] = {
+#define END_CONF_STRUCT__LL_TABLE(structname)   \
+  { .member = { .name = NULL } }                \
+    };
+#define CONF_VAR__LL_TABLE(varname, vartype, varflags, initval) \
+  {                                                             \
+   .member =                                                    \
+   { .name = #varname,                                          \
+     .type = CONFIG_TYPE_ ## vartype,                           \
+     .offset=offsetof(config_var_reference__obj, varname),      \
+   },                                                           \
+   .flags = varflags,                                           \
+   .initvalue = initval                                         \
+  },
+/**@}*/
+
+/** @defgroup STUB_TABLE_MACROS Internal macros: stub table declarations,
+ * for use when a module is disabled.
+ * Implementation helpers: the regular confdecl macros expand to these
+ * when CONF_CONTEXT is defined to LL_TABLE.  Don't use them directly.
+ * @{ */
+#define BEGIN_CONF_STRUCT__STUB_TABLE(structname)                       \
+  static const config_var_t structname##_vars[] = {
+#define END_CONF_STRUCT__STUB_TABLE(structname)   \
+  { .member = { .name = NULL } }                \
+    };
+#define CONF_VAR__STUB_TABLE(varname, vartype, varflags, initval)       \
+  {                                                             \
+   .member =                                                    \
+   { .name = #varname,                                          \
+     .type = CONFIG_TYPE_IGNORE,                                \
+     .offset = -1,                                              \
+   },                                                           \
+   .flags = CFLG_GROUP_DISABLED,                                \
+  },
+/**@}*/
+
+#endif /* !defined(COCCI) */
+
+/** Type aliases for the "commonly used" configuration types.
+ *
+ * Defining them in this way allows our CONF_VAR__STRUCT() macro to declare
+ * structure members corresponding to the configuration types.  For example,
+ * when the macro sees us declare a configuration option "foo" of type STRING,
+ * it can emit `config_decl_STRING foo;`, which is an alias for `char *foo`.
+ */
+/**@{*/
+typedef char *config_decl_STRING;
+typedef char *config_decl_FILENAME;
+/* Yes, "POSINT" is really an int, and not an unsigned int.  For
+ * historical reasons, many configuration values are restricted
+ * to the range [0,INT_MAX], and stored in signed ints.
+ */
+typedef int config_decl_POSINT;
+typedef uint64_t config_decl_UINT64;
+typedef int config_decl_INT;
+typedef int config_decl_INTERVAL;
+typedef int config_decl_MSEC_INTERVAL;
+typedef uint64_t config_decl_MEMUNIT;
+typedef double config_decl_DOUBLE;
+typedef int config_decl_BOOL;
+typedef int config_decl_AUTOBOOL;
+typedef time_t config_decl_ISOTIME;
+typedef struct smartlist_t config_decl_CSV;
+typedef int config_decl_CSV_INTERVAL;
+typedef struct config_line_t *config_decl_LINELIST;
+typedef struct config_line_t *config_decl_LINELIST_V;
+typedef struct nonexistent_struct *config_decl_LINELIST_S;
+/**@}*/
+
+struct var_type_def_t;
+
+/* Forward declarations for configuration type definitions. These are used by
+ * the CONF_VAR__TABLE macro to set the definition of each variable type
+ * correctly.
+ */
+/**@{*/
+extern const struct var_type_def_t STRING_type_defn;
+extern const struct var_type_def_t FILENAME_type_defn;
+extern const struct var_type_def_t POSINT_type_defn;
+extern const struct var_type_def_t UINT64_type_defn;
+extern const struct var_type_def_t INT_type_defn;
+extern const struct var_type_def_t INTERVAL_type_defn;
+extern const struct var_type_def_t MSEC_INTERVAL_type_defn;
+extern const struct var_type_def_t MEMUNIT_type_defn;
+extern const struct var_type_def_t DOUBLE_type_defn;
+extern const struct var_type_def_t BOOL_type_defn;
+extern const struct var_type_def_t AUTOBOOL_type_defn;
+extern const struct var_type_def_t ISOTIME_type_defn;
+extern const struct var_type_def_t CSV_type_defn;
+extern const struct var_type_def_t CSV_INTERVAL_type_defn;
+extern const struct var_type_def_t LINELIST_type_defn;
+extern const struct var_type_def_t LINELIST_V_type_defn;
+extern const struct var_type_def_t LINELIST_S_type_defn;
+extern const struct var_type_def_t IGNORE_type_defn;
+extern const struct var_type_def_t OBSOLETE_type_defn;
+/**@}*/
+
+#endif /* !defined(TOR_LIB_CONF_CONFDECL_H) */
diff --git a/src/lib/conf/config.md b/src/lib/conf/config.md
new file mode 100644
index 0000000000..7741e21f42
--- /dev/null
+++ b/src/lib/conf/config.md
@@ -0,0 +1,126 @@
+
+@page configuration Configuration options and persistent state
+
+@tableofcontents
+
+## Introduction
+
+Tor uses a shared, table-driven mechanism to handle its
+configuration (torrc) files and its state files.  Each module can
+declare a set of named fields for these files, and get notified
+whenever the configuration changes, or when the state is about to be
+flushed to disk.
+
+## Declaring options
+
+Most modules will only need to use the macros in confdecl.h to
+declare a configuration or state structure.
+
+You'll write something like this:
+
+    // my_module_config.inc
+    BEGIN_CONF_STRUCT(module_options_t)
+    CONF_VAR(FieldOne, INT, 0, "7")
+    CONF_VAR(FieldTwo, STRING, 0, NULL)
+    END_CONF_STRUCT(module_options_t)
+
+The above example will result in a structure called module_config_t
+with two fields: one an integer called FieldOne and one a string
+called FieldTwo.  The integer gets a default value of 7; the
+string's default value is NULL.
+
+After making a definition file like that, you include it twice: once
+in a header, after saying \#define CONF_CONTEXT STRUCT, and once in
+a C file, after saying \#define CONF_CONTEXT TABLE.  The first time
+defines a module_options_t structure, and the second time defines a
+table that tells the configuration manager how to use it.
+
+Using the table, you declare a `const` config_format_t, which
+associates the fields with a set of functions for validating and
+normalizing them, a list of abbreviations and deprecations, and
+other features.
+
+See confdecl.h and conftypes.h for more information. For example
+usage, see crypto_options.inc or mainloop_state.inc.
+
+## Getting notifications
+
+After using those macros, you must tell the subsystem management
+code about your module's configuration/state.
+
+If you're writing configuration code, you'll need a function that receives
+the configuration object, and acts upon it.  This function needs to be safe
+to call multiple times, since Tor will reconfigure its subsystems whenever it
+re-reads the torrc, gets a configuration change from a controller, or
+restarts in process.  This function goes in your subsystem's
+subsys_fns_t.set_options field.
+
+If you're writing state code, you'll need a function that receives
+state (subsys_fns_t.set_state), and a function that flushes the
+application state into a state object (subsys_fns_t.flush_state).
+The `set_state` function will be called once (@ref config_once_per
+"1") when Tor is starting, whereas the `flush_state` function will
+be called whenever Tor is about to save the state to disk.
+
+See subsys_fns_t for more information here, and \ref initialization
+for more information about initialization and subsystems in general.
+
+> @anchor config_once_per 1. Technically, state is set once _per startup_.
+> Remember that Tor can be stopped and started multiple times in
+> the same process.  If this happens, then your set_state() function
+> is called once every time Tor starts.
+
+## How it works
+
+The common logic used to handle configuration and state files lives
+in @refdir{lib/confmgt}.  At the highest level, a configuration
+manager object (config_mgr_t) maintains a list of each module's
+configuration objects, and a list of all their fields.  When the
+user specifies a configuration value, the manager finds out how to
+parse it, where to store it, and which configuration object is
+affected.
+
+The top-level configuration module (config.c) and state module
+(statefile.c) use config_mgr_t to create, initialize, set, compare,
+and free a "top level configuration object".  This object contains a
+list of sub-objects: one for each module that participates in the
+configuration/state system.  This top-level code then invokes the
+subsystem manager code (subsysmgr.c) to pass the corresponding
+configuration or state objects to each module that has one.
+
+Note that the top level code does not have easy access to the
+configuration objects used by the sub-modules.  This is by design.  A
+module _may_ expose some or all of its configuration or state object via
+accessor functions, if it likes, but if it does not, that object should
+be considered module-local.
+
+## Adding new types
+
+Configuration and state fields each have a "type".  These types
+specify how the fields' values are represented in C; how they are
+stored in files; and how they are encoded back and forth.
+
+There is a set of built-in types listed in conftypes.h, but
+higher-level code can define its own types.  To do so, you make an
+instance of var_type_fns_t that describes how to manage your type,
+and an instance of var_type_def_t that wraps your var_type_fns_t
+with a name and optional parameters and flags.
+
+For an example of how a higher-level type is defined, see
+ROUTERSET_type_defn in routerset.c.  Also see the typedef
+`config_decl_ROUTERSET`.  Together, these let the routerset type be
+used with the macros in confdecl.h.
+
+## Legacy configuration and state
+
+As of this writing (November 2019), most of the configuration and state is
+still handled directly in config.c and statefile.c, and stored in the
+monolithic structures or_options_t and or_state_t respectively.
+
+These top-level structures are accessed with get_options() and
+get_state(), and used throughout much of the code, at the level of
+@refdir{core} and higher.
+
+With time we hope to refactor this configuration into more
+reasonable pieces, so that they are no longer (effectively) global
+variables used throughout the code.
diff --git a/src/lib/conf/confmacros.h b/src/lib/conf/confmacros.h
new file mode 100644
index 0000000000..9f85d21740
--- /dev/null
+++ b/src/lib/conf/confmacros.h
@@ -0,0 +1,72 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file confmacros.h
+ * @brief Macro definitions for declaring configuration variables
+ **/
+
+#ifndef TOR_LIB_CONF_CONFMACROS_H
+#define TOR_LIB_CONF_CONFMACROS_H
+
+#include "orconfig.h"
+#include "lib/conf/conftesting.h"
+
+#ifndef COCCI
+/**
+ * Used to indicate the end of an array of configuration variables.
+ **/
+#define END_OF_CONFIG_VARS                                      \
+  { .member = { .name = NULL } DUMMY_CONF_TEST_MEMBERS }
+#endif /* !defined(COCCI) */
+
+/**
+ * Declare a config_var_t as a member named <b>membername</b> of the structure
+ * <b>structtype</b>, whose user-visible name is <b>varname</b>, whose
+ * type corresponds to the config_type_t member CONFIG_TYPE_<b>vartype</b>,
+ * and whose initial value is <b>intval</b>.
+ *
+ * Most modules that use this macro should wrap it in a local macro that
+ * sets structtype to the local configuration type.
+ **/
+#define CONFIG_VAR_ETYPE(structtype, varname, vartype, membername,      \
+                         varflags, initval)                             \
+  { .member =                                                           \
+    { .name = varname,                                                  \
+      .type = CONFIG_TYPE_ ## vartype,                                  \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+/**
+ * As CONFIG_VAR_ETYPE, but declares a value using an extension type whose
+ * type definition is <b>vartype</b>_type_defn.
+ **/
+#define CONFIG_VAR_DEFN(structtype, varname, vartype, membername,       \
+                        varflags, initval)                              \
+  { .member =                                                           \
+    { .name = varname,                                                \
+      .type = CONFIG_TYPE_EXTENDED,                                     \
+      .type_def = &vartype ## _type_defn,                               \
+      .offset = offsetof(structtype, membername),                       \
+    },                                                                  \
+    .flags = varflags,                                                  \
+    .initvalue = initval                                                \
+    CONF_TEST_MEMBERS(structtype, vartype, membername)                  \
+  }
+
+/**
+ * Declare an obsolete configuration variable with a given name.
+ **/
+#define CONFIG_VAR_OBSOLETE(varname)            \
+  { .member = { .name = varname, .type = CONFIG_TYPE_OBSOLETE },        \
+    .flags = CFLG_GROUP_OBSOLETE                                        \
+  }
+
+#endif /* !defined(TOR_LIB_CONF_CONFMACROS_H) */
diff --git a/src/lib/conf/conftesting.h b/src/lib/conf/conftesting.h
new file mode 100644
index 0000000000..4707c919d3
--- /dev/null
+++ b/src/lib/conf/conftesting.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftesting.h
+ * @brief Macro and type declarations for testing
+ **/
+
+#ifndef TOR_LIB_CONF_CONFTESTING_H
+#define TOR_LIB_CONF_CONFTESTING_H
+
+#include "lib/cc/torint.h"
+
+#ifndef COCCI
+#ifdef TOR_UNIT_TESTS
+#define USE_CONF_TESTING
+/**
+ * Union used when building in test mode typechecking the members of a type
+ * used with confmgt.c.  See CONF_CHECK_VAR_TYPE for a description of how
+ * it is used. */
+typedef union {
+  char **STRING;
+  char **FILENAME;
+  int *POSINT; /* yes, this is really an int, and not an unsigned int.  For
+                * historical reasons, many configuration values are restricted
+                * to the range [0,INT_MAX], and stored in signed ints.
+                */
+  uint64_t *UINT64;
+  int *INT;
+  int *INTERVAL;
+  int *MSEC_INTERVAL;
+  uint64_t *MEMUNIT;
+  double *DOUBLE;
+  int *BOOL;
+  int *AUTOBOOL;
+  time_t *ISOTIME;
+  struct smartlist_t **CSV;
+  int *CSV_INTERVAL;
+  struct config_line_t **LINELIST;
+  struct config_line_t **LINELIST_S;
+  struct config_line_t **LINELIST_V;
+  // XXXX this doesn't belong at this level of abstraction.
+  struct routerset_t **ROUTERSET;
+} confparse_dummy_values_t;
+
+/* Macros to define extra members inside config_var_t fields, and at the
+ * end of a list of them.
+ */
+/* This is a somewhat magic type-checking macro for users of confmgt.c.
+ * It initializes a union member "confparse_dummy_values_t.conftype" with
+ * the address of a static member "tp_dummy.member".   This
+ * will give a compiler warning unless the member field is of the correct
+ * type.
+ *
+ * (This warning is mandatory, because a type mismatch here violates the type
+ * compatibility constraint for simple assignment, and requires a diagnostic,
+ * according to the C spec.)
+ *
+ * For example, suppose you say:
+ *     "CONF_CHECK_VAR_TYPE(or_options_t, STRING, Address)".
+ * Then this macro will evaluate to:
+ *     { .STRING = &or_options_t_dummy.Address }
+ * And since confparse_dummy_values_t.STRING has type "char **", that
+ * expression will create a warning unless or_options_t.Address also
+ * has type "char *".
+ */
+#define CONF_CHECK_VAR_TYPE(tp, conftype, member)       \
+  { . conftype = &tp ## _dummy . member }
+#define CONF_TEST_MEMBERS(tp, conftype, member) \
+  , .var_ptr_dummy=CONF_CHECK_VAR_TYPE(tp, conftype, member)
+#define DUMMY_CONF_TEST_MEMBERS , .var_ptr_dummy={ .INT=NULL }
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  static tp tp ## _dummy
+#endif /* defined(TOR_UNIT_TESTS) */
+#endif /* !defined(COCCI) */
+
+#ifndef USE_CONF_TESTING
+#define CONF_TEST_MEMBERS(tp, conftype, member)
+/* Repeatedly declarable incomplete struct to absorb redundant semicolons */
+#define DUMMY_TYPECHECK_INSTANCE(tp)            \
+  struct tor_semicolon_eater
+#define DUMMY_CONF_TEST_MEMBERS
+
+#endif /* !defined(USE_CONF_TESTING) */
+
+#endif /* !defined(TOR_LIB_CONF_CONFTESTING_H) */
diff --git a/src/lib/conf/conftypes.h b/src/lib/conf/conftypes.h
new file mode 100644
index 0000000000..081ebf397f
--- /dev/null
+++ b/src/lib/conf/conftypes.h
@@ -0,0 +1,383 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file conftypes.h
+ * @brief Types used to specify configurable options.
+ *
+ * This header defines the types that different modules will use in order to
+ * declare their configuration and state variables, and tell the configuration
+ * management code about those variables.  From the individual module's point
+ * of view, its configuration and state are simply data structures.
+ *
+ * For defining new variable types, see var_type_def_st.h.
+ *
+ * For the code that manipulates variables defined via this module, see
+ * lib/confmgt/, especially typedvar.h and (later) structvar.h.  The
+ * configuration manager is responsible for encoding, decoding, and
+ * maintaining the configuration structures used by the various modules.
+ *
+ * STATUS NOTE: This is a work in process refactoring.  It is not yet possible
+ *   for modules to define their own variables, and much of the configuration
+ *   management code is still in src/app/config/.
+ **/
+
+#ifndef TOR_SRC_LIB_CONF_CONFTYPES_H
+#define TOR_SRC_LIB_CONF_CONFTYPES_H
+
+#include "lib/cc/torint.h"
+#ifdef TOR_UNIT_TESTS
+#include "lib/conf/conftesting.h"
+#endif
+
+#include <stddef.h>
+
+/** Enumeration of types which option values can take */
+typedef enum config_type_t {
+  CONFIG_TYPE_STRING = 0,   /**< An arbitrary string. */
+  CONFIG_TYPE_FILENAME,     /**< A filename: some prefixes get expanded. */
+  CONFIG_TYPE_POSINT,       /**< A non-negative integer less than MAX_INT */
+  CONFIG_TYPE_INT,          /**< Any integer. */
+  CONFIG_TYPE_UINT64,       /**< A value in range 0..UINT64_MAX */
+  CONFIG_TYPE_INTERVAL,     /**< A number of seconds, with optional units*/
+  CONFIG_TYPE_MSEC_INTERVAL,/**< A number of milliseconds, with optional
+                              * units */
+  CONFIG_TYPE_MEMUNIT,      /**< A number of bytes, with optional units*/
+  CONFIG_TYPE_DOUBLE,       /**< A floating-point value */
+  CONFIG_TYPE_BOOL,         /**< A boolean value, expressed as 0 or 1. */
+  CONFIG_TYPE_AUTOBOOL,     /**< A boolean+auto value, expressed 0 for false,
+                             * 1 for true, and -1 for auto  */
+  CONFIG_TYPE_ISOTIME,      /**< An ISO-formatted time relative to UTC. */
+  CONFIG_TYPE_CSV,          /**< A list of strings, separated by commas and
+                              * optional whitespace. */
+  CONFIG_TYPE_CSV_INTERVAL, /**< A list of strings, separated by commas and
+                              * optional whitespace, representing intervals in
+                              * seconds, with optional units.  We allow
+                              * multiple values here for legacy reasons, but
+                              * ignore every value after the first. */
+  CONFIG_TYPE_LINELIST,     /**< Uninterpreted config lines */
+  CONFIG_TYPE_LINELIST_S,   /**< Uninterpreted, context-sensitive config lines,
+                             * mixed with other keywords. */
+  CONFIG_TYPE_LINELIST_V,   /**< Catch-all "virtual" option to summarize
+                             * context-sensitive config lines when fetching.
+                             */
+  /** Ignored (obsolete) option. Uses no storage.
+   *
+   * Reported as "obsolete" when its type is queried.
+   */
+  CONFIG_TYPE_OBSOLETE,
+  /** Ignored option. Uses no storage.
+   *
+   * Reported as "ignored" when its type is queried. For use with options used
+   * by disabled modules.
+   **/
+  CONFIG_TYPE_IGNORE,
+
+  /**
+   * Extended type: definition appears in the <b>type_def</b> pointer
+   * of the corresponding struct_member_t.
+   *
+   * For some types, we cannot define them as particular values of this
+   * enumeration, since those types are abstractions defined at a higher level
+   * than this module.  (For example, parsing a routerset_t is higher-level
+   * than this module.)  To handle this, we use CONFIG_TYPE_EXTENDED for those
+   * types, and give a definition for them in the struct_member_t.type_def.
+   **/
+  CONFIG_TYPE_EXTENDED,
+} config_type_t;
+
+/* Forward delcaration for var_type_def_t, for extended types. */
+struct var_type_def_t;
+
+/** Structure to specify a named, typed member within a structure. */
+typedef struct struct_member_t {
+  /** Name of the field. */
+  const char *name;
+  /**
+   * Type of the field, according to the config_type_t enumeration.
+   *
+   * For any type not otherwise listed in config_type_t, this field's value
+   * should be CONFIG_TYPE_EXTENDED.  When it is, the <b>type_def</b> pointer
+   * must be set.
+   **/
+  /*
+   * NOTE: In future refactoring, we might remove this field entirely, along
+   * with its corresponding enumeration.  In that case, we will require that
+   * type_def be set in all cases. If we do, we will also need a new mechanism
+   * to enforce consistency between configuration variable types and their
+   * corresponding structures, since our current design in
+   * lib/conf/conftesting.h won't work any more.
+   */
+  config_type_t type;
+  /**
+   * Pointer to a type definition for the type of this field. Overrides
+   * <b>type</b> if it is not NULL.  Must be set when <b>type</b> is
+   * CONFIG_TYPE_EXTENDED.
+   **/
+  const struct var_type_def_t *type_def;
+  /**
+   * Offset of this field within the structure.  Compute this with
+   * offsetof(structure, fieldname).
+   **/
+  ptrdiff_t offset;
+} struct_member_t;
+
+/**
+ * Structure to describe the location and preferred value of a "magic number"
+ * field within a structure.
+ *
+ * These 'magic numbers' are 32-bit values used to tag objects to make sure
+ * that they have the correct type.
+ *
+ * If all fields in this structure are zero or 0, the magic-number check is
+ * not performed.
+ */
+typedef struct struct_magic_decl_t {
+  /** The name of the structure */
+  const char *typename;
+  /** A value used to recognize instances of this structure. */
+  uint32_t magic_val;
+  /** The location within the structure at which we expect to find
+   * <b>magic_val</b>. */
+  ptrdiff_t magic_offset;
+} struct_magic_decl_t;
+
+/**
+ * Flag to indicate that an option or type is "undumpable". An
+ * undumpable option is never saved to disk.
+ *
+ * For historical reasons its name is usually is prefixed with __.
+ **/
+#define CFLG_NODUMP    (1u<<0)
+/**
+ * Flag to indicate that an option or type is "unlisted".
+ *
+ * We don't tell the controller about unlisted options when it asks for a
+ * list of them.
+ **/
+#define CFLG_NOLIST (1u<<1)
+/**
+ * Flag to indicate that an option or type is "unsettable".
+ *
+ * An unsettable option can never be set directly by name.
+ **/
+#define CFLG_NOSET (1u<<2)
+/**
+ * Flag to indicate that an option or type does not need to be copied when
+ * copying the structure that contains it.
+ *
+ * (Usually, if an option does not need to be copied, then either it contains
+ * no data, or the data that it does contain is completely contained within
+ * another option.)
+ **/
+#define CFLG_NOCOPY (1u<<3)
+/**
+ * Flag to indicate that an option or type does not need to be compared
+ * when telling the controller about the differences between two
+ * configurations.
+ *
+ * (Usually, if an option does not need to be compared, then either it
+ * contains no data, or the data that it does contain is completely contained
+ * within another option.)
+ **/
+#define CFLG_NOCMP (1u<<4)
+/**
+ * Flag to indicate that an option or type should not be replaced when setting
+ * it.
+ *
+ * For most options, setting them replaces their old value.  For some options,
+ * however, setting them appends to their old value.
+ */
+#define CFLG_NOREPLACE    (1u<<5)
+/**
+ * Flag to indicate that an option or type cannot be changed while Tor is
+ * running.
+ **/
+#define CFLG_IMMUTABLE (1u<<6)
+/**
+ * Flag to indicate that we should warn that an option or type is obsolete
+ * whenever the user tries to use it.
+ **/
+#define CFLG_WARN_OBSOLETE (1u<<7)
+/**
+ * Flag to indicate that we should warn that an option applies only to
+ * a disabled module, whenever the user tries to use it.
+ **/
+#define CFLG_WARN_DISABLED (1u<<8)
+
+/**
+ * A group of flags that should be set on all obsolete options and types.
+ **/
+#define CFLG_GROUP_OBSOLETE \
+  (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST|\
+   CFLG_WARN_OBSOLETE)
+
+/**
+ * A group of fflags that should be set on all disabled options.
+ **/
+#define CFLG_GROUP_DISABLED \
+  (CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET|CFLG_NOLIST|\
+   CFLG_WARN_DISABLED)
+
+/** A variable allowed in the configuration file or on the command line. */
+typedef struct config_var_t {
+  struct_member_t member; /** A struct member corresponding to this
+                           * variable. */
+  const char *initvalue; /**< String (or null) describing initial value. */
+  uint32_t flags; /**< One or more flags describing special handling for this
+                   * variable */
+#ifdef TOR_UNIT_TESTS
+  /** Used for compiler-magic to typecheck the corresponding field in the
+   * corresponding struct. Only used in unit test mode, at compile-time. */
+  confparse_dummy_values_t var_ptr_dummy;
+#endif
+} config_var_t;
+
+/**
+ * An abbreviation or alias for a configuration option.
+ **/
+typedef struct config_abbrev_t {
+  /** The option name as abbreviated.  Not case-sensitive. */
+  const char *abbreviated;
+  /** The full name of the option. Not case-sensitive. */
+  const char *full;
+  /** True if this abbreviation should only be allowed on the command line. */
+  int commandline_only;
+  /** True if we should warn whenever this abbreviation is used. */
+  int warn;
+} config_abbrev_t;
+
+/**
+ * A note that a configuration option is deprecated, with an explanation why.
+ */
+typedef struct config_deprecation_t {
+  /** The option that is deprecated. */
+  const char *name;
+  /** A user-facing string explaining why the option is deprecated. */
+  const char *why_deprecated;
+} config_deprecation_t;
+
+#ifndef COCCI
+/**
+ * Handy macro for declaring "In the config file or on the command line, you
+ * can abbreviate <b>tok</b>s as <b>tok</b>". Used inside an array of
+ * config_abbrev_t.
+ *
+ * For example, to declare "NumCpu" as an abbreviation for "NumCPUs",
+ * you can say PLURAL(NumCpu).
+ **/
+#define PLURAL(tok) { (#tok), (#tok "s"), 0, 0 }
+#endif /* !defined(COCCI) */
+
+/**
+ * Validation function: verify whether a configuation object is well-formed
+ * and consistent.
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*validate_fn_t)(const void *value, char **msg_out);
+/**
+ * Validation function: verify whether a configuration object (`value`) is an
+ * allowable value given the previous configuration value (`old_value`).
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*check_transition_fn_t)(const void *old_value, const void *value,
+                                     char **msg_out);
+/**
+ * Validation function: normalize members of `value`, and compute derived
+ * members.
+ *
+ * This function is called before any other validation of `value`, and must
+ * not assume that validate_fn or check_transition_fn has passed.
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*pre_normalize_fn_t)(void *value, char **msg_out);
+/**
+ * Validation function: normalize members of `value`, and compute derived
+ * members.
+ *
+ * This function is called after validation of `value`, and may
+ * assume that validate_fn or check_transition_fn has passed.
+ *
+ * On success, return 0.  On failure, set <b>msg_out</b> to a newly allocated
+ * string containing an error message, and return -1. */
+typedef int (*post_normalize_fn_t)(void *value, char **msg_out);
+
+/**
+ * Legacy function to validate whether a given configuration is
+ * well-formed and consistent.
+ *
+ * The configuration to validate is passed as <b>newval</b>. The previous
+ * configuration, if any, is provided in <b>oldval</b>.
+ *
+ * This API is deprecated, since it mixes the responsibilities of
+ * pre_normalize_fn_t, post_normalize_fn_t, validate_fn_t, and
+ * check_transition_fn_t.  No new instances of this function type should
+ * be written.
+ *
+ * On success, return 0.  On failure, set *<b>msg_out</b> to a newly allocated
+ * error message, and return -1.
+ */
+typedef int (*legacy_validate_fn_t)(const void *oldval,
+                                    void *newval,
+                                    char **msg_out);
+
+struct config_mgr_t;
+
+/**
+ * Callback to clear all non-managed fields of a configuration object.
+ *
+ * <b>obj</b> is the configuration object whose non-managed fields should be
+ * cleared.
+ *
+ * (Regular fields get cleared by config_reset(), but you might have fields
+ * in the object that do not correspond to configuration variables.  If those
+ * fields need to be cleared or freed, this is where to do it.)
+ */
+typedef void (*clear_cfg_fn_t)(const struct config_mgr_t *mgr, void *obj);
+
+/** Information on the keys, value types, key-to-struct-member mappings,
+ * variable descriptions, validation functions, and abbreviations for a
+ * configuration or storage format. */
+typedef struct config_format_t {
+  size_t size; /**< Size of the struct that everything gets parsed into. */
+  struct_magic_decl_t magic; /**< Magic number info for this struct. */
+  const config_abbrev_t *abbrevs; /**< List of abbreviations that we expand
+                             * when parsing this format. */
+  const config_deprecation_t *deprecations; /** List of deprecated options */
+  const config_var_t *vars; /**< List of variables we recognize, their default
+                             * values, and where we stick them in the
+                             * structure. */
+
+  /** Early-stage normalization callback. Invoked by config_validate(). */
+  pre_normalize_fn_t pre_normalize_fn;
+  /** Configuration validation function. Invoked by config_validate(). */
+  validate_fn_t validate_fn;
+    /** Legacy validation function. Invoked by config_validate(). */
+  legacy_validate_fn_t legacy_validate_fn;
+  /** Transition checking function. Invoked by config_validate(). */
+  check_transition_fn_t check_transition_fn;
+  /** Late-stage normalization callback. Invoked by config_validate(). */
+  post_normalize_fn_t post_normalize_fn;
+
+  clear_cfg_fn_t clear_fn; /**< Function to clear the configuration. */
+  /** If present, extra denotes a LINELIST variable for unrecognized
+   * lines.  Otherwise, unrecognized lines are an error. */
+  const struct_member_t *extra;
+  /**
+   * If true, this format describes a top-level configuration, with
+   * a suite containing multiple sub-configuration objects.
+   */
+  bool has_config_suite;
+  /** The position of a config_suite_t pointer within the toplevel object.
+   * Ignored unless have_config_suite is true.
+   */
+  ptrdiff_t config_suite_offset;
+} config_format_t;
+
+#endif /* !defined(TOR_SRC_LIB_CONF_CONFTYPES_H) */
diff --git a/src/lib/conf/include.am b/src/lib/conf/include.am
new file mode 100644
index 0000000000..cb0b83fa64
--- /dev/null
+++ b/src/lib/conf/include.am
@@ -0,0 +1,7 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/conf/confdecl.h              \
+	src/lib/conf/conftesting.h              \
+	src/lib/conf/conftypes.h                \
+	src/lib/conf/confmacros.h
diff --git a/src/lib/conf/lib_conf.md b/src/lib/conf/lib_conf.md
new file mode 100644
index 0000000000..60dd04e99e
--- /dev/null
+++ b/src/lib/conf/lib_conf.md
@@ -0,0 +1,3 @@
+@dir /lib/conf
+@brief lib/conf: Types and macros for declaring configuration options.
+
diff --git a/src/lib/confmgt/.may_include b/src/lib/confmgt/.may_include
new file mode 100644
index 0000000000..5ff949f103
--- /dev/null
+++ b/src/lib/confmgt/.may_include
@@ -0,0 +1,12 @@
+orconfig.h
+lib/cc/*.h
+lib/conf/*.h
+lib/confmgt/*.h
+lib/container/*.h
+lib/encoding/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/string/*.h
+lib/testsupport/*.h
+ext/*.h
diff --git a/src/lib/confmgt/confmgt.c b/src/lib/confmgt/confmgt.c
new file mode 100644
index 0000000000..bf2764160e
--- /dev/null
+++ b/src/lib/confmgt/confmgt.c
@@ -0,0 +1,1406 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confmgt.c
+ *
+ * \brief Back-end for parsing and generating key-value files, used to
+ *   implement the torrc file format and the state file.
+ *
+ * This module is used by config.c to parse and encode torrc
+ * configuration files, and by statefile.c to parse and encode the
+ * $DATADIR/state file.
+ *
+ * To use this module, its callers provide an instance of
+ * config_format_t to describe the mappings from a set of configuration
+ * options to a number of fields in a C structure.  With this mapping,
+ * the functions here can convert back and forth between the C structure
+ * specified, and a linked list of key-value pairs.
+ */
+
+#define CONFMGT_PRIVATE
+#include "orconfig.h"
+#include "lib/confmgt/confmgt.h"
+
+#include "lib/confmgt/structvar.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "ext/siphash.h"
+
+/**
+ * A managed_var_t is an internal wrapper around a config_var_t in
+ * a config_format_t structure.  It is used by config_mgr_t to
+ * keep track of which option goes with which structure. */
+typedef struct managed_var_t {
+  /**
+   * A pointer to the config_var_t for this option.
+   */
+  const config_var_t *cvar;
+  /**
+   * The index of the object in which this option is stored.  It is
+   * IDX_TOPLEVEL to indicate that the object is the top-level object.
+   **/
+  int object_idx;
+} managed_var_t;
+
+static void config_reset(const config_mgr_t *fmt, void *options,
+                         const managed_var_t *var, int use_defaults);
+static void config_mgr_register_fmt(config_mgr_t *mgr,
+                                    const config_format_t *fmt,
+                                    int object_idx);
+
+/** Release all storage held in a managed_var_t. */
+static void
+managed_var_free_(managed_var_t *mv)
+{
+  if (!mv)
+    return;
+  tor_free(mv);
+}
+#define managed_var_free(mv) \
+  FREE_AND_NULL(managed_var_t, managed_var_free_, (mv))
+
+struct config_suite_t {
+  /** A list of configuration objects managed by a given configuration
+   * manager. They are stored in the same order as the config_format_t
+   * objects in the manager's list of subformats. */
+  smartlist_t *configs;
+};
+
+/**
+ * Allocate a new empty config_suite_t.
+ **/
+static config_suite_t *
+config_suite_new(void)
+{
+  config_suite_t *suite = tor_malloc_zero(sizeof(config_suite_t));
+  suite->configs = smartlist_new();
+  return suite;
+}
+
+/** Release all storage held by a config_suite_t.  (Does not free
+ * any configuration objects it holds; the caller must do that first.) */
+static void
+config_suite_free_(config_suite_t *suite)
+{
+  if (!suite)
+    return;
+  smartlist_free(suite->configs);
+  tor_free(suite);
+}
+
+#define config_suite_free(suite) \
+  FREE_AND_NULL(config_suite_t, config_suite_free_, (suite))
+
+struct config_mgr_t {
+  /** The 'top-level' configuration format.  This one is used for legacy
+   * options that have not yet been assigned to different sub-modules.
+   *
+   * (NOTE: for now, this is the only config_format_t that a config_mgr_t
+   * contains.  A subsequent commit will add more. XXXX)
+   */
+  const config_format_t *toplevel;
+  /**
+   * List of second-level configuration format objects that this manager
+   * also knows about.
+   */
+  smartlist_t *subconfigs;
+  /** A smartlist of managed_var_t objects for all configuration formats. */
+  smartlist_t *all_vars;
+  /** A smartlist of config_abbrev_t objects for all configuration
+   * formats. These objects are used to track synonyms and abbreviations for
+   * different configuration options. */
+  smartlist_t *all_abbrevs;
+  /** A smartlist of config_deprecation_t for all configuration formats. */
+  smartlist_t *all_deprecations;
+  /** True if this manager has been frozen and cannot have any more formats
+   * added to it. A manager must be frozen before it can be used to construct
+   * or manipulate objects. */
+  bool frozen;
+  /** A replacement for the magic number of the toplevel object. We override
+   * that number to make it unique for this particular config_mgr_t, so that
+   * an object constructed with one mgr can't be used with another, even if
+   * those managers' contents are equal.
+   */
+  struct_magic_decl_t toplevel_magic;
+};
+
+#define IDX_TOPLEVEL (-1)
+
+/** Create a new config_mgr_t to manage a set of configuration objects to be
+ * wrapped under <b>toplevel_fmt</b>. */
+config_mgr_t *
+config_mgr_new(const config_format_t *toplevel_fmt)
+{
+  config_mgr_t *mgr = tor_malloc_zero(sizeof(config_mgr_t));
+  mgr->subconfigs = smartlist_new();
+  mgr->all_vars = smartlist_new();
+  mgr->all_abbrevs = smartlist_new();
+  mgr->all_deprecations = smartlist_new();
+
+  config_mgr_register_fmt(mgr, toplevel_fmt, IDX_TOPLEVEL);
+  mgr->toplevel = toplevel_fmt;
+
+  return mgr;
+}
+
+/** Add a config_format_t to a manager, with a specified (unique) index. */
+static void
+config_mgr_register_fmt(config_mgr_t *mgr,
+                        const config_format_t *fmt,
+                        int object_idx)
+{
+  int i;
+
+  tor_assertf(!mgr->frozen,
+              "Tried to add a format to a configuration manager after "
+              "it had been frozen.");
+
+  if (object_idx != IDX_TOPLEVEL) {
+    tor_assertf(! fmt->has_config_suite,
+          "Tried to register a toplevel format in a non-toplevel position");
+  }
+  if (fmt->config_suite_offset) {
+    tor_assertf(fmt->has_config_suite,
+                "config_suite_offset was set, but has_config_suite was not.");
+  }
+
+  tor_assertf(fmt != mgr->toplevel &&
+              ! smartlist_contains(mgr->subconfigs, fmt),
+              "Tried to register an already-registered format.");
+
+  /* register variables */
+  for (i = 0; fmt->vars[i].member.name; ++i) {
+    managed_var_t *mv = tor_malloc_zero(sizeof(managed_var_t));
+    mv->cvar = &fmt->vars[i];
+    mv->object_idx = object_idx;
+    smartlist_add(mgr->all_vars, mv);
+  }
+
+  /* register abbrevs */
+  if (fmt->abbrevs) {
+    for (i = 0; fmt->abbrevs[i].abbreviated; ++i) {
+      smartlist_add(mgr->all_abbrevs, (void*)&fmt->abbrevs[i]);
+    }
+  }
+
+  /* register deprecations. */
+  if (fmt->deprecations) {
+    const config_deprecation_t *d;
+    for (d = fmt->deprecations; d->name; ++d) {
+      smartlist_add(mgr->all_deprecations, (void*)d);
+    }
+  }
+}
+
+/**
+ * Add a new format to this configuration object.  Asserts on failure.
+ *
+ * Returns an internal "index" value used to identify this format within
+ * all of those formats contained in <b>mgr</b>.  This index value
+ * should not generally be used outside of this module.
+ **/
+int
+config_mgr_add_format(config_mgr_t *mgr,
+                      const config_format_t *fmt)
+{
+  tor_assert(mgr);
+  int idx = smartlist_len(mgr->subconfigs);
+  config_mgr_register_fmt(mgr, fmt, idx);
+  smartlist_add(mgr->subconfigs, (void *)fmt);
+  return idx;
+}
+
+/** Return a pointer to the config_suite_t * pointer inside a
+ * configuration object; returns NULL if there is no such member. */
+static inline config_suite_t **
+config_mgr_get_suite_ptr(const config_mgr_t *mgr, void *toplevel)
+{
+  if (! mgr->toplevel->has_config_suite)
+    return NULL;
+  return STRUCT_VAR_P(toplevel, mgr->toplevel->config_suite_offset);
+}
+
+/**
+ * Return a pointer to the configuration object within <b>toplevel</b> whose
+ * index is <b>idx</b>.
+ *
+ * NOTE: XXXX Eventually, there will be multiple objects supported within the
+ * toplevel object.  For example, the or_options_t will contain pointers
+ * to configuration objects for other modules.  This function gets
+ * the sub-object for a particular module.
+ */
+void *
+config_mgr_get_obj_mutable(const config_mgr_t *mgr, void *toplevel, int idx)
+{
+  tor_assert(mgr);
+  tor_assert(toplevel);
+  if (idx == IDX_TOPLEVEL)
+    return toplevel;
+
+  tor_assertf(idx >= 0 && idx < smartlist_len(mgr->subconfigs),
+              "Index %d is out of range.", idx);
+  config_suite_t **suite = config_mgr_get_suite_ptr(mgr, toplevel);
+  tor_assert(suite);
+  tor_assert(smartlist_len(mgr->subconfigs) ==
+             smartlist_len((*suite)->configs));
+
+  return smartlist_get((*suite)->configs, idx);
+}
+
+/** As config_mgr_get_obj_mutable(), but return a const pointer. */
+const void *
+config_mgr_get_obj(const config_mgr_t *mgr, const void *toplevel, int idx)
+{
+  return config_mgr_get_obj_mutable(mgr, (void*)toplevel, idx);
+}
+
+/** Sorting helper for smartlist of managed_var_t */
+static int
+managed_var_cmp(const void **a, const void **b)
+{
+  const managed_var_t *mv1 = *(const managed_var_t**)a;
+  const managed_var_t *mv2 = *(const managed_var_t**)b;
+
+  return strcasecmp(mv1->cvar->member.name, mv2->cvar->member.name);
+}
+
+/**
+ * Mark a configuration manager as "frozen", so that no more formats can be
+ * added, and so that it can be used for manipulating configuration objects.
+ **/
+void
+config_mgr_freeze(config_mgr_t *mgr)
+{
+  static uint64_t mgr_count = 0;
+
+  smartlist_sort(mgr->all_vars, managed_var_cmp);
+  memcpy(&mgr->toplevel_magic, &mgr->toplevel->magic,
+         sizeof(struct_magic_decl_t));
+  uint64_t magic_input[3] = { mgr->toplevel_magic.magic_val,
+                              (uint64_t) (uintptr_t) mgr,
+                              ++mgr_count };
+  mgr->toplevel_magic.magic_val =
+    (uint32_t)siphash24g(magic_input, sizeof(magic_input));
+  mgr->frozen = true;
+}
+
+/** Release all storage held in <b>mgr</b> */
+void
+config_mgr_free_(config_mgr_t *mgr)
+{
+  if (!mgr)
+    return;
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv, managed_var_free(mv));
+  smartlist_free(mgr->all_vars);
+  smartlist_free(mgr->all_abbrevs);
+  smartlist_free(mgr->all_deprecations);
+  smartlist_free(mgr->subconfigs);
+  memset(mgr, 0, sizeof(*mgr));
+  tor_free(mgr);
+}
+
+/** Return a new smartlist_t containing a config_var_t for every variable that
+ * <b>mgr</b> knows about.  The elements of this smartlist do not need
+ * to be freed; they have the same lifespan as <b>mgr</b>. */
+smartlist_t *
+config_mgr_list_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_vars, managed_var_t *, mv,
+                    smartlist_add(result, (void*) mv->cvar));
+  return result;
+}
+
+/** Return a new smartlist_t containing the names of all deprecated variables.
+ * The elements of this smartlist do not need to be freed; they have the same
+ * lifespan as <b>mgr</b>.
+ */
+smartlist_t *
+config_mgr_list_deprecated_vars(const config_mgr_t *mgr)
+{
+  smartlist_t *result = smartlist_new();
+  tor_assert(mgr);
+  SMARTLIST_FOREACH(mgr->all_deprecations, config_deprecation_t *, d,
+                    smartlist_add(result, (char*)d->name));
+  return result;
+}
+
+/**
+ * Check the magic number on <b>object</b> to make sure it's a valid toplevel
+ * object, created with <b>mgr</b>.  Exit with an assertion if it isn't.
+ **/
+void
+config_check_toplevel_magic(const config_mgr_t *mgr,
+                            const void *object)
+{
+  struct_check_magic(object, &mgr->toplevel_magic);
+}
+
+/** Assert that the magic fields in <b>options</b> and its subsidiary
+ * objects are all okay. */
+static void
+config_mgr_assert_magic_ok(const config_mgr_t *mgr,
+                           const void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+  tor_assert(mgr->frozen);
+  struct_check_magic(options, &mgr->toplevel_magic);
+
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, (void*)options);
+  if (suitep == NULL) {
+    tor_assert(smartlist_len(mgr->subconfigs) == 0);
+    return;
+  }
+
+  tor_assert(smartlist_len((*suitep)->configs) ==
+             smartlist_len(mgr->subconfigs));
+  SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+    void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+    tor_assert(obj);
+    struct_check_magic(obj, &fmt->magic);
+  } SMARTLIST_FOREACH_END(fmt);
+}
+
+/** Macro: assert that <b>cfg</b> has the right magic field for
+ * <b>mgr</b>. */
+#define CONFIG_CHECK(mgr, cfg) STMT_BEGIN                               \
+    config_mgr_assert_magic_ok((mgr), (cfg));                           \
+  STMT_END
+
+/** Allocate an empty configuration object of a given format type. */
+void *
+config_new(const config_mgr_t *mgr)
+{
+  tor_assert(mgr->frozen);
+  void *opts = tor_malloc_zero(mgr->toplevel->size);
+  struct_set_magic(opts, &mgr->toplevel_magic);
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, opts);
+  if (suitep) {
+    *suitep = config_suite_new();
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = tor_malloc_zero(fmt->size);
+      struct_set_magic(obj, &fmt->magic);
+      smartlist_add((*suitep)->configs, obj);
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+  CONFIG_CHECK(mgr, opts);
+  return opts;
+}
+
+/*
+ * Functions to parse config options
+ */
+
+/** If <b>option</b> is an official abbreviation for a longer option,
+ * return the longer option.  Otherwise return <b>option</b>.
+ * If <b>command_line</b> is set, apply all abbreviations.  Otherwise, only
+ * apply abbreviations that work for the config file and the command line.
+ * If <b>warn_obsolete</b> is set, warn about deprecated names. */
+const char *
+config_expand_abbrev(const config_mgr_t *mgr, const char *option,
+                     int command_line, int warn_obsolete)
+{
+  SMARTLIST_FOREACH_BEGIN(mgr->all_abbrevs, const config_abbrev_t *, abbrev) {
+    /* Abbreviations are case insensitive. */
+    if (!strcasecmp(option, abbrev->abbreviated) &&
+        (command_line || !abbrev->commandline_only)) {
+      if (warn_obsolete && abbrev->warn) {
+        log_warn(LD_CONFIG,
+                 "The configuration option '%s' is deprecated; "
+                 "use '%s' instead.",
+                 abbrev->abbreviated,
+                 abbrev->full);
+      }
+      /* Keep going through the list in case we want to rewrite it more.
+       * (We could imagine recursing here, but I don't want to get the
+       * user into an infinite loop if we craft our list wrong.) */
+      option = abbrev->full;
+    }
+  } SMARTLIST_FOREACH_END(abbrev);
+  return option;
+}
+
+/** If <b>key</b> is a deprecated configuration option, return the message
+ * explaining why it is deprecated (which may be an empty string). Return NULL
+ * if it is not deprecated. The <b>key</b> field must be fully expanded. */
+const char *
+config_find_deprecation(const config_mgr_t *mgr, const char *key)
+{
+  if (BUG(mgr == NULL) || BUG(key == NULL))
+    return NULL; // LCOV_EXCL_LINE
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_deprecations, const config_deprecation_t *,
+                          d) {
+    if (!strcasecmp(d->name, key)) {
+      return d->why_deprecated ? d->why_deprecated : "";
+    }
+  } SMARTLIST_FOREACH_END(d);
+  return NULL;
+}
+
+/**
+ * Find the managed_var_t object for a variable whose name is <b>name</b>
+ * according to <b>mgr</b>.   Return that object, or NULL if none exists.
+ *
+ * If <b>allow_truncated</b> is true, then accept any variable whose
+ * name begins with <b>name</b>.
+ *
+ * If <b>idx_out</b> is not NULL, set *<b>idx_out</b> to the position of
+ * that variable within mgr-&gt;all_vars, or to -1 if the variable is
+ * not found.
+ */
+static const managed_var_t *
+config_mgr_find_var(const config_mgr_t *mgr,
+                    const char *key,
+                    bool allow_truncated, int *idx_out)
+{
+  const size_t keylen = strlen(key);
+  if (idx_out)
+    *idx_out = -1;
+
+  if (!keylen)
+    return NULL; /* if they say "--" on the command line, it's not an option */
+
+  /* First, check for an exact (case-insensitive) match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strcasecmp(mv->cvar->member.name, key)) {
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (!allow_truncated)
+    return NULL;
+
+  /* If none, check for an abbreviated match */
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!strncasecmp(key, mv->cvar->member.name, keylen)) {
+      log_warn(LD_CONFIG, "The abbreviation '%s' is deprecated. "
+               "Please use '%s' instead",
+               key, mv->cvar->member.name);
+      if (idx_out)
+        *idx_out = mv_sl_idx;
+      return mv;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  /* Okay, unrecognized option */
+  return NULL;
+}
+
+/**
+ * If <b>key</b> is a name or an abbreviation configuration option, return
+ * the corresponding canonical name for it.  Warn if the abbreviation is
+ * non-standard.  Return NULL if the option does not exist.
+ */
+const char *
+config_find_option_name(const config_mgr_t *mgr, const char *key)
+{
+  key = config_expand_abbrev(mgr, key, 0, 0);
+  const managed_var_t *mv = config_mgr_find_var(mgr, key, true, NULL);
+  if (mv)
+    return mv->cvar->member.name;
+  else
+    return NULL;
+}
+
+/** Return the number of option entries in <b>fmt</b>. */
+static int
+config_count_options(const config_mgr_t *mgr)
+{
+  return smartlist_len(mgr->all_vars);
+}
+
+/**
+ * Return true iff at least one bit from <b>flag</b> is set on <b>var</b>,
+ * either in <b>var</b>'s flags, or on the flags of its type.
+ **/
+static bool
+config_var_has_flag(const config_var_t *var, uint32_t flag)
+{
+  uint32_t have_flags = var->flags | struct_var_get_flags(&var->member);
+
+  return (have_flags & flag) != 0;
+}
+
+/**
+ * Return true if assigning a value to <b>var</b> replaces the previous
+ * value.  Return false if assigning a value to <b>var</b> appends
+ * to the previous value.
+ **/
+static bool
+config_var_is_replaced_on_set(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOREPLACE);
+}
+
+/**
+ * Return true iff <b>var</b> may be assigned by name (e.g., via the
+ * CLI, the configuration files, or the controller API).
+ **/
+bool
+config_var_is_settable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOSET);
+}
+
+/**
+ * Return true iff the controller is allowed to fetch the value of
+ * <b>var</b>.
+ **/
+static bool
+config_var_is_gettable(const config_var_t *var)
+{
+  /* Arguably, invisible or obsolete options should not be gettable.  However,
+   * they have been gettable for a long time, and making them ungettable could
+   * have compatibility effects.  For now, let's leave them alone.
+   */
+
+  // return ! config_var_has_flag(var, CVFLAG_OBSOLETE|CFGLAGS_INVISIBLE);
+  (void)var;
+  return true;
+}
+
+/**
+ * Return true iff we need to check <b>var</b> for changes when we are
+ * comparing config options for changes.
+ *
+ * A false result might mean that the variable is a derived variable, and that
+ * comparing the variable it derives from compares this one too-- or it might
+ * mean that there is no data to compare.
+ **/
+static bool
+config_var_should_list_changes(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCMP);
+}
+
+/**
+ * Return true iff we need to copy the data for <b>var</b> when we are
+ * copying a config option.
+ *
+ * A false option might mean that the variable is a derived variable, and that
+ * copying the variable it derives from copies it-- or it might mean that
+ * there is no data to copy.
+ **/
+static bool
+config_var_needs_copy(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOCOPY);
+}
+
+/**
+ * Return true iff variable <b>var</b> should appear on list of variable
+ * names given to the controller or the CLI.
+ *
+ * (Note that this option is imperfectly obeyed. The
+ * --list-torrc-options command looks at the "settable" flag, whereas
+ * "GETINFO config/defaults" and "list_deprecated_*()" do not filter
+ * their results. It would be good for consistency to try to converge
+ * these behaviors in the future.)
+ **/
+bool
+config_var_is_listable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NOLIST);
+}
+
+/**
+ * Return true iff variable <b>var</b> should be written out when we
+ * are writing our configuration to disk, to a controller, or via the
+ * --dump-config command.
+ *
+ * This option may be set because a variable is hidden, or because it is
+ * derived from another variable which will already be written out.
+ **/
+static bool
+config_var_is_dumpable(const config_var_t *var)
+{
+  return ! config_var_has_flag(var, CFLG_NODUMP);
+}
+
+/*
+ * Functions to assign config options.
+ */
+
+/** <b>c</b>-\>key is known to be a real key. Update <b>options</b>
+ * with <b>c</b>-\>value and return 0, or return -1 if bad value.
+ *
+ * Called from config_assign_line() and option_reset().
+ */
+static int
+config_assign_value(const config_mgr_t *mgr, void *options,
+                    config_line_t *c, char **msg)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, c->key, true, NULL);
+  tor_assert(var);
+  tor_assert(!strcmp(c->key, var->cvar->member.name));
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+
+  if (config_var_has_flag(var->cvar, CFLG_WARN_OBSOLETE)) {
+    log_warn(LD_GENERAL, "Skipping obsolete configuration option \"%s\".",
+             var->cvar->member.name);
+  } else if (config_var_has_flag(var->cvar, CFLG_WARN_DISABLED)) {
+    log_warn(LD_GENERAL, "This copy of Tor was built without support for "
+             "the option \"%s\". Skipping.", var->cvar->member.name);
+  }
+
+  return struct_var_kvassign(object, c, msg, &var->cvar->member);
+}
+
+/** Mark every linelist in <b>options</b> "fragile", so that fresh assignments
+ * to it will replace old ones. */
+static void
+config_mark_lists_fragile(const config_mgr_t *mgr, void *options)
+{
+  tor_assert(mgr);
+  tor_assert(options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    void *object = config_mgr_get_obj_mutable(mgr, options, mv->object_idx);
+    struct_var_mark_fragile(object, &mv->cvar->member);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/**
+ * Log a warning that declaring that the option called <b>what</b>
+ * is deprecated because of the reason in <b>why</b>.
+ *
+ * (Both arguments must be non-NULL.)
+ **/
+void
+warn_deprecated_option(const char *what, const char *why)
+{
+  const char *space = (why && strlen(why)) ? " " : "";
+  log_warn(LD_CONFIG, "The %s option is deprecated, and will most likely "
+           "be removed in a future version of Tor.%s%s (If you think this is "
+           "a mistake, please let us know!)",
+           what, space, why);
+}
+
+/** If <b>c</b> is a syntactically valid configuration line, update
+ * <b>options</b> with its value and return 0.  Otherwise return -1 for bad
+ * key, -2 for bad value.
+ *
+ * If <b>clear_first</b> is set, clear the value first. Then if
+ * <b>use_defaults</b> is set, set the value to the default.
+ *
+ * Called from config_assign().
+ */
+static int
+config_assign_line(const config_mgr_t *mgr, void *options,
+                   config_line_t *c, unsigned flags,
+                   bitarray_t *options_seen, char **msg)
+{
+  const unsigned use_defaults = flags & CAL_USE_DEFAULTS;
+  const unsigned clear_first = flags & CAL_CLEAR_FIRST;
+  const unsigned warn_deprecations = flags & CAL_WARN_DEPRECATIONS;
+  const managed_var_t *mvar;
+
+  CONFIG_CHECK(mgr, options);
+
+  int var_index = -1;
+  mvar = config_mgr_find_var(mgr, c->key, true, &var_index);
+  if (!mvar) {
+    const config_format_t *fmt = mgr->toplevel;
+    if (fmt->extra) {
+      void *lvalue = STRUCT_VAR_P(options, fmt->extra->offset);
+      log_info(LD_CONFIG,
+               "Found unrecognized option '%s'; saving it.", c->key);
+      config_line_append((config_line_t**)lvalue, c->key, c->value);
+      return 0;
+    } else {
+      tor_asprintf(msg,
+                "Unknown option '%s'.  Failing.", c->key);
+      return -1;
+    }
+  }
+
+  const config_var_t *cvar = mvar->cvar;
+  tor_assert(cvar);
+
+  /* Put keyword into canonical case. */
+  if (strcmp(cvar->member.name, c->key)) {
+    tor_free(c->key);
+    c->key = tor_strdup(cvar->member.name);
+  }
+
+  const char *deprecation_msg;
+  if (warn_deprecations &&
+      (deprecation_msg = config_find_deprecation(mgr, cvar->member.name))) {
+    warn_deprecated_option(cvar->member.name, deprecation_msg);
+  }
+
+  if (!strlen(c->value)) {
+    /* reset or clear it, then return */
+    if (!clear_first) {
+      if (! config_var_is_replaced_on_set(cvar) &&
+          c->command != CONFIG_LINE_CLEAR) {
+        /* We got an empty linelist from the torrc or command line.
+           As a special case, call this an error. Warn and ignore. */
+        log_warn(LD_CONFIG,
+                 "Linelist option '%s' has no value. Skipping.", c->key);
+      } else { /* not already cleared */
+        config_reset(mgr, options, mvar, use_defaults);
+      }
+    }
+    return 0;
+  } else if (c->command == CONFIG_LINE_CLEAR && !clear_first) {
+    // This block is unreachable, since a CLEAR line always has an
+    // empty value, and so will trigger be handled by the previous
+    // "if (!strlen(c->value))" block.
+
+    // LCOV_EXCL_START
+    tor_assert_nonfatal_unreached();
+    config_reset(mgr, options, mvar, use_defaults);
+    // LCOV_EXCL_STOP
+  }
+
+  if (options_seen && config_var_is_replaced_on_set(cvar)) {
+    /* We're tracking which options we've seen, and this option is not
+     * supposed to occur more than once. */
+    tor_assert(var_index >= 0);
+    if (bitarray_is_set(options_seen, var_index)) {
+      log_warn(LD_CONFIG, "Option '%s' used more than once; all but the last "
+               "value will be ignored.", cvar->member.name);
+    }
+    bitarray_set(options_seen, var_index);
+  }
+
+  if (config_assign_value(mgr, options, c, msg) < 0)
+    return -2;
+  return 0;
+}
+
+/** Restore the option named <b>key</b> in options to its default value.
+ * Called from config_assign(). */
+STATIC void
+config_reset_line(const config_mgr_t *mgr, void *options,
+                  const char *key, int use_defaults)
+{
+  const managed_var_t *var;
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var)
+    return; /* give error on next pass. */
+
+  config_reset(mgr, options, var, use_defaults);
+}
+
+/** Return true iff value needs to be quoted and escaped to be used in
+ * a configuration file. */
+static int
+config_value_needs_escape(const char *value)
+{
+  if (*value == '\"')
+    return 1;
+  while (*value) {
+    switch (*value)
+    {
+    case '\r':
+    case '\n':
+    case '#':
+      /* Note: quotes and backspaces need special handling when we are using
+       * quotes, not otherwise, so they don't trigger escaping on their
+       * own. */
+      return 1;
+    default:
+      if (!TOR_ISPRINT(*value))
+        return 1;
+    }
+    ++value;
+  }
+  return 0;
+}
+
+/** Return newly allocated line or lines corresponding to <b>key</b> in the
+ * configuration <b>options</b>.  If <b>escape_val</b> is true and a
+ * value needs to be quoted before it's put in a config file, quote and
+ * escape that value. Return NULL if no such key exists. */
+config_line_t *
+config_get_assigned_option(const config_mgr_t *mgr, const void *options,
+                           const char *key, int escape_val)
+{
+  const managed_var_t *var;
+  config_line_t *result;
+
+  tor_assert(options && key);
+
+  CONFIG_CHECK(mgr, options);
+
+  var = config_mgr_find_var(mgr, key, true, NULL);
+  if (!var) {
+    log_warn(LD_CONFIG, "Unknown option '%s'.  Failing.", key);
+    return NULL;
+  }
+  if (! config_var_is_gettable(var->cvar)) {
+    log_warn(LD_CONFIG, "Option '%s' is obsolete or unfetchable. Failing.",
+             key);
+    return NULL;
+  }
+  const void *object = config_mgr_get_obj(mgr, options, var->object_idx);
+
+  result = struct_var_kvencode(object, &var->cvar->member);
+
+  if (escape_val) {
+    config_line_t *line;
+    for (line = result; line; line = line->next) {
+      if (line->value && config_value_needs_escape(line->value)) {
+        char *newval = esc_for_log(line->value);
+        tor_free(line->value);
+        line->value = newval;
+      }
+    }
+  }
+
+  return result;
+}
+/** Iterate through the linked list of requested options <b>list</b>.
+ * For each item, convert as appropriate and assign to <b>options</b>.
+ * If an item is unrecognized, set *msg and return -1 immediately,
+ * else return 0 for success.
+ *
+ * If <b>clear_first</b>, interpret config options as replacing (not
+ * extending) their previous values. If <b>clear_first</b> is set,
+ * then <b>use_defaults</b> to decide if you set to defaults after
+ * clearing, or make the value 0 or NULL.
+ *
+ * Here are the use cases:
+ * 1. A non-empty AllowInvalid line in your torrc. Appends to current
+ *    if linelist, replaces current if csv.
+ * 2. An empty AllowInvalid line in your torrc. Should clear it.
+ * 3. "RESETCONF AllowInvalid" sets it to default.
+ * 4. "SETCONF AllowInvalid" makes it NULL.
+ * 5. "SETCONF AllowInvalid=foo" clears it and sets it to "foo".
+ *
+ * Use_defaults   Clear_first
+ *    0                0       "append"
+ *    1                0       undefined, don't use
+ *    0                1       "set to null first"
+ *    1                1       "set to defaults first"
+ * Return 0 on success, -1 on bad key, -2 on bad value.
+ *
+ * As an additional special case, if a LINELIST config option has
+ * no value and clear_first is 0, then warn and ignore it.
+ */
+
+/*
+There are three call cases for config_assign() currently.
+
+Case one: Torrc entry
+options_init_from_torrc() calls config_assign(0, 0)
+  calls config_assign_line(0, 0).
+    if value is empty, calls config_reset(0) and returns.
+    calls config_assign_value(), appends.
+
+Case two: setconf
+options_trial_assign() calls config_assign(0, 1)
+  calls config_reset_line(0)
+    calls config_reset(0)
+      calls option_clear().
+  calls config_assign_line(0, 1).
+    if value is empty, returns.
+    calls config_assign_value(), appends.
+
+Case three: resetconf
+options_trial_assign() calls config_assign(1, 1)
+  calls config_reset_line(1)
+    calls config_reset(1)
+      calls option_clear().
+      calls config_assign_value(default)
+  calls config_assign_line(1, 1).
+    returns.
+*/
+int
+config_assign(const config_mgr_t *mgr, void *options, config_line_t *list,
+              unsigned config_assign_flags, char **msg)
+{
+  config_line_t *p;
+  bitarray_t *options_seen;
+  const int n_options = config_count_options(mgr);
+  const unsigned clear_first = config_assign_flags & CAL_CLEAR_FIRST;
+  const unsigned use_defaults = config_assign_flags & CAL_USE_DEFAULTS;
+
+  CONFIG_CHECK(mgr, options);
+
+  /* pass 1: normalize keys */
+  for (p = list; p; p = p->next) {
+    const char *full = config_expand_abbrev(mgr, p->key, 0, 1);
+    if (strcmp(full,p->key)) {
+      tor_free(p->key);
+      p->key = tor_strdup(full);
+    }
+  }
+
+  /* pass 2: if we're reading from a resetting source, clear all
+   * mentioned config options, and maybe set to their defaults. */
+  if (clear_first) {
+    for (p = list; p; p = p->next)
+      config_reset_line(mgr, options, p->key, use_defaults);
+  }
+
+  options_seen = bitarray_init_zero(n_options);
+  /* pass 3: assign. */
+  while (list) {
+    int r;
+    if ((r=config_assign_line(mgr, options, list, config_assign_flags,
+                              options_seen, msg))) {
+      bitarray_free(options_seen);
+      return r;
+    }
+    list = list->next;
+  }
+  bitarray_free(options_seen);
+
+  /** Now we're done assigning a group of options to the configuration.
+   * Subsequent group assignments should _replace_ linelists, not extend
+   * them. */
+  config_mark_lists_fragile(mgr, options);
+
+  return 0;
+}
+
+/** Reset config option <b>var</b> to 0, 0.0, NULL, or the equivalent.
+ * Called from config_reset() and config_free(). */
+static void
+config_clear(const config_mgr_t *mgr, void *options, const managed_var_t *var)
+{
+  void *object = config_mgr_get_obj_mutable(mgr, options, var->object_idx);
+  struct_var_free(object, &var->cvar->member);
+}
+
+/** Clear the option indexed by <b>var</b> in <b>options</b>. Then if
+ * <b>use_defaults</b>, set it to its default value.
+ * Called by config_init() and option_reset_line() and option_assign_line(). */
+static void
+config_reset(const config_mgr_t *mgr, void *options,
+             const managed_var_t *var, int use_defaults)
+{
+  config_line_t *c;
+  char *msg = NULL;
+  CONFIG_CHECK(mgr, options);
+  config_clear(mgr, options, var); /* clear it first */
+
+  if (!use_defaults)
+    return; /* all done */
+
+  if (var->cvar->initvalue) {
+    c = tor_malloc_zero(sizeof(config_line_t));
+    c->key = tor_strdup(var->cvar->member.name);
+    c->value = tor_strdup(var->cvar->initvalue);
+    if (config_assign_value(mgr, options, c, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_warn(LD_BUG, "Failed to assign default: %s", msg);
+      tor_free(msg); /* if this happens it's a bug */
+      // LCOV_EXCL_STOP
+    }
+    config_free_lines(c);
+  }
+}
+
+/** Release storage held by <b>options</b>. */
+void
+config_free_(const config_mgr_t *mgr, void *options)
+{
+  if (!options)
+    return;
+
+  tor_assert(mgr);
+
+  if (mgr->toplevel->clear_fn) {
+    mgr->toplevel->clear_fn(mgr, options);
+  }
+  config_suite_t **suitep = config_mgr_get_suite_ptr(mgr, options);
+  if (suitep) {
+    tor_assert(smartlist_len((*suitep)->configs) ==
+               smartlist_len(mgr->subconfigs));
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = smartlist_get((*suitep)->configs, fmt_sl_idx);
+      if (fmt->clear_fn) {
+        fmt->clear_fn(mgr, obj);
+      }
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    config_clear(mgr, options, mv);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (mgr->toplevel->extra) {
+    config_line_t **linep = STRUCT_VAR_P(options,
+                                         mgr->toplevel->extra->offset);
+    config_free_lines(*linep);
+    *linep = NULL;
+  }
+
+  if (suitep) {
+    SMARTLIST_FOREACH((*suitep)->configs, void *, obj, tor_free(obj));
+    config_suite_free(*suitep);
+  }
+
+  tor_free(options);
+}
+
+/** Return true iff the option <b>name</b> has the same value in <b>o1</b>
+ * and <b>o2</b>.  Must not be called for LINELIST_S or OBSOLETE options.
+ */
+int
+config_is_same(const config_mgr_t *mgr,
+               const void *o1, const void *o2,
+               const char *name)
+{
+  CONFIG_CHECK(mgr, o1);
+  CONFIG_CHECK(mgr, o2);
+
+  const managed_var_t *var = config_mgr_find_var(mgr, name, true, NULL);
+  if (!var) {
+    return true;
+  }
+  const void *obj1 = config_mgr_get_obj(mgr, o1, var->object_idx);
+  const void *obj2 = config_mgr_get_obj(mgr, o2, var->object_idx);
+
+  return struct_var_eq(obj1, obj2, &var->cvar->member);
+}
+
+/**
+ * Return a list of the options which have changed between <b>options1</b> and
+ * <b>options2</b>. If an option has reverted to its default value, it has a
+ * value entry of NULL.
+ *
+ * <b>options1</b> and <b>options2</b> must be top-level configuration objects
+ * of the type managed by <b>mgr</b>.
+ **/
+config_line_t *
+config_get_changes(const config_mgr_t *mgr,
+                   const void *options1, const void *options2)
+{
+  config_line_t *result = NULL;
+  config_line_t **next = &result;
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_should_list_changes(mv->cvar)) {
+      /* something else will check this var, or it doesn't need checking */
+      continue;
+    }
+    const void *obj1 = config_mgr_get_obj(mgr, options1, mv->object_idx);
+    const void *obj2 = config_mgr_get_obj(mgr, options2, mv->object_idx);
+
+    if (struct_var_eq(obj1, obj2, &mv->cvar->member)) {
+      continue;
+    }
+
+    const char *varname = mv->cvar->member.name;
+    config_line_t *line =
+      config_get_assigned_option(mgr, options2, varname, 1);
+
+    if (line) {
+      *next = line;
+    } else {
+      *next = tor_malloc_zero(sizeof(config_line_t));
+      (*next)->key = tor_strdup(varname);
+    }
+    while (*next)
+      next = &(*next)->next;
+  } SMARTLIST_FOREACH_END(mv);
+
+  return result;
+}
+
+/** Copy storage held by <b>old</b> into a new or_options_t and return it. */
+void *
+config_dup(const config_mgr_t *mgr, const void *old)
+{
+  void *newopts;
+
+  newopts = config_new(mgr);
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    if (! config_var_needs_copy(mv->cvar)) {
+      // Something else will copy this option, or it doesn't need copying.
+      continue;
+    }
+    const void *oldobj = config_mgr_get_obj(mgr, old, mv->object_idx);
+    void *newobj = config_mgr_get_obj_mutable(mgr, newopts, mv->object_idx);
+    if (struct_var_copy(newobj, oldobj, &mv->cvar->member) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Unable to copy value for %s.",
+              mv->cvar->member.name);
+      tor_assert_unreached();
+      // LCOV_EXCL_STOP
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return newopts;
+}
+/** Set all vars in the configuration object <b>options</b> to their default
+ * values. */
+void
+config_init(const config_mgr_t *mgr, void *options)
+{
+  CONFIG_CHECK(mgr, options);
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!mv->cvar->initvalue)
+      continue; /* defaults to NULL or 0 */
+    config_reset(mgr, options, mv, 1);
+  } SMARTLIST_FOREACH_END(mv);
+}
+
+/**
+ * Helper for config_validate_single: see whether any immutable option
+ * has changed between old_options and new_options.
+ *
+ * On success return 0; on failure set *msg_out to a newly allocated
+ * string explaining what is wrong, and return -1.
+ */
+static int
+config_check_immutable_flags(const config_format_t *fmt,
+                             const void *old_options,
+                             const void *new_options,
+                             char **msg_out)
+{
+  tor_assert(fmt);
+  tor_assert(new_options);
+  if (BUG(! old_options))
+    return 0;
+
+  unsigned i;
+  for (i = 0; fmt->vars[i].member.name; ++i) {
+    const config_var_t *v = &fmt->vars[i];
+    if (! config_var_has_flag(v, CFLG_IMMUTABLE))
+      continue;
+
+    if (! struct_var_eq(old_options, new_options, &v->member)) {
+      tor_asprintf(msg_out,
+                   "While Tor is running, changing %s is not allowed",
+                   v->member.name);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Normalize and validate a single object `options` within a configuration
+ * suite, according to its format.  `options` may be modified as appropriate
+ * in order to set ancillary data.  If `old_options` is provided, make sure
+ * that the transition from `old_options` to `options` is permitted.
+ *
+ * On success return VSTAT_OK; on failure set *msg_out to a newly allocated
+ * string explaining what is wrong, and return a different validation_status_t
+ * to describe which step failed.
+ **/
+static validation_status_t
+config_validate_single(const config_format_t *fmt,
+                       const void *old_options, void *options,
+                       char **msg_out)
+{
+  tor_assert(fmt);
+  tor_assert(options);
+
+  if (fmt->pre_normalize_fn) {
+    if (fmt->pre_normalize_fn(options, msg_out) < 0) {
+      return VSTAT_PRE_NORMALIZE_ERR;
+    }
+  }
+
+  if (fmt->legacy_validate_fn) {
+    if (fmt->legacy_validate_fn(old_options, options, msg_out) < 0) {
+      return VSTAT_LEGACY_ERR;
+    }
+  }
+
+  if (fmt->validate_fn) {
+    if (fmt->validate_fn(options, msg_out) < 0) {
+      return VSTAT_VALIDATE_ERR;
+    }
+  }
+
+  if (old_options) {
+    if (config_check_immutable_flags(fmt, old_options, options, msg_out) < 0) {
+      return VSTAT_TRANSITION_ERR;
+    }
+
+    if (fmt->check_transition_fn) {
+      if (fmt->check_transition_fn(old_options, options, msg_out) < 0) {
+        return VSTAT_TRANSITION_ERR;
+      }
+    }
+  }
+
+  if (fmt->post_normalize_fn) {
+    if (fmt->post_normalize_fn(options, msg_out) < 0) {
+      return VSTAT_POST_NORMALIZE_ERR;
+    }
+  }
+
+  return VSTAT_OK;
+}
+
+/**
+ * Normalize and validate all the options in configuration object `options`
+ * and its sub-objects. `options` may be modified as appropriate in order to
+ * set ancillary data.  If `old_options` is provided, make sure that the
+ * transition from `old_options` to `options` is permitted.
+ *
+ * On success return VSTAT_OK; on failure set *msg_out to a newly allocated
+ * string explaining what is wrong, and return a different validation_status_t
+ * to describe which step failed.
+ **/
+validation_status_t
+config_validate(const config_mgr_t *mgr,
+                const void *old_options, void *options,
+                char **msg_out)
+{
+  validation_status_t rv;
+  CONFIG_CHECK(mgr, options);
+  if (old_options) {
+    CONFIG_CHECK(mgr, old_options);
+  }
+
+  config_suite_t **suitep_new = config_mgr_get_suite_ptr(mgr, options);
+  config_suite_t **suitep_old = NULL;
+  if (old_options)
+    suitep_old = config_mgr_get_suite_ptr(mgr, (void*) old_options);
+
+  /* Validate the sub-objects */
+  if (suitep_new) {
+    SMARTLIST_FOREACH_BEGIN(mgr->subconfigs, const config_format_t *, fmt) {
+      void *obj = smartlist_get((*suitep_new)->configs, fmt_sl_idx);
+      const void *obj_old=NULL;
+      if (suitep_old)
+        obj_old = smartlist_get((*suitep_old)->configs, fmt_sl_idx);
+
+      rv = config_validate_single(fmt, obj_old, obj, msg_out);
+      if (rv < 0)
+        return rv;
+    } SMARTLIST_FOREACH_END(fmt);
+  }
+
+  /* Validate the top-level object. */
+  rv = config_validate_single(mgr->toplevel, old_options, options, msg_out);
+  if (rv < 0)
+    return rv;
+
+  return VSTAT_OK;
+}
+
+/** Allocate and return a new string holding the written-out values of the vars
+ * in 'options'.  If 'minimal', do not write out any default-valued vars.
+ * Else, if comment_defaults, write default values as comments.
+ */
+char *
+config_dump(const config_mgr_t *mgr, const void *default_options,
+            const void *options, int minimal,
+            int comment_defaults)
+{
+  const config_format_t *fmt = mgr->toplevel;
+  smartlist_t *elements;
+  const void *defaults = default_options;
+  void *defaults_tmp = NULL;
+  config_line_t *line, *assigned;
+  char *result;
+  char *msg = NULL;
+
+  if (defaults == NULL) {
+    defaults = defaults_tmp = config_new(mgr);
+    config_init(mgr, defaults_tmp);
+  }
+
+  /* XXX use a 1 here so we don't add a new log line while dumping */
+  if (default_options == NULL) {
+    if (config_validate(mgr, NULL, defaults_tmp, &msg) < 0) {
+      // LCOV_EXCL_START
+      log_err(LD_BUG, "Failed to validate default config: %s", msg);
+      tor_free(msg);
+      tor_assert(0);
+      // LCOV_EXCL_STOP
+    }
+  }
+
+  elements = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, managed_var_t *, mv) {
+    int comment_option = 0;
+    /* Don't save 'hidden' control variables. */
+    if (! config_var_is_dumpable(mv->cvar))
+      continue;
+    const char *name = mv->cvar->member.name;
+    if (minimal && config_is_same(mgr, options, defaults, name))
+      continue;
+    else if (comment_defaults &&
+             config_is_same(mgr, options, defaults, name))
+      comment_option = 1;
+
+    line = assigned =
+      config_get_assigned_option(mgr, options, name, 1);
+
+    for (; line; line = line->next) {
+      if (!strcmpstart(line->key, "__")) {
+        /* This check detects "hidden" variables inside LINELIST_V structures.
+         */
+        continue;
+      }
+      int value_exists = line->value && *(line->value);
+      smartlist_add_asprintf(elements, "%s%s%s%s\n",
+                   comment_option ? "# " : "",
+                   line->key, value_exists ? " " : "", line->value);
+    }
+    config_free_lines(assigned);
+  } SMARTLIST_FOREACH_END(mv);
+
+  if (fmt->extra) {
+    line = *(config_line_t**)STRUCT_VAR_P(options, fmt->extra->offset);
+    for (; line; line = line->next) {
+      int value_exists = line->value && *(line->value);
+      smartlist_add_asprintf(elements, "%s%s%s\n",
+                             line->key, value_exists ? " " : "", line->value);
+    }
+  }
+
+  result = smartlist_join_strings(elements, "", 0, NULL);
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+  config_free(mgr, defaults_tmp);
+  return result;
+}
+
+/**
+ * Return true if every member of <b>options</b> is in-range and well-formed.
+ * Return false otherwise.  Log errors at level <b>severity</b>.
+ */
+bool
+config_check_ok(const config_mgr_t *mgr, const void *options, int severity)
+{
+  bool all_ok = true;
+
+  SMARTLIST_FOREACH_BEGIN(mgr->all_vars, const managed_var_t *, mv) {
+    if (!struct_var_ok(options, &mv->cvar->member)) {
+      log_fn(severity, LD_BUG, "Invalid value for %s",
+             mv->cvar->member.name);
+      all_ok = false;
+    }
+  } SMARTLIST_FOREACH_END(mv);
+
+  return all_ok;
+}
diff --git a/src/lib/confmgt/confmgt.h b/src/lib/confmgt/confmgt.h
new file mode 100644
index 0000000000..5065c13b60
--- /dev/null
+++ b/src/lib/confmgt/confmgt.h
@@ -0,0 +1,136 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file confmgt.h
+ *
+ * \brief Header for confmgt.c.
+ */
+
+#ifndef TOR_CONFMGT_H
+#define TOR_CONFMGT_H
+
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confmacros.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * A collection of config_format_t objects to describe several objects
+ * that are all configured with the same configuration file.
+ *
+ * (NOTE: for now, this only handles a single config_format_t.)
+ **/
+typedef struct config_mgr_t config_mgr_t;
+
+config_mgr_t *config_mgr_new(const config_format_t *toplevel_fmt);
+void config_mgr_free_(config_mgr_t *mgr);
+int config_mgr_add_format(config_mgr_t *mgr,
+                          const config_format_t *fmt);
+void config_mgr_freeze(config_mgr_t *mgr);
+#define config_mgr_free(mgr) \
+  FREE_AND_NULL(config_mgr_t, config_mgr_free_, (mgr))
+struct smartlist_t *config_mgr_list_vars(const config_mgr_t *mgr);
+struct smartlist_t *config_mgr_list_deprecated_vars(const config_mgr_t *mgr);
+
+/** A collection of managed configuration objects. */
+typedef struct config_suite_t config_suite_t;
+
+/**
+ * Flag for config_assign: if set, then "resetting" an option changes it to
+ * its default value, as specified in the config_var_t.  Otherwise,
+ * "resetting" an option changes it to a type-dependent null value --
+ * typically 0 or NULL.
+ *
+ * (An option is "reset" when it is set to an empty value, or as described in
+ * CAL_CLEAR_FIRST).
+ **/
+#define CAL_USE_DEFAULTS      (1u<<0)
+/**
+ * Flag for config_assign: if set, then we reset every provided config
+ * option before we set it.
+ *
+ * For example, if this flag is not set, then passing a multi-line option to
+ * config_assign will cause any previous value to be extended. But if this
+ * flag is set, then a multi-line option will replace any previous value.
+ **/
+#define CAL_CLEAR_FIRST       (1u<<1)
+/**
+ * Flag for config_assign: if set, we warn about deprecated options.
+ **/
+#define CAL_WARN_DEPRECATIONS (1u<<2)
+
+void *config_new(const config_mgr_t *fmt);
+void config_free_(const config_mgr_t *fmt, void *options);
+#define config_free(mgr, options) do {                \
+    config_free_((mgr), (options));                   \
+    (options) = NULL;                                 \
+  } while (0)
+
+struct config_line_t *config_get_assigned_option(const config_mgr_t *mgr,
+                                          const void *options, const char *key,
+                                          int escape_val);
+int config_is_same(const config_mgr_t *fmt,
+                   const void *o1, const void *o2,
+                   const char *name);
+struct config_line_t *config_get_changes(const config_mgr_t *mgr,
+                                  const void *options1, const void *options2);
+void config_init(const config_mgr_t *mgr, void *options);
+
+/** An enumeration to report which validation step failed. */
+typedef enum {
+  VSTAT_PRE_NORMALIZE_ERR = -5,
+  VSTAT_VALIDATE_ERR = -4,
+  VSTAT_LEGACY_ERR = -3,
+  VSTAT_TRANSITION_ERR = -2,
+  VSTAT_POST_NORMALIZE_ERR = -1,
+  VSTAT_OK = 0,
+} validation_status_t;
+
+validation_status_t config_validate(const config_mgr_t *mgr,
+                                    const void *old_options, void *options,
+                                    char **msg_out);
+void *config_dup(const config_mgr_t *mgr, const void *old);
+char *config_dump(const config_mgr_t *mgr, const void *default_options,
+                  const void *options, int minimal,
+                  int comment_defaults);
+void config_check_toplevel_magic(const config_mgr_t *mgr,
+                                 const void *object);
+bool config_check_ok(const config_mgr_t *mgr, const void *options,
+                     int severity);
+int config_assign(const config_mgr_t *mgr, void *options,
+                  struct config_line_t *list,
+                  unsigned flags, char **msg);
+const char *config_find_deprecation(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_find_option_name(const config_mgr_t *mgr,
+                                    const char *key);
+const char *config_expand_abbrev(const config_mgr_t *mgr,
+                                 const char *option,
+                                 int command_line, int warn_obsolete);
+void warn_deprecated_option(const char *what, const char *why);
+
+bool config_var_is_settable(const config_var_t *var);
+bool config_var_is_listable(const config_var_t *var);
+
+/* Helper macros to compare an option across two configuration objects */
+#define CFG_EQ_BOOL(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_INT(a,b,opt) ((a)->opt == (b)->opt)
+#define CFG_EQ_STRING(a,b,opt) (!strcmp_opt((a)->opt, (b)->opt))
+#define CFG_EQ_SMARTLIST(a,b,opt) smartlist_strings_eq((a)->opt, (b)->opt)
+#define CFG_EQ_LINELIST(a,b,opt) config_lines_eq((a)->opt, (b)->opt)
+#define CFG_EQ_ROUTERSET(a,b,opt) routerset_equal((a)->opt, (b)->opt)
+
+void *config_mgr_get_obj_mutable(const config_mgr_t *mgr,
+                                 void *toplevel, int idx);
+const void *config_mgr_get_obj(const config_mgr_t *mgr,
+                               const void *toplevel, int idx);
+
+#ifdef CONFMGT_PRIVATE
+STATIC void config_reset_line(const config_mgr_t *mgr, void *options,
+                              const char *key, int use_defaults);
+#endif /* defined(CONFMGT_PRIVATE) */
+
+#endif /* !defined(TOR_CONFMGT_H) */
diff --git a/src/lib/confmgt/include.am b/src/lib/confmgt/include.am
new file mode 100644
index 0000000000..d3a7a7cd69
--- /dev/null
+++ b/src/lib/confmgt/include.am
@@ -0,0 +1,27 @@
+noinst_LIBRARIES += src/lib/libtor-confmgt.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-confmgt-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_confmgt_a_SOURCES =			\
+	src/lib/confmgt/confmgt.c      			\
+	src/lib/confmgt/structvar.c                      \
+	src/lib/confmgt/type_defs.c                      \
+	src/lib/confmgt/typedvar.c                      \
+	src/lib/confmgt/unitparse.c
+
+src_lib_libtor_confmgt_testing_a_SOURCES = \
+	$(src_lib_libtor_confmgt_a_SOURCES)
+src_lib_libtor_confmgt_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_confmgt_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/confmgt/confmgt.h			\
+	src/lib/confmgt/structvar.h			\
+	src/lib/confmgt/type_defs.h			\
+	src/lib/confmgt/typedvar.h			\
+	src/lib/confmgt/unitparse.h			\
+	src/lib/confmgt/var_type_def_st.h
diff --git a/src/lib/confmgt/lib_confmgt.md b/src/lib/confmgt/lib_confmgt.md
new file mode 100644
index 0000000000..861e720f64
--- /dev/null
+++ b/src/lib/confmgt/lib_confmgt.md
@@ -0,0 +1,7 @@
+@dir /lib/confmgt
+@brief lib/confmgt: Parse, encode, manipulate configuration files.
+
+This logic is used in common by our state files (statefile.c) and
+configuration files (config.c) to manage a set of named, typed fields,
+reading and writing them to disk and to the controller.
+
diff --git a/src/lib/confmgt/structvar.c b/src/lib/confmgt/structvar.c
new file mode 100644
index 0000000000..55deb4759c
--- /dev/null
+++ b/src/lib/confmgt/structvar.c
@@ -0,0 +1,239 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.c
+ * @brief Functions to manipulate named and typed elements of
+ *    a structure.
+ *
+ * These functions represent a low-level API for accessing a member of a
+ * structure.  They use typedvar.c to work, and they are used in turn by the
+ * configuration system to examine and set fields in configuration objects
+ * used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/structvar.h"
+#include "lib/cc/compat_compiler.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/log/util_bug.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+
+/**
+ * Return true iff all fields on <b>decl</b> are NULL or 0, indicating that
+ * there is no object or no magic number to check.
+ **/
+static inline bool
+magic_is_null(const struct_magic_decl_t *decl)
+{
+  return decl->typename == NULL &&
+    decl->magic_offset == 0 &&
+    decl->magic_val == 0;
+}
+
+/**
+ * Set the 'magic number' on <b>object</b> to correspond to decl.
+ **/
+void
+struct_set_magic(void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(decl);
+  if (magic_is_null(decl))
+    return;
+
+  tor_assert(object);
+  uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  *ptr = decl->magic_val;
+}
+
+/**
+ * Assert that the 'magic number' on <b>object</b> to corresponds to decl.
+ **/
+void
+struct_check_magic(const void *object, const struct_magic_decl_t *decl)
+{
+  tor_assert(decl);
+  if (magic_is_null(decl))
+    return;
+
+  tor_assert(object);
+
+  const uint32_t *ptr = STRUCT_VAR_P(object, decl->magic_offset);
+  tor_assertf(*ptr == decl->magic_val,
+              "Bad magic number on purported %s object. "
+              "Expected %"PRIu32"x but got %"PRIu32"x.",
+              decl->typename, decl->magic_val, *ptr);
+}
+
+/**
+ * Return a mutable pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+void *
+struct_get_mptr(void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Return a const pointer to the member of <b>object</b> described
+ * by <b>member</b>.
+ **/
+const void *
+struct_get_ptr(const void *object, const struct_member_t *member)
+{
+  tor_assert(object);
+  return STRUCT_VAR_P(object, member->offset);
+}
+
+/**
+ * Helper: given a struct_member_t, look up the type definition for its
+ * variable.
+ */
+static const var_type_def_t *
+get_type_def(const struct_member_t *member)
+{
+  if (member->type_def)
+    return member->type_def;
+
+  return lookup_type_def(member->type);
+}
+
+/**
+ * (As typed_var_free, but free and clear the member of <b>object</b> defined
+ * by <b>member</b>.)
+ **/
+void
+struct_var_free(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  typed_var_free(p, def);
+}
+
+/**
+ * (As typed_var_copy, but copy from <b>src</b> to <b>dest</b> the member
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_copy(void *dest, const void *src, const struct_member_t *member)
+{
+  void *p_dest = struct_get_mptr(dest, member);
+  const void *p_src = struct_get_ptr(src, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_copy(p_dest, p_src, def);
+}
+
+/**
+ * (As typed_var_eq, but compare the members of <b>a</b> and <b>b</b>
+ * defined by <b>member</b>.)
+ **/
+bool
+struct_var_eq(const void *a, const void *b, const struct_member_t *member)
+{
+  const void *p_a = struct_get_ptr(a, member);
+  const void *p_b = struct_get_ptr(b, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_eq(p_a, p_b, def);
+}
+
+/**
+ * (As typed_var_ok, but validate the member of <b>object</b> defined by
+ * <b>member</b>.)
+ **/
+bool
+struct_var_ok(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_ok(p, def);
+}
+
+/**
+ * (As typed_var_kvassign, but assign a value to the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+int
+struct_var_kvassign(void *object, const struct config_line_t *line,
+                    char **errmsg,
+                    const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvassign(p, line, errmsg, def);
+}
+
+/**
+ * (As typed_var_kvencode, but encode the value of the member of <b>object</b>
+ * defined by <b>member</b>.)
+ **/
+struct config_line_t *
+struct_var_kvencode(const void *object, const struct_member_t *member)
+{
+  const void *p = struct_get_ptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+
+  return typed_var_kvencode(member->name, p, def);
+}
+
+/**
+ * Mark the field in <b>object</b> determined by <b>member</b> -- a variable
+ * that ordinarily would be extended by assignment -- as "fragile", so that it
+ * will get replaced by the next assignment instead.
+ */
+void
+struct_var_mark_fragile(void *object, const struct_member_t *member)
+{
+  void *p = struct_get_mptr(object, member);
+  const var_type_def_t *def = get_type_def(member);
+  return typed_var_mark_fragile(p, def);
+}
+
+/**
+ * Return the official name of this struct member.
+ **/
+const char *
+struct_var_get_name(const struct_member_t *member)
+{
+  return member->name;
+}
+
+/**
+ * Return the type name for this struct member.
+ *
+ * Do not use the output of this function to inspect a type within Tor.  It is
+ * suitable for debugging, informing the controller or user of a variable's
+ * type, etc.
+ **/
+const char *
+struct_var_get_typename(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->name : NULL;
+}
+
+/** Return all of the flags set for this struct member. */
+uint32_t
+struct_var_get_flags(const struct_member_t *member)
+{
+  const var_type_def_t *def = get_type_def(member);
+
+  return def ? def->flags : 0;
+}
diff --git a/src/lib/confmgt/structvar.h b/src/lib/confmgt/structvar.h
new file mode 100644
index 0000000000..91334fa8c5
--- /dev/null
+++ b/src/lib/confmgt/structvar.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file structvar.h
+ * @brief Header for lib/confmgt/structvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_STRUCTVAR_H
+#define TOR_LIB_CONFMGT_STRUCTVAR_H
+
+struct struct_magic_decl_t;
+struct struct_member_t;
+struct config_line_t;
+
+#include <stdbool.h>
+#include "lib/cc/torint.h"
+
+void struct_set_magic(void *object,
+                      const struct struct_magic_decl_t *decl);
+void struct_check_magic(const void *object,
+                        const struct struct_magic_decl_t *decl);
+
+void *struct_get_mptr(void *object,
+                      const struct struct_member_t *member);
+const void *struct_get_ptr(const void *object,
+                           const struct struct_member_t *member);
+
+void struct_var_free(void *object,
+                     const struct struct_member_t *member);
+int struct_var_copy(void *dest, const void *src,
+                    const struct struct_member_t *member);
+bool struct_var_eq(const void *a, const void *b,
+                   const struct struct_member_t *member);
+bool struct_var_ok(const void *object,
+                   const struct struct_member_t *member);
+void struct_var_mark_fragile(void *object,
+                             const struct struct_member_t *member);
+
+const char *struct_var_get_name(const struct struct_member_t *member);
+const char *struct_var_get_typename(const struct struct_member_t *member);
+uint32_t struct_var_get_flags(const struct struct_member_t *member);
+
+int struct_var_kvassign(void *object, const struct config_line_t *line,
+                        char **errmsg,
+                        const struct struct_member_t *member);
+struct config_line_t *struct_var_kvencode(
+                        const void *object,
+                        const struct struct_member_t *member);
+
+#endif /* !defined(TOR_LIB_CONFMGT_STRUCTVAR_H) */
diff --git a/src/lib/confmgt/type_defs.c b/src/lib/confmgt/type_defs.c
new file mode 100644
index 0000000000..d9e5e1e4c2
--- /dev/null
+++ b/src/lib/confmgt/type_defs.c
@@ -0,0 +1,849 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.c
+ * @brief Definitions for various low-level configuration types.
+ *
+ * This module creates a number of var_type_def_t objects, to be used by
+ * typedvar.c in manipulating variables.
+ *
+ * The types here are common types that can be implemented with Tor's
+ * low-level functionality.  To define new types, see var_type_def_st.h.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/conf/confdecl.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/unitparse.h"
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/time_fmt.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+#include <errno.h>
+
+//////
+// CONFIG_TYPE_STRING
+// CONFIG_TYPE_FILENAME
+//
+// These two types are the same for now, but they have different names.
+//
+// Warning: For this type, the default value (NULL) and "" are considered
+// different values.  That is generally risky, and best avoided for other
+// types in the future.
+//////
+
+static int
+string_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  char **p = (char**)target;
+  *p = tor_strdup(value);
+  return 0;
+}
+
+static char *
+string_encode(const void *value, const void *params)
+{
+  (void)params;
+  const char **p = (const char**)value;
+  return *p ? tor_strdup(*p) : NULL;
+}
+
+static void
+string_clear(void *value, const void *params)
+{
+  (void)params;
+  char **p = (char**)value;
+  tor_free(*p); // sets *p to NULL.
+}
+
+static const var_type_fns_t string_fns = {
+  .parse = string_parse,
+  .encode = string_encode,
+  .clear = string_clear,
+};
+
+/////
+// CONFIG_TYPE_INT
+// CONFIG_TYPE_POSINT
+//
+// These types are implemented as int, possibly with a restricted range.
+/////
+
+/**
+ * Parameters for parsing an integer type.
+ **/
+typedef struct int_type_params_t {
+  int minval; /**< Lowest allowed value */
+  int maxval; /**< Highest allowed value */
+} int_parse_params_t;
+
+static const int_parse_params_t INT_PARSE_UNRESTRICTED = {
+  .minval = INT_MIN,
+  .maxval = INT_MAX,
+};
+
+static const int_parse_params_t INT_PARSE_POSINT = {
+  .minval = 0,
+  .maxval = INT_MAX,
+};
+
+static int
+int_parse(void *target, const char *value, char **errmsg, const void *params)
+{
+  const int_parse_params_t *pp;
+  if (params) {
+    pp = params;
+  } else {
+    pp = &INT_PARSE_UNRESTRICTED;
+  }
+  int *p = target;
+  int ok=0;
+  *p = (int)tor_parse_long(value, 10, pp->minval, pp->maxval, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds. "
+                 "Allowed values are between %d and %d.",
+                 value, pp->minval, pp->maxval);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+int_encode(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  char *result;
+  tor_asprintf(&result, "%d", v);
+  return result;
+}
+
+static void
+int_clear(void *value, const void *params)
+{
+  (void)params;
+  *(int*)value = 0;
+}
+
+static bool
+int_ok(const void *value, const void *params)
+{
+  const int_parse_params_t *pp = params;
+  if (pp) {
+    int v = *(int*)value;
+    return pp->minval <= v && v <= pp->maxval;
+  } else {
+    return true;
+  }
+}
+
+static const var_type_fns_t int_fns = {
+  .parse = int_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+  .ok = int_ok,
+};
+
+/////
+// CONFIG_TYPE_UINT64
+//
+// This type is an unrestricted u64.
+/////
+
+static int
+uint64_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  uint64_t *p = target;
+  int ok=0;
+  *p = tor_parse_uint64(value, 10, 0, UINT64_MAX, &ok, NULL);
+  if (!ok) {
+    tor_asprintf(errmsg, "Integer %s is malformed or out of bounds.",
+                 value);
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+uint64_encode(const void *value, const void *params)
+{
+  (void)params;
+  uint64_t v = *(uint64_t*)value;
+  char *result;
+  tor_asprintf(&result, "%"PRIu64, v);
+  return result;
+}
+
+static void
+uint64_clear(void *value, const void *params)
+{
+  (void)params;
+  *(uint64_t*)value = 0;
+}
+
+static const var_type_fns_t uint64_fns = {
+  .parse = uint64_parse,
+  .encode = uint64_encode,
+  .clear = uint64_clear,
+};
+
+/////
+// CONFIG_TYPE_INTERVAL
+// CONFIG_TYPE_MSEC_INTERVAL
+// CONFIG_TYPE_MEMUNIT
+//
+// These types are implemented using the config_parse_units() function.
+// The intervals are stored as ints, whereas memory units are stored as
+// uint64_ts.
+/////
+
+static int
+units_parse_u64(void *target, const char *value, char **errmsg,
+                const void *params)
+{
+  const unit_table_t *table = params;
+  tor_assert(table);
+  uint64_t *v = (uint64_t*)target;
+  int ok=1;
+  char *msg = NULL;
+  *v = config_parse_units(value, table, &ok, &msg);
+  if (!ok) {
+    tor_asprintf(errmsg, "Provided value is malformed or out of bounds: %s",
+                 msg);
+    tor_free(msg);
+    return -1;
+  }
+  if (BUG(msg)) {
+    tor_free(msg);
+  }
+  return 0;
+}
+
+static int
+units_parse_int(void *target, const char *value, char **errmsg,
+               const void *params)
+{
+  const unit_table_t *table = params;
+  tor_assert(table);
+  int *v = (int*)target;
+  int ok=1;
+  char *msg = NULL;
+  uint64_t u64 = config_parse_units(value, table, &ok, &msg);
+  if (!ok) {
+    tor_asprintf(errmsg, "Provided value is malformed or out of bounds: %s",
+                 msg);
+    tor_free(msg);
+    return -1;
+  }
+  if (BUG(msg)) {
+    tor_free(msg);
+  }
+  if (u64 > INT_MAX) {
+    tor_asprintf(errmsg, "Provided value %s is too large", value);
+    return -1;
+  }
+  *v = (int) u64;
+  return 0;
+}
+
+static bool
+units_ok_int(const void *value, const void *params)
+{
+  (void)params;
+  int v = *(int*)value;
+  return v >= 0;
+}
+
+static const var_type_fns_t memunit_fns = {
+  .parse = units_parse_u64,
+  .encode = uint64_encode, // doesn't use params
+  .clear = uint64_clear, // doesn't use params
+};
+
+static const var_type_fns_t interval_fns = {
+  .parse = units_parse_int,
+  .encode = int_encode, // doesn't use params
+  .clear = int_clear, // doesn't use params,
+  .ok = units_ok_int // can't use int_ok, since that expects int params.
+};
+
+/////
+// CONFIG_TYPE_DOUBLE
+//
+// This is a nice simple double.
+/////
+
+static int
+double_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  double *v = (double*)target;
+  char *endptr=NULL;
+  errno = 0;
+  *v = strtod(value, &endptr);
+  if (endptr == value || *endptr != '\0') {
+    // Either there are no converted characters, or there were some characters
+    // that didn't get converted.
+    tor_asprintf(errmsg, "Could not convert %s to a number.", escaped(value));
+    return -1;
+  }
+  if (errno == ERANGE) {
+    // strtod will set errno to ERANGE on underflow or overflow.
+    bool underflow = -.00001 < *v && *v < .00001;
+    tor_asprintf(errmsg,
+                 "%s is too %s to express as a floating-point number.",
+                 escaped(value), underflow ? "small" : "large");
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+double_encode(const void *value, const void *params)
+{
+  (void)params;
+  double v = *(double*)value;
+  char *result;
+  tor_asprintf(&result, "%f", v);
+  return result;
+}
+
+static void
+double_clear(void *value, const void *params)
+{
+  (void)params;
+  double *v = (double *)value;
+  *v = 0.0;
+}
+
+static const var_type_fns_t double_fns = {
+  .parse = double_parse,
+  .encode = double_encode,
+  .clear = double_clear,
+};
+
+/////
+// CONFIG_TYPE_BOOL
+// CONFIG_TYPE_AUTOBOOL
+//
+// These types are implemented as a case-insensitive string-to-integer
+// mapping.
+/////
+
+typedef struct enumeration_table_t {
+  const char *name;
+  int value;
+} enumeration_table_t;
+
+typedef struct enumeration_params_t {
+  const char *allowed_val_string;
+  const enumeration_table_t *table;
+} enumeration_params_t;
+
+static int
+enum_parse(void *target, const char *value, char **errmsg,
+           const void *params_)
+{
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  int *p = (int *)target;
+  for (; table->name; ++table) {
+    if (!strcasecmp(value, table->name)) {
+      *p = table->value;
+      return 0;
+    }
+  }
+  tor_asprintf(errmsg, "Unrecognized value %s. %s",
+               value, params->allowed_val_string);
+  return -1;
+}
+
+static char *
+enum_encode(const void *value, const void *params_)
+{
+  int v = *(const int*)value;
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return tor_strdup(table->name);
+  }
+  return NULL; // error.
+}
+
+static void
+enum_clear(void *value, const void *params_)
+{
+  int *p = (int*)value;
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  tor_assert(table->name);
+  *p = table->value;
+}
+
+static bool
+enum_ok(const void *value, const void *params_)
+{
+  int v = *(const int*)value;
+  const enumeration_params_t *params = params_;
+  const enumeration_table_t *table = params->table;
+  for (; table->name; ++table) {
+    if (v == table->value)
+      return true;
+  }
+  return false;
+}
+
+static const enumeration_table_t enum_table_bool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { NULL, 0 },
+};
+
+static const enumeration_params_t enum_params_bool = {
+  "Allowed values are 0 and 1.",
+  enum_table_bool
+};
+
+static const enumeration_table_t enum_table_autobool[] = {
+  { "0", 0 },
+  { "1", 1 },
+  { "auto", -1 },
+  { NULL, 0 },
+};
+
+static const enumeration_params_t enum_params_autobool = {
+  "Allowed values are 0, 1, and auto.",
+  enum_table_autobool
+};
+
+static const var_type_fns_t enum_fns = {
+  .parse = enum_parse,
+  .encode = enum_encode,
+  .clear = enum_clear,
+  .ok = enum_ok,
+};
+
+/////
+// CONFIG_TYPE_ISOTIME
+//
+// This is a time_t, encoded in ISO8601 format.
+/////
+
+static int
+time_parse(void *target, const char *value, char **errmsg,
+           const void *params)
+{
+  (void) params;
+  time_t *p = target;
+  if (parse_iso_time(value, p) < 0) {
+    tor_asprintf(errmsg, "Invalid time %s", escaped(value));
+    return -1;
+  }
+  return 0;
+}
+
+static char *
+time_encode(const void *value, const void *params)
+{
+  (void)params;
+  time_t v = *(const time_t *)value;
+  char *result = tor_malloc(ISO_TIME_LEN+1);
+  format_iso_time(result, v);
+  return result;
+}
+
+static void
+time_clear(void *value, const void *params)
+{
+  (void)params;
+  time_t *t = value;
+  *t = 0;
+}
+
+static const var_type_fns_t time_fns = {
+  .parse = time_parse,
+  .encode = time_encode,
+  .clear = time_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV
+//
+// This type is a comma-separated list of strings, stored in a smartlist_t.
+// An empty list may be encoded either as an empty smartlist, or as NULL.
+/////
+
+static int
+csv_parse(void *target, const char *value, char **errmsg,
+          const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  smartlist_t **sl = (smartlist_t**)target;
+  *sl = smartlist_new();
+  smartlist_split_string(*sl, value, ",",
+                         SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 0);
+  return 0;
+}
+
+static char *
+csv_encode(const void *value, const void *params)
+{
+  (void)params;
+  const smartlist_t *sl = *(const smartlist_t **)value;
+  if (! sl)
+    return tor_strdup("");
+
+  return smartlist_join_strings(*(smartlist_t**)value, ",", 0, NULL);
+}
+
+static void
+csv_clear(void *value, const void *params)
+{
+  (void)params;
+  smartlist_t **sl = (smartlist_t**)value;
+  if (!*sl)
+    return;
+  SMARTLIST_FOREACH(*sl, char *, cp, tor_free(cp));
+  smartlist_free(*sl); // clears pointer.
+}
+
+static const var_type_fns_t csv_fns = {
+  .parse = csv_parse,
+  .encode = csv_encode,
+  .clear = csv_clear,
+};
+
+/////
+// CONFIG_TYPE_CSV_INTERVAL
+//
+// This type used to be a list of time intervals, used to determine a download
+// schedule.  Now, only the first interval counts: everything after the first
+// comma is discarded.
+/////
+
+static int
+legacy_csv_interval_parse(void *target, const char *value, char **errmsg,
+                          const void *params)
+{
+  (void)params;
+  /* We used to have entire smartlists here.  But now that all of our
+   * download schedules use exponential backoff, only the first part
+   * matters. */
+  const char *comma = strchr(value, ',');
+  const char *val = value;
+  char *tmp = NULL;
+  if (comma) {
+    tmp = tor_strndup(val, comma - val);
+    val = tmp;
+  }
+
+  int rv = units_parse_int(target, val, errmsg, &time_units);
+  tor_free(tmp);
+  return rv;
+}
+
+static const var_type_fns_t legacy_csv_interval_fns = {
+  .parse = legacy_csv_interval_parse,
+  .encode = int_encode,
+  .clear = int_clear,
+};
+
+/////
+// CONFIG_TYPE_LINELIST
+// CONFIG_TYPE_LINELIST_S
+// CONFIG_TYPE_LINELIST_V
+//
+// A linelist is a raw config_line_t list.  Order is preserved.
+//
+// The LINELIST type is used for homogeneous lists, where all the lines
+// have the same key.
+//
+// The LINELIST_S and LINELIST_V types are used for the case where multiple
+// lines of different keys are kept in a single list, to preserve their
+// relative order.  The unified list is stored as a "virtual" variable whose
+// type is LINELIST_V; the individual sublists are treated as variables of
+// type LINELIST_S.
+//
+// A linelist may be fragile or non-fragile. Assigning a line to a fragile
+// linelist replaces the list with the line.  If the line has the "APPEND"
+// command set on it, or if the list is non-fragile, the line is appended.
+// Either way, the new list is non-fragile.
+/////
+
+static int
+linelist_kv_parse(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params)
+{
+  (void)params;
+  (void)errmsg;
+  config_line_t **lines = target;
+
+  if (*lines && (*lines)->fragile) {
+    if (line->command == CONFIG_LINE_APPEND) {
+      (*lines)->fragile = 0;
+    } else {
+      config_free_lines(*lines); // sets it to NULL
+    }
+  }
+
+  config_line_append(lines, line->key, line->value);
+  return 0;
+}
+
+static int
+linelist_kv_virt_noparse(void *target, const struct config_line_t *line,
+                         char **errmsg, const void *params)
+{
+  (void)target;
+  (void)line;
+  (void)params;
+  *errmsg = tor_strdup("Cannot assign directly to virtual option.");
+  return -1;
+}
+
+static struct config_line_t *
+linelist_kv_encode(const char *key, const void *value,
+                   const void *params)
+{
+  (void)key;
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup(lines);
+}
+
+static struct config_line_t *
+linelist_s_kv_encode(const char *key, const void *value,
+                     const void *params)
+{
+  (void)params;
+  config_line_t *lines = *(config_line_t **)value;
+  return config_lines_dup_and_filter(lines, key);
+}
+
+static void
+linelist_clear(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **lines = target;
+  config_free_lines(*lines); // sets it to NULL
+}
+
+static bool
+linelist_eq(const void *a, const void *b, const void *params)
+{
+  (void)params;
+  const config_line_t *lines_a = *(const config_line_t **)a;
+  const config_line_t *lines_b = *(const config_line_t **)b;
+  return config_lines_eq(lines_a, lines_b);
+}
+
+static int
+linelist_copy(void *target, const void *value, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  const config_line_t *val = *(const config_line_t **)value;
+  config_free_lines(*ptr);
+  *ptr = config_lines_dup(val);
+  return 0;
+}
+
+static void
+linelist_mark_fragile(void *target, const void *params)
+{
+  (void)params;
+  config_line_t **ptr = (config_line_t **)target;
+  if (*ptr)
+    (*ptr)->fragile = 1;
+}
+
+static const var_type_fns_t linelist_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_v_fns = {
+  .kv_parse = linelist_kv_virt_noparse,
+  .kv_encode = linelist_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+  .mark_fragile = linelist_mark_fragile,
+};
+
+static const var_type_fns_t linelist_s_fns = {
+  .kv_parse = linelist_kv_parse,
+  .kv_encode = linelist_s_kv_encode,
+  .clear = linelist_clear,
+  .eq = linelist_eq,
+  .copy = linelist_copy,
+};
+
+/////
+// CONFIG_TYPE_ROUTERSET
+//
+// XXXX to this module.
+/////
+
+/////
+// CONFIG_TYPE_IGNORE
+//
+// Used to indicate an option that cannot be stored or encoded.
+/////
+
+static int
+ignore_parse(void *target, const char *value, char **errmsg,
+             const void *params)
+{
+  (void)target;
+  (void)value;
+  (void)errmsg;
+  (void)params;
+  return 0;
+}
+
+static char *
+ignore_encode(const void *value, const void *params)
+{
+  (void)value;
+  (void)params;
+  return NULL;
+}
+
+static const var_type_fns_t ignore_fns = {
+  .parse = ignore_parse,
+  .encode = ignore_encode,
+};
+
+const var_type_def_t STRING_type_defn = {
+  .name="String", .fns=&string_fns };
+const var_type_def_t FILENAME_type_defn = {
+  .name="Filename", .fns=&string_fns };
+const var_type_def_t INT_type_defn = {
+  .name="SignedInteger", .fns=&int_fns,
+  .params=&INT_PARSE_UNRESTRICTED };
+const var_type_def_t POSINT_type_defn = {
+  .name="Integer", .fns=&int_fns,
+  .params=&INT_PARSE_POSINT };
+const var_type_def_t UINT64_type_defn = {
+  .name="Integer", .fns=&uint64_fns, };
+const var_type_def_t MEMUNIT_type_defn = {
+  .name="DataSize", .fns=&memunit_fns,
+  .params=&memory_units };
+const var_type_def_t INTERVAL_type_defn = {
+  .name="TimeInterval", .fns=&interval_fns,
+  .params=&time_units };
+const var_type_def_t MSEC_INTERVAL_type_defn = {
+  .name="TimeMsecInterval",
+  .fns=&interval_fns,
+  .params=&time_msec_units };
+const var_type_def_t DOUBLE_type_defn = {
+  .name="Float", .fns=&double_fns, };
+const var_type_def_t BOOL_type_defn = {
+  .name="Boolean", .fns=&enum_fns,
+  .params=&enum_params_bool };
+const var_type_def_t AUTOBOOL_type_defn = {
+  .name="Boolean+Auto", .fns=&enum_fns,
+  .params=&enum_params_autobool };
+const var_type_def_t ISOTIME_type_defn = {
+  .name="Time", .fns=&time_fns, };
+const var_type_def_t CSV_type_defn = {
+  .name="CommaList", .fns=&csv_fns, };
+const var_type_def_t CSV_INTERVAL_type_defn = {
+  .name="TimeInterval",
+  .fns=&legacy_csv_interval_fns, };
+const var_type_def_t LINELIST_type_defn = {
+  .name="LineList", .fns=&linelist_fns,
+  .flags=CFLG_NOREPLACE };
+/*
+ * A "linelist_s" is a derived view of a linelist_v: inspecting
+ * it gets part of a linelist_v, and setting it adds to the linelist_v.
+ */
+const var_type_def_t LINELIST_S_type_defn = {
+  .name="Dependent", .fns=&linelist_s_fns,
+  .flags=CFLG_NOREPLACE|
+  /* The operations we disable here are
+   * handled by the linelist_v. */
+  CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP };
+const var_type_def_t LINELIST_V_type_defn = {
+  .name="Virtual", .fns=&linelist_v_fns,
+  .flags=CFLG_NOREPLACE|CFLG_NOSET };
+const var_type_def_t IGNORE_type_defn = {
+  .name="Ignored", .fns=&ignore_fns,
+  .flags=CFLG_NOCOPY|CFLG_NOCMP|CFLG_NODUMP|CFLG_NOSET,
+};
+const var_type_def_t OBSOLETE_type_defn = {
+  .name="Obsolete", .fns=&ignore_fns,
+  .flags=CFLG_GROUP_OBSOLETE,
+};
+
+/**
+ * Table mapping conf_type_t values to var_type_def_t objects.
+ **/
+static const var_type_def_t *type_definitions_table[] = {
+  [CONFIG_TYPE_STRING] = &STRING_type_defn,
+  [CONFIG_TYPE_FILENAME] = &FILENAME_type_defn,
+  [CONFIG_TYPE_INT] = &INT_type_defn,
+  [CONFIG_TYPE_POSINT] = &POSINT_type_defn,
+  [CONFIG_TYPE_UINT64] = &UINT64_type_defn,
+  [CONFIG_TYPE_MEMUNIT] = &MEMUNIT_type_defn,
+  [CONFIG_TYPE_INTERVAL] = &INTERVAL_type_defn,
+  [CONFIG_TYPE_MSEC_INTERVAL] = &MSEC_INTERVAL_type_defn,
+  [CONFIG_TYPE_DOUBLE] = &DOUBLE_type_defn,
+  [CONFIG_TYPE_BOOL] = &BOOL_type_defn,
+  [CONFIG_TYPE_AUTOBOOL] = &AUTOBOOL_type_defn,
+  [CONFIG_TYPE_ISOTIME] = &ISOTIME_type_defn,
+  [CONFIG_TYPE_CSV] = &CSV_type_defn,
+  [CONFIG_TYPE_CSV_INTERVAL] = &CSV_INTERVAL_type_defn,
+  [CONFIG_TYPE_LINELIST] = &LINELIST_type_defn,
+  [CONFIG_TYPE_LINELIST_S] = &LINELIST_S_type_defn,
+  [CONFIG_TYPE_LINELIST_V] = &LINELIST_V_type_defn,
+  [CONFIG_TYPE_IGNORE] = &IGNORE_type_defn,
+  [CONFIG_TYPE_OBSOLETE] = &OBSOLETE_type_defn,
+};
+
+/**
+ * Return a pointer to the var_type_def_t object for the given
+ * config_type_t value, or NULL if no such type definition exists.
+ **/
+const var_type_def_t *
+lookup_type_def(config_type_t type)
+{
+  int t = type;
+  tor_assert(t >= 0);
+  if (t >= (int)ARRAY_LENGTH(type_definitions_table))
+    return NULL;
+  return type_definitions_table[t];
+}
diff --git a/src/lib/confmgt/type_defs.h b/src/lib/confmgt/type_defs.h
new file mode 100644
index 0000000000..fec002b1d3
--- /dev/null
+++ b/src/lib/confmgt/type_defs.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file type_defs.h
+ * @brief Header for lib/confmgt/type_defs.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPE_DEFS_H
+#define TOR_LIB_CONFMGT_TYPE_DEFS_H
+
+const struct var_type_def_t *lookup_type_def(config_type_t type);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPE_DEFS_H) */
diff --git a/src/lib/confmgt/typedvar.c b/src/lib/confmgt/typedvar.c
new file mode 100644
index 0000000000..1955302cdc
--- /dev/null
+++ b/src/lib/confmgt/typedvar.c
@@ -0,0 +1,236 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.c
+ * @brief Functions for accessing a pointer as an object of a given type.
+ *
+ * These functions represent a low-level API for accessing a typed variable.
+ * They are used in the configuration system to examine and set fields in
+ * configuration objects used by individual modules.
+ *
+ * Almost no code should call these directly.
+ **/
+
+#include "orconfig.h"
+#include "lib/conf/conftypes.h"
+#include "lib/confmgt/type_defs.h"
+#include "lib/confmgt/typedvar.h"
+#include "lib/encoding/confline.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+
+#include "lib/confmgt/var_type_def_st.h"
+
+#include <stddef.h>
+#include <string.h>
+
+/**
+ * Try to parse a string in <b>value</b> that encodes an object of the type
+ * defined by <b>def</b>.
+ *
+ * On success, adjust the lvalue pointed to by <b>target</b> to hold that
+ * value, and return 0.  On failure, set *<b>errmsg</b> to a newly allocated
+ * string holding an error message, and return -1.
+ **/
+int
+typed_var_assign(void *target, const char *value, char **errmsg,
+                    const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  // clear old value if needed.
+  typed_var_free(target, def);
+
+  tor_assert(def->fns->parse);
+  return def->fns->parse(target, value, errmsg, def->params);
+}
+
+/**
+ * Try to parse a single line from the head of<b>line</b> that encodes an
+ * object of the type defined in <b>def</b>. On success and failure, behave as
+ * typed_var_assign().
+ *
+ * All types for which keys are significant should use this function.
+ *
+ * Note that although multiple lines may be provided in <b>line</b>,
+ * only the first one is handled by this function.
+ **/
+int
+typed_var_kvassign(void *target, const config_line_t *line,
+                      char **errmsg, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+
+  if (def->fns->kv_parse) {
+    // We do _not_ free the old value here, since linelist options
+    // sometimes have append semantics.
+    return def->fns->kv_parse(target, line, errmsg, def->params);
+  }
+
+  int rv = typed_var_assign(target, line->value, errmsg, def);
+  if (rv < 0 && *errmsg != NULL) {
+    /* typed_var_assign() didn't know the line's keyword, but we do.
+     * Let's add it to the error message. */
+    char *oldmsg = *errmsg;
+    tor_asprintf(errmsg, "Could not parse %s: %s", line->key, oldmsg);
+    tor_free(oldmsg);
+  }
+  return rv;
+}
+
+/**
+ * Release storage held by a variable in <b>target</b> of type defined by
+ * <b>def</b>, and set <b>target</b> to a reasonable default.
+ **/
+void
+typed_var_free(void *target, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return; // LCOV_EXCL_LINE
+  if (def->fns->clear) {
+    def->fns->clear(target, def->params);
+  }
+}
+
+/**
+ * Encode a value of type <b>def</b> pointed to by <b>value</b>, and return
+ * its result in a newly allocated string.  The string may need to be escaped.
+ *
+ * Returns NULL if this option has a NULL value, or on internal error.
+ **/
+char *
+typed_var_encode(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  tor_assert(def->fns->encode);
+  return def->fns->encode(value, def->params);
+}
+
+/**
+ * As typed_var_encode(), but returns a newly allocated config_line_t
+ * object.  The provided <b>key</b> is used as the key of the lines, unless
+ * the type is one (line a linelist) that encodes its own keys.
+ *
+ * This function may return a list of multiple lines.
+ *
+ * Returns NULL if there are no lines to encode, or on internal error.
+ */
+config_line_t *
+typed_var_kvencode(const char *key, const void *value,
+                      const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return NULL; // LCOV_EXCL_LINE
+  if (def->fns->kv_encode) {
+    return def->fns->kv_encode(key, value, def->params);
+  }
+  char *encoded_value = typed_var_encode(value, def);
+  if (!encoded_value)
+    return NULL;
+
+  config_line_t *result = tor_malloc_zero(sizeof(config_line_t));
+  result->key = tor_strdup(key);
+  result->value = encoded_value;
+  return result;
+}
+
+/**
+ * Set <b>dest</b> to contain the same value as <b>src</b>. Both types
+ * must be as defined by <b>def</b>.
+ *
+ * Return 0 on success, and -1 on failure.
+ **/
+int
+typed_var_copy(void *dest, const void *src, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return -1; // LCOV_EXCL_LINE
+  if (def->fns->copy) {
+    // If we have been provided a copy fuction, use it.
+    return def->fns->copy(dest, src, def);
+  }
+
+  // Otherwise, encode 'src' and parse the result into 'def'.
+  char *enc = typed_var_encode(src, def);
+  if (!enc) {
+    typed_var_free(dest, def);
+    return 0;
+  }
+  char *err = NULL;
+  int rv = typed_var_assign(dest, enc, &err, def);
+  if (BUG(rv < 0)) {
+    // LCOV_EXCL_START
+    log_warn(LD_BUG, "Encoded value %s was not parseable as a %s: %s",
+             escaped(enc), def->name, err?err:"");
+    // LCOV_EXCL_STOP
+  }
+  tor_free(err);
+  tor_free(enc);
+  return rv;
+}
+
+/**
+ * Return true if <b>a</b> and <b>b</b> are semantically equivalent.
+ * Both types must be as defined by <b>def</b>.
+ **/
+bool
+typed_var_eq(const void *a, const void *b, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->eq) {
+    // Use a provided eq function if we got one.
+    return def->fns->eq(a, b, def->params);
+  }
+
+  // Otherwise, encode the values and compare them.
+  char *enc_a = typed_var_encode(a, def);
+  char *enc_b = typed_var_encode(b, def);
+  bool eq = !strcmp_opt(enc_a,enc_b);
+  tor_free(enc_a);
+  tor_free(enc_b);
+  return eq;
+}
+
+/**
+ * Check whether <b>value</b> encodes a valid value according to the
+ * type definition in <b>def</b>.
+ */
+bool
+typed_var_ok(const void *value, const var_type_def_t *def)
+{
+  if (BUG(!def))
+    return false; // LCOV_EXCL_LINE
+
+  if (def->fns->ok)
+    return def->fns->ok(value, def->params);
+
+  return true;
+}
+
+/**
+ * Mark <b>value</b> -- a variable that ordinarily would be extended by
+ * assignment -- as "fragile", so that it will get replaced by the next
+ * assignment instead.
+ **/
+void
+typed_var_mark_fragile(void *value, const var_type_def_t *def)
+{
+  if (BUG(!def)) {
+    return; // LCOV_EXCL_LINE
+  }
+
+  if (def->fns->mark_fragile)
+    def->fns->mark_fragile(value, def->params);
+}
diff --git a/src/lib/confmgt/typedvar.h b/src/lib/confmgt/typedvar.h
new file mode 100644
index 0000000000..cc90ed10a3
--- /dev/null
+++ b/src/lib/confmgt/typedvar.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file typedvar.h
+ * @brief Header for lib/confmgt/typedvar.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_TYPEDVAR_H
+#define TOR_LIB_CONFMGT_TYPEDVAR_H
+
+#include <stdbool.h>
+
+enum config_type_t;
+struct config_line_t;
+
+typedef struct var_type_fns_t var_type_fns_t;
+typedef struct var_type_def_t var_type_def_t;
+
+int typed_var_assign(void *target, const char *value, char **errmsg,
+                        const var_type_def_t *def);
+void typed_var_free(void *target, const var_type_def_t *def);
+char *typed_var_encode(const void *value, const var_type_def_t *def);
+int typed_var_copy(void *dest, const void *src, const var_type_def_t *def);
+bool typed_var_eq(const void *a, const void *b, const var_type_def_t *def);
+bool typed_var_ok(const void *value, const var_type_def_t *def);
+
+int typed_var_kvassign(void *target, const struct config_line_t *line,
+                           char **errmsg, const var_type_def_t *def);
+struct config_line_t *typed_var_kvencode(const char *key, const void *value,
+                                            const var_type_def_t *def);
+
+void typed_var_mark_fragile(void *value, const var_type_def_t *def);
+
+#endif /* !defined(TOR_LIB_CONFMGT_TYPEDVAR_H) */
diff --git a/src/lib/confmgt/unitparse.c b/src/lib/confmgt/unitparse.c
new file mode 100644
index 0000000000..99716e8d9d
--- /dev/null
+++ b/src/lib/confmgt/unitparse.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.c
+ * @brief Functions for parsing values with units from a configuration file.
+ **/
+
+#include "orconfig.h"
+#include "lib/confmgt/unitparse.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/parse_int.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/intmath/muldiv.h"
+
+#include <string.h>
+
+/** Table to map the names of memory units to the number of bytes they
+ * contain. */
+// clang-format off
+const struct unit_table_t memory_units[] = {
+  { "",          1 },
+  { "b",         1<< 0 },
+  { "byte",      1<< 0 },
+  { "bytes",     1<< 0 },
+  { "kb",        1<<10 },
+  { "kbyte",     1<<10 },
+  { "kbytes",    1<<10 },
+  { "kilobyte",  1<<10 },
+  { "kilobytes", 1<<10 },
+  { "kilobits",  1<<7  },
+  { "kilobit",   1<<7  },
+  { "kbits",     1<<7  },
+  { "kbit",      1<<7  },
+  { "m",         1<<20 },
+  { "mb",        1<<20 },
+  { "mbyte",     1<<20 },
+  { "mbytes",    1<<20 },
+  { "megabyte",  1<<20 },
+  { "megabytes", 1<<20 },
+  { "megabits",  1<<17 },
+  { "megabit",   1<<17 },
+  { "mbits",     1<<17 },
+  { "mbit",      1<<17 },
+  { "gb",        1<<30 },
+  { "gbyte",     1<<30 },
+  { "gbytes",    1<<30 },
+  { "gigabyte",  1<<30 },
+  { "gigabytes", 1<<30 },
+  { "gigabits",  1<<27 },
+  { "gigabit",   1<<27 },
+  { "gbits",     1<<27 },
+  { "gbit",      1<<27 },
+  { "tb",        UINT64_C(1)<<40 },
+  { "tbyte",     UINT64_C(1)<<40 },
+  { "tbytes",    UINT64_C(1)<<40 },
+  { "terabyte",  UINT64_C(1)<<40 },
+  { "terabytes", UINT64_C(1)<<40 },
+  { "terabits",  UINT64_C(1)<<37 },
+  { "terabit",   UINT64_C(1)<<37 },
+  { "tbits",     UINT64_C(1)<<37 },
+  { "tbit",      UINT64_C(1)<<37 },
+  { NULL, 0 },
+};
+// clang-format on
+
+/** Table to map the names of time units to the number of seconds they
+ * contain. */
+// clang-format off
+const struct unit_table_t time_units[] = {
+  { "",         1 },
+  { "second",   1 },
+  { "seconds",  1 },
+  { "minute",   60 },
+  { "minutes",  60 },
+  { "hour",     60*60 },
+  { "hours",    60*60 },
+  { "day",      24*60*60 },
+  { "days",     24*60*60 },
+  { "week",     7*24*60*60 },
+  { "weeks",    7*24*60*60 },
+  { "month",    2629728, }, /* about 30.437 days */
+  { "months",   2629728, },
+  { NULL, 0 },
+};
+// clang-format on
+
+/** Table to map the names of time units to the number of milliseconds
+ * they contain. */
+// clang-format off
+const struct unit_table_t time_msec_units[] = {
+  { "",         1 },
+  { "msec",     1 },
+  { "millisecond", 1 },
+  { "milliseconds", 1 },
+  { "second",   1000 },
+  { "seconds",  1000 },
+  { "minute",   60*1000 },
+  { "minutes",  60*1000 },
+  { "hour",     60*60*1000 },
+  { "hours",    60*60*1000 },
+  { "day",      24*60*60*1000 },
+  { "days",     24*60*60*1000 },
+  { "week",     7*24*60*60*1000 },
+  { "weeks",    7*24*60*60*1000 },
+  { NULL, 0 },
+};
+// clang-format on
+
+/** Parse a string <b>val</b> containing a number, zero or more
+ * spaces, and an optional unit string.  If the unit appears in the
+ * table <b>u</b>, then multiply the number by the unit multiplier.
+ * On success, set *<b>ok</b> to 1 and return this product.
+ * Otherwise, set *<b>ok</b> to 0.
+ *
+ * If an error (like overflow or a negative value is detected), put an error
+ * message in *<b>errmsg_out</b> if that pointer is non-NULL, and otherwise
+ * log a warning.
+ */
+uint64_t
+config_parse_units(const char *val, const unit_table_t *u, int *ok,
+                   char **errmsg_out)
+{
+  uint64_t v = 0;
+  double d = 0;
+  int use_float = 0;
+  char *cp;
+  char *errmsg = NULL;
+
+  tor_assert(ok);
+
+  v = tor_parse_uint64(val, 10, 0, UINT64_MAX, ok, &cp);
+  if (!*ok || (cp && *cp == '.')) {
+    d = tor_parse_double(val, 0, (double)UINT64_MAX, ok, &cp);
+    if (!*ok) {
+      tor_asprintf(&errmsg, "Unable to parse %s as a number", val);
+      goto done;
+    }
+    use_float = 1;
+  }
+
+  if (BUG(!cp)) {
+    // cp should always be non-NULL if the parse operation succeeds.
+
+    // LCOV_EXCL_START
+    *ok = 1;
+    v = use_float ? ((uint64_t)d) :  v;
+    goto done;
+    // LCOV_EXCL_STOP
+  }
+
+  cp = (char*) eat_whitespace(cp);
+
+  for ( ;u->unit;++u) {
+    if (!strcasecmp(u->unit, cp)) {
+      if (use_float) {
+        d = u->multiplier * d;
+
+        if (d < 0) {
+          tor_asprintf(&errmsg, "Got a negative value while parsing %s %s",
+                       val, u->unit);
+          *ok = 0;
+          goto done;
+        }
+
+        // Some compilers may warn about casting a double to an unsigned type
+        // because they don't know if d is >= 0
+        if (d >= 0 && (d > (double)INT64_MAX || (uint64_t)d > INT64_MAX)) {
+          tor_asprintf(&errmsg, "Overflow while parsing %s %s",
+                       val, u->unit);
+          *ok = 0;
+          goto done;
+        }
+
+        v = (uint64_t) d;
+      } else {
+        v = tor_mul_u64_nowrap(v, u->multiplier);
+
+        if (v > INT64_MAX) {
+          tor_asprintf(&errmsg, "Overflow while parsing %s %s",
+                       val, u->unit);
+          *ok = 0;
+          goto done;
+        }
+      }
+
+      *ok = 1;
+      goto done;
+    }
+  }
+  tor_asprintf(&errmsg, "Unknown unit in %s", val);
+  *ok = 0;
+ done:
+
+  if (errmsg) {
+    tor_assert_nonfatal(!*ok);
+    if (errmsg_out) {
+      *errmsg_out = errmsg;
+    } else {
+      log_warn(LD_CONFIG, "%s", errmsg);
+      tor_free(errmsg);
+    }
+  }
+
+  if (*ok)
+    return v;
+  else
+    return 0;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * information (byte, KB, M, etc).  On success, set *<b>ok</b> to true
+ * and return the number of bytes specified.  Otherwise, set
+ * *<b>ok</b> to false and return 0. */
+uint64_t
+config_parse_memunit(const char *s, int *ok)
+{
+  uint64_t u = config_parse_units(s, memory_units, ok, NULL);
+  return u;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of
+ * time in milliseconds.  On success, set *<b>ok</b> to true and return
+ * the number of milliseconds in the provided interval.  Otherwise, set
+ * *<b>ok</b> to 0 and return -1. */
+int
+config_parse_msec_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_msec_units, ok, NULL);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Msec interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
+
+/** Parse a string in the format "number unit", where unit is a unit of time.
+ * On success, set *<b>ok</b> to true and return the number of seconds in
+ * the provided interval.  Otherwise, set *<b>ok</b> to 0 and return -1.
+ */
+int
+config_parse_interval(const char *s, int *ok)
+{
+  uint64_t r;
+  r = config_parse_units(s, time_units, ok, NULL);
+  if (r > INT_MAX) {
+    log_warn(LD_CONFIG, "Interval '%s' is too long", s);
+    *ok = 0;
+    return -1;
+  }
+  return (int)r;
+}
diff --git a/src/lib/confmgt/unitparse.h b/src/lib/confmgt/unitparse.h
new file mode 100644
index 0000000000..047e11b424
--- /dev/null
+++ b/src/lib/confmgt/unitparse.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file unitparse.h
+ * @brief Header for lib/confmgt/unitparse.c
+ **/
+
+#ifndef TOR_LIB_CONFMGT_UNITPARSE_H
+#define TOR_LIB_CONFMGT_UNITPARSE_H
+
+#include <lib/cc/torint.h>
+
+/** Mapping from a unit name to a multiplier for converting that unit into a
+ * base unit.  Used by config_parse_unit. */
+typedef struct unit_table_t {
+  const char *unit; /**< The name of the unit */
+  uint64_t multiplier; /**< How many of the base unit appear in this unit */
+} unit_table_t;
+
+extern const unit_table_t memory_units[];
+extern const unit_table_t time_units[];
+extern const struct unit_table_t time_msec_units[];
+
+uint64_t config_parse_units(const char *val, const unit_table_t *u, int *ok,
+                            char **errmsg_out);
+
+uint64_t config_parse_memunit(const char *s, int *ok);
+int config_parse_msec_interval(const char *s, int *ok);
+int config_parse_interval(const char *s, int *ok);
+
+#endif /* !defined(TOR_LIB_CONFMGT_UNITPARSE_H) */
diff --git a/src/lib/confmgt/var_type_def_st.h b/src/lib/confmgt/var_type_def_st.h
new file mode 100644
index 0000000000..2519b86aa0
--- /dev/null
+++ b/src/lib/confmgt/var_type_def_st.h
@@ -0,0 +1,167 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file var_type_def_st.h
+ * @brief Structure declarations for typedvar type definitions.
+ *
+ * This structure is used for defining new variable types.  If you are not
+ * defining a new variable type for use by the configuration management
+ * system, you don't need this structure.
+ *
+ * For defining new variables, see the types in conftypes.h.
+ *
+ * For data-driven access to configuration variables, see the other members of
+ * lib/confmgt/.
+ *
+ * STATUS NOTE: It is not yet possible to actually define new variables
+ *   outside of config.c, and many of the types that will eventually be used
+ *   to do so are not yet moved. This will change as more of #29211 is
+ *   completed.
+ **/
+
+#ifndef TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+#define TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H
+
+#include <stdbool.h>
+
+struct config_line_t;
+
+/**
+ * A structure full of functions pointers to implement a variable type.
+ *
+ * Every type MUST implement parse or kv_parse and encode or kv_encode;
+ * the other functions pointers MAY be NULL.
+ *
+ * All functions here take a <b>params</b> argument, whose value
+ * is determined by the type definition.  Two types may have the
+ * same functions, but differ only in parameters.
+ *
+ * Implementation considerations: If "" encodes a valid value for a type, try
+ * to make sure that it encodes the same thing as the default value for the
+ * type (that is, the value that is set by config_clear() or memset(0)). If
+ * this is not the case, you need to make extra certain that your parse/encode
+ * implementations preserve the NULL/"" distinction.
+ **/
+struct var_type_fns_t {
+  /**
+   * Try to parse a string in <b>value</b> that encodes an object of this
+   * type.  On success, adjust the lvalue pointed to by <b>target</b> to hold
+   * that value, and return 0.  On failure, set *<b>errmsg</b> to a newly
+   * allocated string holding an error message, and return -1.
+   **/
+  int (*parse)(void *target, const char *value, char **errmsg,
+               const void *params);
+  /**
+   * Try to parse a single line from the head of<b>line</b> that encodes
+   * an object of this type.  On success and failure, behave as in the parse()
+   * function.
+   *
+   * If this function is absent, it is implemented in terms of parse().
+   *
+   * All types for which keys are significant should use this method. For
+   * example, a "linelist" type records the actual keys that are given
+   * for each line, and so should use this method.
+   *
+   * Note that although multiple lines may be provided in <b>line</b>,
+   * only the first one should be handled by this function.
+   **/
+  int (*kv_parse)(void *target, const struct config_line_t *line,
+                  char **errmsg, const void *params);
+  /**
+   * Encode a value pointed to by <b>value</b> and return its result
+   * in a newly allocated string.  The string may need to be escaped.
+   *
+   * If this function is absent, it is implemented in terms of kv_encode().
+   *
+   * Returns NULL if this option has a NULL value, or on internal error.
+   *
+   * Requirement: all strings generated by encode() should produce a
+   * semantically equivalent value when given to parse().
+   **/
+  char *(*encode)(const void *value, const void *params);
+  /**
+   * As encode(), but returns a newly allocated config_line_t object.  The
+   * provided <b>key</b> is used as the key of the lines, unless the type is
+   * one that encodes its own keys.
+   *
+   * Unlike kv_parse(), this function will return a list of multiple lines,
+   * if <b>value</b> is such that it must be encoded by multiple lines.
+   *
+   * Returns NULL if there are no lines to encode, or on internal error.
+   *
+   * If this function is absent, it is implemented in terms of encode().
+   **/
+  struct config_line_t *(*kv_encode)(const char *key, const void *value,
+                                     const void *params);
+  /**
+   * Free all storage held in <b>arg</b>, and set <b>arg</b> to a default
+   * value -- usually zero or NULL.
+   *
+   * If this function is absent, the default implementation does nothing.
+   **/
+  void (*clear)(void *arg, const void *params);
+  /**
+   * Return true if <b>a</b> and <b>b</b> hold the same value, and false
+   * otherwise.
+   *
+   * If this function is absent, it is implemented by encoding both a and
+   * b and comparing their encoded strings for equality.
+   **/
+  bool (*eq)(const void *a, const void *b, const void *params);
+  /**
+   * Try to copy the value from <b>value</b> into <b>target</b>.
+   * On success return 0; on failure return -1.
+   *
+   * If this function is absent, it is implemented by encoding the value
+   * into a string, and then parsing it into the target.
+   **/
+  int (*copy)(void *target, const void *value, const void *params);
+  /**
+   * Check whether <b>value</b> holds a valid value according to the
+   * rules of this type; return true if it does and false if it doesn't.
+   *
+   * The default implementation for this function assumes that all
+   * values are valid.
+   **/
+  bool (*ok)(const void *value, const void *params);
+  /**
+   * Mark a value of this variable as "fragile", so that future attempts to
+   * assign to this variable will replace rather than extending it.
+   *
+   * The default implementation for this function does nothing.
+   *
+   * Only meaningful for types with is_cumulative set.
+   **/
+  void (*mark_fragile)(void *value, const void *params);
+};
+
+/**
+ * A structure describing a type that can be manipulated with the typedvar_*
+ * functions.
+ **/
+struct var_type_def_t {
+  /**
+   * The name of this type. Should not include spaces. Used for
+   * debugging, log messages, and the controller API. */
+  const char *name;
+  /**
+   * A function table for this type.
+   */
+  const struct var_type_fns_t *fns;
+  /**
+   * A pointer to a value that should be passed as the 'params' argument when
+   * calling the functions in this type's function table.
+   */
+  const void *params;
+  /**
+   * A bitwise OR of one or more VTFLAG_* values, describing properties
+   * for all values of this type.
+   **/
+  uint32_t flags;
+};
+
+#endif /* !defined(TOR_LIB_CONFMGT_VAR_TYPE_DEF_ST_H) */
diff --git a/src/lib/container/.may_include b/src/lib/container/.may_include
index 90de5eda40..81507527d3 100644
--- a/src/lib/container/.may_include
+++ b/src/lib/container/.may_include
@@ -7,12 +7,9 @@ lib/malloc/*.h
 lib/err/*.h
 lib/smartlist_core/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/intmath/*.h
 lib/log/*.h
 
-# XXXX I am unsure about this one. It's only here for buffers.c
-lib/time/*.h
-
-ht.h
-siphash.h
+ext/ht.h
+ext/siphash.h
diff --git a/src/lib/container/bitarray.h b/src/lib/container/bitarray.h
index 910d5fea65..41409e350a 100644
--- a/src/lib/container/bitarray.h
+++ b/src/lib/container/bitarray.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BITARRAY_H
@@ -83,4 +83,4 @@ bitarray_is_set(bitarray_t *b, int bit)
   return b[bit >> BITARRAY_SHIFT] & (1u << (bit & BITARRAY_MASK));
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_BITARRAY_H) */
diff --git a/src/lib/container/bloomfilt.c b/src/lib/container/bloomfilt.c
index 9aa9b1ee56..34b1265d81 100644
--- a/src/lib/container/bloomfilt.c
+++ b/src/lib/container/bloomfilt.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,7 +14,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/intmath/bits.h"
 #include "lib/log/util_bug.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** How many bloom-filter bits we set per address. This is twice the
  * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit
diff --git a/src/lib/container/bloomfilt.h b/src/lib/container/bloomfilt.h
index 0ce18bd3ec..6d36056b5a 100644
--- a/src/lib/container/bloomfilt.h
+++ b/src/lib/container/bloomfilt.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BLOOMFILT_H
diff --git a/src/lib/container/handles.h b/src/lib/container/handles.h
index ca7c94559e..6b1bbd5167 100644
--- a/src/lib/container/handles.h
+++ b/src/lib/container/handles.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,33 +16,33 @@
  * To enable a type to have handles, add a HANDLE_ENTRY() field in its
  * definition, as in:
  *
- *     struct walrus {
- *         HANDLE_ENTRY(wlr, walrus);
+ *     struct walrus_t {
+ *         HANDLE_ENTRY(wlr, walrus_t);
  *         // ...
  *     };
  *
- * And invoke HANDLE_DECL(wlr, walrus, [static]) to declare the handle
+ * And invoke HANDLE_DECL(wlr, walrus_t, [static]) to declare the handle
  * manipulation functions (typically in a header):
  *
  *     // opaque handle to walrus.
  *     typedef struct wlr_handle_t wlr_handle_t;
  *
  *     // make a new handle
- *     struct wlr_handle_t *wlr_handle_new(struct walrus *);
+ *     struct wlr_handle_t *wlr_handle_new(struct walrus_t *);
  *
  *     // release a handle
  *     void wlr_handle_free(wlr_handle_t *);
  *
  *     // return the pointed-to walrus, or NULL.
- *     struct walrus *wlr_handle_get(wlr_handle_t *).
+ *     struct walrus_t *wlr_handle_get(wlr_handle_t *).
  *
  *     // call this function when you're about to free the walrus;
  *     // it invalidates all handles. (IF YOU DON'T, YOU WILL HAVE
  *     // DANGLING REFERENCES)
- *     void wlr_handles_clear(struct walrus *);
+ *     void wlr_handles_clear(struct walrus_t *);
  *
  * Finally, use HANDLE_IMPL() to define the above functions in some
- * appropriate C file: HANDLE_IMPL(wlr, walrus, [static])
+ * appropriate C file: HANDLE_IMPL(wlr, walrus_t, [static])
  *
  **/
 
@@ -57,12 +57,13 @@
 #define HANDLE_ENTRY(name, structname)         \
   struct name ## _handle_head_t *handle_head
 
-#define HANDLE_DECL(name, structname, linkage)                          \
+#define HANDLE_DECL(name, structname_t, linkage)                        \
   typedef struct name ## _handle_t name ## _handle_t;                   \
-  linkage  name ## _handle_t *name ## _handle_new(struct structname *object); \
+  linkage name ## _handle_t *name ## _handle_new(                       \
+                                          struct structname_t *object); \
   linkage void name ## _handle_free_(name ## _handle_t *);              \
-  linkage struct structname *name ## _handle_get(name ## _handle_t *);  \
-  linkage void name ## _handles_clear(struct structname *object);
+  linkage struct structname_t *name ## _handle_get(name ## _handle_t *); \
+  linkage void name ## _handles_clear(struct structname_t *object);
 
 /*
  * Implementation notes: there are lots of possible implementations here.  We
diff --git a/src/lib/container/include.am b/src/lib/container/include.am
index e6492098b5..00d7b8e587 100644
--- a/src/lib/container/include.am
+++ b/src/lib/container/include.am
@@ -5,10 +5,11 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-container-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_container_a_SOURCES =			\
 	src/lib/container/bloomfilt.c			\
-	src/lib/container/buffers.c			\
 	src/lib/container/map.c				\
+	src/lib/container/namemap.c			\
 	src/lib/container/order.c			\
 	src/lib/container/smartlist.c
 
@@ -17,11 +18,13 @@ src_lib_libtor_container_testing_a_SOURCES = \
 src_lib_libtor_container_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/container/bitarray.h			\
 	src/lib/container/bloomfilt.h			\
-	src/lib/container/buffers.h			\
 	src/lib/container/handles.h			\
 	src/lib/container/map.h				\
+	src/lib/container/namemap.h			\
+	src/lib/container/namemap_st.h			\
 	src/lib/container/order.h			\
 	src/lib/container/smartlist.h
diff --git a/src/lib/container/lib_container.md b/src/lib/container/lib_container.md
new file mode 100644
index 0000000000..f4902ca44a
--- /dev/null
+++ b/src/lib/container/lib_container.md
@@ -0,0 +1,49 @@
+@dir /lib/container
+@brief lib/container: Hash tables, dynamic arrays, bit arrays, etc.
+
+### Smartlists: Neither lists, nor especially smart.
+
+For historical reasons, we call our dynamic-allocated array type
+`smartlist_t`.  It can grow or shrink as elements are added and removed.
+
+All smartlists hold an array of `void *`.  Whenever you expose a smartlist
+in an API you *must* document which types its pointers actually hold.
+
+<!-- It would be neat to fix that, wouldn't it? -NM  -->
+
+Smartlists are created empty with `smartlist_new()` and freed with
+`smartlist_free()`.  See the `containers.h` header documentation for more
+information; there are many convenience functions for commonly needed
+operations.
+
+For low-level operations on smartlists, see also
+\refdir{lib/smartlist_core}.
+
+<!-- TODO: WRITE more about what you can do with smartlists. -->
+
+### Digest maps, string maps, and more.
+
+Tor makes frequent use of maps from 160-bit digests, 256-bit digests,
+or nul-terminated strings to `void *`. These types are `digestmap_t`,
+`digest256map_t`, and `strmap_t` respectively.  See the containers.h
+module documentation for more information.
+
+### Intrusive lists and hashtables
+
+For performance-sensitive cases, we sometimes want to use "intrusive"
+collections: ones where the bookkeeping pointers are stuck inside the
+structures that belong to the collection.  If you've used the
+BSD-style sys/queue.h macros, you'll be familiar with these.
+
+Unfortunately, the `sys/queue.h` macros vary significantly between the
+platforms that have them, so we provide our own variants in
+`ext/tor_queue.h`.
+
+We also provide an intrusive hashtable implementation in `ext/ht.h`.
+When you're using it, you'll need to define your own hash
+functions. If attacker-induced collisions are a worry here, use the
+cryptographic siphash24g function to extract hashes.
+
+<!-- TODO: WRITE about bloom filters, namemaps, bit-arrays, order functions.
+-->
+
diff --git a/src/lib/container/map.c b/src/lib/container/map.c
index d213ad50bf..7db84313ea 100644
--- a/src/lib/container/map.c
+++ b/src/lib/container/map.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,7 +21,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "ht.h"
+#include "ext/ht.h"
 
 /** Helper: Declare an entry type and a map type to implement a mapping using
  * ht.h.  The map type will be called <b>maptype</b>.  The key part of each
@@ -85,21 +85,21 @@ digest256map_entry_hash(const digest256map_entry_t *a)
 }
 
 HT_PROTOTYPE(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq)
+             strmap_entries_eq);
 HT_GENERATE2(strmap_impl, strmap_entry_t, node, strmap_entry_hash,
-             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             strmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 HT_PROTOTYPE(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq)
+             digestmap_entries_eq);
 HT_GENERATE2(digestmap_impl, digestmap_entry_t, node, digestmap_entry_hash,
-             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             digestmap_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 HT_PROTOTYPE(digest256map_impl, digest256map_entry_t, node,
              digest256map_entry_hash,
-             digest256map_entries_eq)
+             digest256map_entries_eq);
 HT_GENERATE2(digest256map_impl, digest256map_entry_t, node,
              digest256map_entry_hash,
-             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_)
+             digest256map_entries_eq, 0.6, tor_reallocarray_, tor_free_);
 
 #define strmap_entry_free(ent) \
   FREE_AND_NULL(strmap_entry_t, strmap_entry_free_, (ent))
diff --git a/src/lib/container/map.h b/src/lib/container/map.h
index a2d1b01d12..dbc1967247 100644
--- a/src/lib/container/map.h
+++ b/src/lib/container/map.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_MAP_H
@@ -15,36 +15,37 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/cc/torint.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
-#define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##entry_t *prefix##iter_t;                       \
-  MOCK_DECL(maptype*, prefix##new, (void));                             \
-  void* prefix##set(maptype *map, keytype key, void *val);              \
-  void* prefix##get(const maptype *map, keytype key);                   \
-  void* prefix##remove(maptype *map, keytype key);                      \
-  MOCK_DECL(void, prefix##free_, (maptype *map, void (*free_val)(void*))); \
-  int prefix##isempty(const maptype *map);                              \
-  int prefix##size(const maptype *map);                                 \
-  prefix##iter_t *prefix##iter_init(maptype *map);                      \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter); \
-  prefix##iter_t *prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter); \
-  void prefix##iter_get(prefix##iter_t *iter, keytype *keyp, void **valp); \
-  int prefix##iter_done(prefix##iter_t *iter);                          \
-  void prefix##assert_ok(const maptype *map)
+#define DECLARE_MAP_FNS(mapname_t, keytype, prefix)                     \
+  typedef struct mapname_t mapname_t;                                   \
+  typedef struct prefix##_entry_t *prefix##_iter_t;                       \
+  MOCK_DECL(mapname_t*, prefix##_new, (void));                           \
+  void* prefix##_set(mapname_t *map, keytype key, void *val);            \
+  void* prefix##_get(const mapname_t *map, keytype key);                 \
+  void* prefix##_remove(mapname_t *map, keytype key);                    \
+  MOCK_DECL(void, prefix##_free_, (mapname_t *map, void (*free_val)(void*))); \
+  int prefix##_isempty(const mapname_t *map);                            \
+  int prefix##_size(const mapname_t *map);                               \
+  prefix##_iter_t *prefix##_iter_init(mapname_t *map);                    \
+  prefix##_iter_t *prefix##_iter_next(mapname_t *map, prefix##_iter_t *iter); \
+  prefix##_iter_t *prefix##_iter_next_rmv(mapname_t *map,                 \
+                                        prefix##_iter_t *iter);          \
+  void prefix##_iter_get(prefix##_iter_t *iter, keytype *keyp, void **valp); \
+  int prefix##_iter_done(prefix##_iter_t *iter);                          \
+  void prefix##_assert_ok(const mapname_t *map)
 
 /* Map from const char * to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(strmap_t, const char *, strmap_);
+DECLARE_MAP_FNS(strmap_t, const char *, strmap);
 /* Map from const char[DIGEST_LEN] to void *. Implemented with a hash table. */
-DECLARE_MAP_FNS(digestmap_t, const char *, digestmap_);
+DECLARE_MAP_FNS(digestmap_t, const char *, digestmap);
 /* Map from const uint8_t[DIGEST256_LEN] to void *. Implemented with a hash
  * table. */
-DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
+DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map);
 
-#define MAP_FREE_AND_NULL(maptype, map, fn)     \
+#define MAP_FREE_AND_NULL(mapname_t, map, fn)     \
   do {                                          \
-    maptype ## _free_((map), (fn));             \
+    mapname_t ## _free_((map), (fn));             \
     (map) = NULL;                               \
   } while (0)
 
@@ -55,12 +56,12 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
 #undef DECLARE_MAP_FNS
 
 /** Iterates over the key-value pairs in a map <b>map</b> in order.
- * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap_ or digestmap_).
+ * <b>prefix</b> is as for DECLARE_MAP_FNS (i.e., strmap or digestmap).
  * The map's keys and values are of type keytype and valtype respectively;
  * each iteration assigns them to keyvar and valvar.
  *
  * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *   MAP_FOREACH(digestmap, m, const char *, k, routerinfo_t *, r) {
  *     // use k and r
  *   } MAP_FOREACH_END.
  */
@@ -80,21 +81,21 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  */
 #define MAP_FOREACH(prefix, map, keytype, keyvar, valtype, valvar)      \
   STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
-         keyvar##_iter = prefix##iter_next(map, keyvar##_iter)) {       \
+    prefix##_iter_t *keyvar##_iter;                                      \
+    for (keyvar##_iter = prefix##_iter_init(map);                        \
+         !prefix##_iter_done(keyvar##_iter);                             \
+         keyvar##_iter = prefix##_iter_next(map, keyvar##_iter)) {       \
       keytype keyvar;                                                   \
       void *valvar##_voidp;                                             \
       valtype valvar;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      prefix##_iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
       valvar = valvar##_voidp;
 
 /** As MAP_FOREACH, except allows members to be removed from the map
  * during the iteration via MAP_DEL_CURRENT.  Example use:
  *
  * Example use:
- *   MAP_FOREACH(digestmap_, m, const char *, k, routerinfo_t *, r) {
+ *   MAP_FOREACH(digestmap, m, const char *, k, routerinfo_t *, r) {
  *      if (is_very_old(r))
  *       MAP_DEL_CURRENT(k);
  *   } MAP_FOREACH_END.
@@ -120,18 +121,18 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  */
 #define MAP_FOREACH_MODIFY(prefix, map, keytype, keyvar, valtype, valvar) \
   STMT_BEGIN                                                            \
-    prefix##iter_t *keyvar##_iter;                                      \
+    prefix##_iter_t *keyvar##_iter;                                      \
     int keyvar##_del=0;                                                 \
-    for (keyvar##_iter = prefix##iter_init(map);                        \
-         !prefix##iter_done(keyvar##_iter);                             \
+    for (keyvar##_iter = prefix##_iter_init(map);                        \
+         !prefix##_iter_done(keyvar##_iter);                             \
          keyvar##_iter = keyvar##_del ?                                 \
-           prefix##iter_next_rmv(map, keyvar##_iter) :                  \
-           prefix##iter_next(map, keyvar##_iter)) {                     \
+           prefix##_iter_next_rmv(map, keyvar##_iter) :                  \
+           prefix##_iter_next(map, keyvar##_iter)) {                     \
       keytype keyvar;                                                   \
       void *valvar##_voidp;                                             \
       valtype valvar;                                                   \
       keyvar##_del=0;                                                   \
-      prefix##iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
+      prefix##_iter_get(keyvar##_iter, &keyvar, &valvar##_voidp);        \
       valvar = valvar##_voidp;
 
 /** Used with MAP_FOREACH_MODIFY to remove the currently-iterated-upon
@@ -151,7 +152,7 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  *   } DIGESTMAP_FOREACH_END.
  */
 #define DIGESTMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(digestmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH(digestmap, map, const char *, keyvar, valtype, valvar)
 
 /** As MAP_FOREACH_MODIFY, but does not require declaration of prefix or
  * keytype.
@@ -162,89 +163,89 @@ DECLARE_MAP_FNS(digest256map_t, const uint8_t *, digest256map_);
  *   } DIGESTMAP_FOREACH_END.
  */
 #define DIGESTMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(digestmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH_MODIFY(digestmap, map, const char *, keyvar, valtype, valvar)
 /** Used to end a DIGESTMAP_FOREACH() block. */
 #define DIGESTMAP_FOREACH_END MAP_FOREACH_END
 
 #define DIGEST256MAP_FOREACH(map, keyvar, valtype, valvar)               \
-  MAP_FOREACH(digest256map_, map, const uint8_t *, keyvar, valtype, valvar)
+  MAP_FOREACH(digest256map, map, const uint8_t *, keyvar, valtype, valvar)
 #define DIGEST256MAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)       \
-  MAP_FOREACH_MODIFY(digest256map_, map, const uint8_t *,               \
+  MAP_FOREACH_MODIFY(digest256map, map, const uint8_t *,               \
                      keyvar, valtype, valvar)
 #define DIGEST256MAP_FOREACH_END MAP_FOREACH_END
 
 #define STRMAP_FOREACH(map, keyvar, valtype, valvar)                 \
-  MAP_FOREACH(strmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH(strmap, map, const char *, keyvar, valtype, valvar)
 #define STRMAP_FOREACH_MODIFY(map, keyvar, valtype, valvar)          \
-  MAP_FOREACH_MODIFY(strmap_, map, const char *, keyvar, valtype, valvar)
+  MAP_FOREACH_MODIFY(strmap, map, const char *, keyvar, valtype, valvar)
 #define STRMAP_FOREACH_END MAP_FOREACH_END
 
 void* strmap_set_lc(strmap_t *map, const char *key, void *val);
 void* strmap_get_lc(const strmap_t *map, const char *key);
 void* strmap_remove_lc(strmap_t *map, const char *key);
 
-#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, maptype, valtype)           \
-  typedef struct maptype maptype;                                       \
-  typedef struct prefix##iter_t *prefix##iter_t;                        \
-  ATTR_UNUSED static inline maptype*                                    \
-  prefix##new(void)                                                     \
+#define DECLARE_TYPED_DIGESTMAP_FNS(prefix, mapname_t, valtype)           \
+  typedef struct mapname_t mapname_t;                                   \
+  typedef struct prefix##_iter_t *prefix##_iter_t;                        \
+  ATTR_UNUSED static inline mapname_t*                                  \
+  prefix##_new(void)                                                     \
   {                                                                     \
-    return (maptype*)digestmap_new();                                   \
+    return (mapname_t*)digestmap_new();                                 \
   }                                                                     \
   ATTR_UNUSED static inline digestmap_t*                                \
-  prefix##to_digestmap(maptype *map)                                    \
+  prefix##_to_digestmap(mapname_t *map)                                  \
   {                                                                     \
     return (digestmap_t*)map;                                           \
   }                                                                     \
   ATTR_UNUSED static inline valtype*                                    \
-  prefix##get(maptype *map, const char *key)     \
+  prefix##_get(mapname_t *map, const char *key)                          \
   {                                                                     \
     return (valtype*)digestmap_get((digestmap_t*)map, key);             \
   }                                                                     \
   ATTR_UNUSED static inline valtype*                                    \
-  prefix##set(maptype *map, const char *key, valtype *val)              \
+  prefix##_set(mapname_t *map, const char *key, valtype *val)            \
   {                                                                     \
     return (valtype*)digestmap_set((digestmap_t*)map, key, val);        \
   }                                                                     \
   ATTR_UNUSED static inline valtype*                                    \
-  prefix##remove(maptype *map, const char *key)                         \
+  prefix##_remove(mapname_t *map, const char *key)                       \
   {                                                                     \
     return (valtype*)digestmap_remove((digestmap_t*)map, key);          \
   }                                                                     \
   ATTR_UNUSED static inline void                                        \
-  prefix##f##ree_(maptype *map, void (*free_val)(void*))                \
+  prefix##_f##ree_(mapname_t *map, void (*free_val)(void*))              \
   {                                                                     \
     digestmap_free_((digestmap_t*)map, free_val);                       \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
-  prefix##isempty(maptype *map)                                         \
+  prefix##_isempty(mapname_t *map)                                       \
   {                                                                     \
     return digestmap_isempty((digestmap_t*)map);                        \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
-  prefix##size(maptype *map)                                            \
+  prefix##_size(mapname_t *map)                                          \
   {                                                                     \
     return digestmap_size((digestmap_t*)map);                           \
   }                                                                     \
   ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_init(maptype *map)                       \
+  prefix##_iter_t *prefix##_iter_init(mapname_t *map)                     \
   {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_init((digestmap_t*)map);    \
+    return (prefix##_iter_t*) digestmap_iter_init((digestmap_t*)map);    \
   }                                                                     \
   ATTR_UNUSED static inline                                             \
-  prefix##iter_t *prefix##iter_next(maptype *map, prefix##iter_t *iter) \
+  prefix##_iter_t *prefix##_iter_next(mapname_t *map, prefix##_iter_t *iter) \
   {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next(                       \
+    return (prefix##_iter_t*) digestmap_iter_next(                       \
                        (digestmap_t*)map, (digestmap_iter_t*)iter);     \
   }                                                                     \
-  ATTR_UNUSED static inline prefix##iter_t*                             \
-  prefix##iter_next_rmv(maptype *map, prefix##iter_t *iter)             \
+  ATTR_UNUSED static inline prefix##_iter_t*                             \
+  prefix##_iter_next_rmv(mapname_t *map, prefix##_iter_t *iter)           \
   {                                                                     \
-    return (prefix##iter_t*) digestmap_iter_next_rmv(                   \
+    return (prefix##_iter_t*) digestmap_iter_next_rmv(                   \
                        (digestmap_t*)map, (digestmap_iter_t*)iter);     \
   }                                                                     \
   ATTR_UNUSED static inline void                                        \
-  prefix##iter_get(prefix##iter_t *iter,                                \
+  prefix##_iter_get(prefix##_iter_t *iter,                                \
                    const char **keyp,                                   \
                    valtype **valp)                                      \
   {                                                                     \
@@ -253,9 +254,9 @@ void* strmap_remove_lc(strmap_t *map, const char *key);
     *valp = v;                                                          \
   }                                                                     \
   ATTR_UNUSED static inline int                                         \
-  prefix##iter_done(prefix##iter_t *iter)                               \
+  prefix##_iter_done(prefix##_iter_t *iter)                               \
   {                                                                     \
     return digestmap_iter_done((digestmap_iter_t*)iter);                \
   }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_MAP_H) */
diff --git a/src/lib/container/namemap.c b/src/lib/container/namemap.c
new file mode 100644
index 0000000000..e286cad947
--- /dev/null
+++ b/src/lib/container/namemap.c
@@ -0,0 +1,189 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file namemap.c
+ * @brief Mappings between identifiers and 16-bit ints.
+ **/
+
+#include "orconfig.h"
+#include "lib/container/smartlist.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "ext/siphash.h"
+
+#include <string.h>
+
+/** Helper for namemap hashtable implementation: compare two entries. */
+static inline int
+mapped_name_eq(const mapped_name_t *a, const mapped_name_t *b)
+{
+  return !strcmp(a->name, b->name);
+}
+
+/** Helper for namemap hashtable implementation: hash an entry. */
+static inline unsigned
+mapped_name_hash(const mapped_name_t *a)
+{
+  return (unsigned) siphash24g(a->name, strlen(a->name));
+}
+
+HT_PROTOTYPE(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq);
+HT_GENERATE2(namemap_ht, mapped_name_t, node, mapped_name_hash,
+             mapped_name_eq, 0.6, tor_reallocarray_, tor_free_);
+
+/** Set up an uninitialized <b>map</b>. */
+void
+namemap_init(namemap_t *map)
+{
+  memset(map, 0, sizeof(*map));
+  HT_INIT(namemap_ht, &map->ht);
+  map->names = smartlist_new();
+}
+
+/** Return the name that <b>map</b> associates with a given <b>id</b>, or
+ * NULL if there is no such name. */
+const char *
+namemap_get_name(const namemap_t *map, unsigned id)
+{
+  if (map->names && id < (unsigned)smartlist_len(map->names)) {
+    mapped_name_t *name = smartlist_get(map->names, (int)id);
+    return name->name;
+  } else {
+    return NULL;
+  }
+}
+
+/**
+ * Return the name that <b>map</b> associates with a given <b>id</b>, or a
+ * pointer to a statically allocated string describing the value of <b>id</b>
+ * if no such name exists.
+ **/
+const char *
+namemap_fmt_name(const namemap_t *map, unsigned id)
+{
+  static char buf[32];
+
+  const char *name = namemap_get_name(map, id);
+  if (name)
+    return name;
+
+  tor_snprintf(buf, sizeof(buf), "{%u}", id);
+
+  return buf;
+}
+
+/**
+ * Helper: As namemap_get_id(), but requires that <b>name</b> is
+ * <b>namelen</b> charaters long, and that <b>namelen</b> is no more than
+ * MAX_NAMEMAP_NAME_LEN.
+ */
+static unsigned
+namemap_get_id_unchecked(const namemap_t *map,
+                         const char *name,
+                         size_t namelen)
+{
+  union {
+    mapped_name_t n;
+    char storage[MAX_NAMEMAP_NAME_LEN + sizeof(mapped_name_t) + 1];
+  } u;
+  memcpy(u.n.name, name, namelen);
+  u.n.name[namelen] = 0;
+  const mapped_name_t *found = HT_FIND(namemap_ht, &map->ht, &u.n);
+  if (found) {
+    tor_assert(map->names);
+    tor_assert(smartlist_get(map->names, found->intval) == found);
+    return found->intval;
+  }
+
+  return NAMEMAP_ERR;
+}
+
+/**
+ * Return the identifier currently associated by <b>map</b> with the name
+ * <b>name</b>, or NAMEMAP_ERR if no such identifier exists.
+ **/
+unsigned
+namemap_get_id(const namemap_t *map,
+               const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  return namemap_get_id_unchecked(map, name, namelen);
+}
+
+/**
+ * Return the identifier associated by <b>map</b> with the name
+ * <b>name</b>, allocating a new identifier in <b>map</b> if none exists.
+ *
+ * Return NAMEMAP_ERR if <b>name</b> is too long, or if there are no more
+ * identifiers we can allocate.
+ **/
+unsigned
+namemap_get_or_create_id(namemap_t *map,
+                         const char *name)
+{
+  size_t namelen = strlen(name);
+  if (namelen > MAX_NAMEMAP_NAME_LEN) {
+    return NAMEMAP_ERR;
+  }
+
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    map->names = smartlist_new();
+
+  unsigned found = namemap_get_id_unchecked(map, name, namelen);
+  if (found != NAMEMAP_ERR)
+    return found;
+
+  unsigned new_id = (unsigned)smartlist_len(map->names);
+  if (new_id == NAMEMAP_ERR)
+    return NAMEMAP_ERR; /* Can't allocate any more. */
+
+  mapped_name_t *insert = tor_malloc_zero(
+                       offsetof(mapped_name_t, name) + namelen + 1);
+  memcpy(insert->name, name, namelen+1);
+  insert->intval = new_id;
+
+  HT_INSERT(namemap_ht, &map->ht, insert);
+  smartlist_add(map->names, insert);
+
+  return new_id;
+}
+
+/** Return the number of entries in 'names' */
+size_t
+namemap_get_size(const namemap_t *map)
+{
+  if (PREDICT_UNLIKELY(map->names == NULL))
+    return 0;
+
+  return smartlist_len(map->names);
+}
+
+/**
+ * Release all storage held in <b>map</b>.
+ */
+void
+namemap_clear(namemap_t *map)
+{
+  if (!map)
+    return;
+
+  HT_CLEAR(namemap_ht, &map->ht);
+  if (map->names) {
+    SMARTLIST_FOREACH(map->names, mapped_name_t *, n,
+                      tor_free(n));
+    smartlist_free(map->names);
+  }
+  memset(map, 0, sizeof(*map));
+}
diff --git a/src/lib/container/namemap.h b/src/lib/container/namemap.h
new file mode 100644
index 0000000000..b451c18c68
--- /dev/null
+++ b/src/lib/container/namemap.h
@@ -0,0 +1,35 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_NAMEMAP_H
+#define TOR_NAMEMAP_H
+
+/**
+ * \file namemap.h
+ *
+ * \brief Header for namemap.c
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+#include <stddef.h>
+
+typedef struct namemap_t namemap_t;
+
+/** Returned in place of an identifier when an error occurs. */
+#define NAMEMAP_ERR UINT_MAX
+
+void namemap_init(namemap_t *map);
+const char *namemap_get_name(const namemap_t *map, unsigned id);
+const char *namemap_fmt_name(const namemap_t *map, unsigned id);
+unsigned namemap_get_id(const namemap_t *map,
+                        const char *name);
+unsigned namemap_get_or_create_id(namemap_t *map,
+                                  const char *name);
+size_t namemap_get_size(const namemap_t *map);
+void namemap_clear(namemap_t *map);
+
+#endif /* !defined(TOR_NAMEMAP_H) */
diff --git a/src/lib/container/namemap_st.h b/src/lib/container/namemap_st.h
new file mode 100644
index 0000000000..39aa85cc09
--- /dev/null
+++ b/src/lib/container/namemap_st.h
@@ -0,0 +1,41 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef NAMEMAP_ST_H
+#define NAMEMAP_ST_H
+
+/**
+ * @file namemap_st.h
+ * @brief Internal declarations for namemap structure.
+ **/
+
+#include "lib/cc/compat_compiler.h"
+#include "ext/ht.h"
+
+struct smartlist_t;
+
+/** Longest allowed name that's allowed in a namemap_t. */
+#define MAX_NAMEMAP_NAME_LEN 128
+
+/** An entry inside a namemap_t. Maps a string to a numeric identifier. */
+typedef struct mapped_name_t {
+  HT_ENTRY(mapped_name_t) node;
+  unsigned intval;
+  char name[FLEXIBLE_ARRAY_MEMBER];
+} mapped_name_t;
+
+/** A structure that allocates small numeric identifiers for names and maps
+ * back and forth between them.  */
+struct namemap_t {
+  HT_HEAD(namemap_ht, mapped_name_t) ht;
+  struct smartlist_t *names;
+};
+
+#ifndef COCCI
+/** Macro to initialize a namemap. */
+#define NAMEMAP_INIT() { HT_INITIALIZER(), NULL }
+#endif
+
+#endif /* !defined(NAMEMAP_ST_H) */
diff --git a/src/lib/container/order.c b/src/lib/container/order.c
index f6503a124e..cac241f027 100644
--- a/src/lib/container/order.c
+++ b/src/lib/container/order.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/container/order.h b/src/lib/container/order.h
index a176d6d8a6..5bca095f35 100644
--- a/src/lib/container/order.h
+++ b/src/lib/container/order.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_ORDER_H
@@ -57,4 +57,4 @@ third_quartile_uint32(uint32_t *array, int n_elements)
   return find_nth_uint32(array, n_elements, (n_elements*3)/4);
 }
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_ORDER_H) */
diff --git a/src/lib/container/smartlist.c b/src/lib/container/smartlist.c
index 3ab2797d68..7784f83957 100644
--- a/src/lib/container/smartlist.c
+++ b/src/lib/container/smartlist.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -652,7 +652,7 @@ smartlist_sort_pointers(smartlist_t *sl)
 #define LEFT_CHILD(i)  ( 2*(i) + 1 )
 #define RIGHT_CHILD(i) ( 2*(i) + 2 )
 #define PARENT(i)      ( ((i)-1) / 2 )
-/** }@ */
+/** @} */
 
 /** @{ */
 /** Helper macros for heaps: Given a local variable <b>idx_field_offset</b>
@@ -678,7 +678,7 @@ smartlist_sort_pointers(smartlist_t *sl)
 static inline void
 smartlist_heapify(smartlist_t *sl,
                   int (*compare)(const void *a, const void *b),
-                  int idx_field_offset,
+                  ptrdiff_t idx_field_offset,
                   int idx)
 {
   while (1) {
@@ -725,7 +725,7 @@ smartlist_heapify(smartlist_t *sl,
 void
 smartlist_pqueue_add(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset,
+                     ptrdiff_t idx_field_offset,
                      void *item)
 {
   int idx;
@@ -754,7 +754,7 @@ smartlist_pqueue_add(smartlist_t *sl,
 void *
 smartlist_pqueue_pop(smartlist_t *sl,
                      int (*compare)(const void *a, const void *b),
-                     int idx_field_offset)
+                     ptrdiff_t idx_field_offset)
 {
   void *top;
   tor_assert(sl->num_used);
@@ -778,7 +778,7 @@ smartlist_pqueue_pop(smartlist_t *sl,
 void
 smartlist_pqueue_remove(smartlist_t *sl,
                         int (*compare)(const void *a, const void *b),
-                        int idx_field_offset,
+                        ptrdiff_t idx_field_offset,
                         void *item)
 {
   int idx = IDX_OF_ITEM(item);
@@ -802,7 +802,7 @@ smartlist_pqueue_remove(smartlist_t *sl,
 void
 smartlist_pqueue_assert_ok(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset)
+                           ptrdiff_t idx_field_offset)
 {
   int i;
   for (i = sl->num_used - 1; i >= 0; --i) {
diff --git a/src/lib/container/smartlist.h b/src/lib/container/smartlist.h
index 77682db03e..458d564cd5 100644
--- a/src/lib/container/smartlist.h
+++ b/src/lib/container/smartlist.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SMARTLIST_H
@@ -13,6 +13,7 @@
  **/
 
 #include <stdarg.h>
+#include <stddef.h>
 
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -72,18 +73,18 @@ int smartlist_bsearch_idx(const smartlist_t *sl, const void *key,
 
 void smartlist_pqueue_add(smartlist_t *sl,
                           int (*compare)(const void *a, const void *b),
-                          int idx_field_offset,
+                          ptrdiff_t idx_field_offset,
                           void *item);
 void *smartlist_pqueue_pop(smartlist_t *sl,
                            int (*compare)(const void *a, const void *b),
-                           int idx_field_offset);
+                           ptrdiff_t idx_field_offset);
 void smartlist_pqueue_remove(smartlist_t *sl,
                              int (*compare)(const void *a, const void *b),
-                             int idx_field_offset,
+                             ptrdiff_t idx_field_offset,
                              void *item);
 void smartlist_pqueue_assert_ok(smartlist_t *sl,
                                 int (*compare)(const void *a, const void *b),
-                                int idx_field_offset);
+                                ptrdiff_t idx_field_offset);
 
 char *smartlist_join_strings(smartlist_t *sl, const char *join, int terminate,
                              size_t *len_out) ATTR_MALLOC;
@@ -91,6 +92,7 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
                               size_t join_len, int terminate, size_t *len_out)
   ATTR_MALLOC;
 
+#ifndef COCCI
 /* Helper: Given two lists of items, possibly of different types, such that
  * both lists are sorted on some common field (as determined by a comparison
  * expression <b>cmpexpr</b>), and such that one list (<b>sl1</b>) has no
@@ -164,5 +166,6 @@ char *smartlist_join_strings2(smartlist_t *sl, const char *join,
 #define SMARTLIST_FOREACH_JOIN_END(var1, var2)  \
   }                                             \
   STMT_END
+#endif /* !defined(COCCI) */
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_H) */
diff --git a/src/lib/crypt_ops/.may_include b/src/lib/crypt_ops/.may_include
index a0fa4ec05c..810e777271 100644
--- a/src/lib/crypt_ops/.may_include
+++ b/src/lib/crypt_ops/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
+lib/conf/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
 lib/ctime/*.h
@@ -12,13 +13,16 @@ lib/malloc/*.h
 lib/intmath/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/thread/*.h
 lib/log/*.h
 
+lib/crypt_ops/*.inc
+
 trunnel/pwbox.h
 
 keccak-tiny/*.h
 ed25519/*.h
 
-siphash.h
+ext/siphash.h
diff --git a/src/lib/crypt_ops/aes.h b/src/lib/crypt_ops/aes.h
index 7c774062d9..c25417b4e6 100644
--- a/src/lib/crypt_ops/aes.h
+++ b/src/lib/crypt_ops/aes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Implements a minimal interface to counter-mode AES. */
@@ -16,7 +16,7 @@
 #include "lib/cc/torint.h"
 #include "lib/malloc/malloc.h"
 
-typedef struct aes_cnt_cipher aes_cnt_cipher_t;
+typedef struct aes_cnt_cipher_t aes_cnt_cipher_t;
 
 aes_cnt_cipher_t* aes_new_cipher(const uint8_t *key, const uint8_t *iv,
                                  int key_bits);
diff --git a/src/lib/crypt_ops/aes_nss.c b/src/lib/crypt_ops/aes_nss.c
index 4eda5e5902..71d2f01449 100644
--- a/src/lib/crypt_ops/aes_nss.c
+++ b/src/lib/crypt_ops/aes_nss.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,10 +15,10 @@
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/log/util_bug.h"
 
-DISABLE_GCC_WARNING(strict-prototypes)
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
 #include <pk11pub.h>
 #include <secerr.h>
-ENABLE_GCC_WARNING(strict-prototypes)
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
 
 aes_cnt_cipher_t *
 aes_new_cipher(const uint8_t *key, const uint8_t *iv,
diff --git a/src/lib/crypt_ops/aes_openssl.c b/src/lib/crypt_ops/aes_openssl.c
index 2f985d4512..502f7703bd 100644
--- a/src/lib/crypt_ops/aes_openssl.c
+++ b/src/lib/crypt_ops/aes_openssl.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -28,7 +28,7 @@
 #error "We require OpenSSL >= 1.0.0"
 #endif
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <stdlib.h>
 #include <string.h>
@@ -37,9 +37,8 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/engine.h>
 #include <openssl/modes.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
-#include "lib/crypt_ops/aes.h"
 #include "lib/log/log.h"
 #include "lib/ctime/di_ops.h"
 
@@ -148,13 +147,13 @@ evaluate_ctr_for_aes(void)
 {
   return 0;
 }
-#else /* !(defined(USE_EVP_AES_CTR)) */
+#else /* !defined(USE_EVP_AES_CTR) */
 
 /*======================================================================*/
 /* Interface to AES code, and counter implementation */
 
 /** Implements an AES counter-mode cipher. */
-struct aes_cnt_cipher {
+struct aes_cnt_cipher_t {
 /** This next element (however it's defined) is the AES key. */
   union {
     EVP_CIPHER_CTX evp;
diff --git a/src/lib/crypt_ops/certs.md b/src/lib/crypt_ops/certs.md
new file mode 100644
index 0000000000..f3bd8c2c96
--- /dev/null
+++ b/src/lib/crypt_ops/certs.md
@@ -0,0 +1,29 @@
+
+@page certificates Certificates in Tor
+
+We have, alas, several certificate types in Tor.
+
+The tor_x509_cert_t type represents an X.509 certificate. This document
+won't explain X.509 to you -- possibly, no document can. (OTOH, Peter
+Gutmann's "x.509 style guide", though severely dated, does a good job of
+explaining how awful x.509 can be.)  Do not introduce any new usages of
+X.509. Right now we only use it in places where TLS forces us to do so.
+See x509.c for more information about using this type.
+
+
+The authority_cert_t type is used only for directory authority keys. It
+has a medium-term signing key (which the authorities actually keep
+online) signed by a long-term identity key (which the authority operator
+had really better be keeping offline).  Don't use it for any new kind of
+certificate.
+
+For new places where you need a certificate, consider tor_cert_t: it
+represents a typed and dated _something_ signed by an Ed25519 key.  The
+format is described in tor-spec. Unlike x.509, you can write it on a
+napkin.  The torcert.c file is used for manipulating these certificates and
+their associated keys.
+
+(Additionally, the Tor directory design uses a fairly wide variety of
+documents that include keys and which are signed by keys. You can
+consider these documents to be an additional kind of certificate if you
+want.)
diff --git a/src/lib/crypt_ops/compat_openssl.h b/src/lib/crypt_ops/compat_openssl.h
index 6605d01045..c2e1459078 100644
--- a/src/lib/crypt_ops/compat_openssl.h
+++ b/src/lib/crypt_ops/compat_openssl.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_COMPAT_OPENSSL_H
@@ -41,7 +41,7 @@
    ((st) == SSL3_ST_SW_SRVR_HELLO_B))
 #define OSSL_HANDSHAKE_STATE int
 #define CONST_IF_OPENSSL_1_1_API
-#else /* !(!defined(OPENSSL_1_1_API)) */
+#else /* defined(OPENSSL_1_1_API) */
 #define STATE_IS_SW_SERVER_HELLO(st) \
   ((st) == TLS_ST_SW_SRVR_HELLO)
 #define CONST_IF_OPENSSL_1_1_API const
diff --git a/src/lib/crypt_ops/crypto_cipher.c b/src/lib/crypt_ops/crypto_cipher.c
index 7bc2edad54..0a24a580ae 100644
--- a/src/lib/crypt_ops/crypto_cipher.c
+++ b/src/lib/crypt_ops/crypto_cipher.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_cipher.h b/src/lib/crypt_ops/crypto_cipher.h
index cc4fbf7a41..1e22a7c138 100644
--- a/src/lib/crypt_ops/crypto_cipher.h
+++ b/src/lib/crypt_ops/crypto_cipher.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
 /** Length of our symmetric cipher's keys of 256-bit. */
 #define CIPHER256_KEY_LEN 32
 
-typedef struct aes_cnt_cipher crypto_cipher_t;
+typedef struct aes_cnt_cipher_t crypto_cipher_t;
 
 /* environment setup */
 crypto_cipher_t *crypto_cipher_new(const char *key);
@@ -54,4 +54,4 @@ int crypto_cipher_decrypt_with_iv(const char *key,
                                   char *to, size_t tolen,
                                   const char *from, size_t fromlen);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_CIPHER_H) */
diff --git a/src/lib/crypt_ops/crypto_curve25519.c b/src/lib/crypt_ops/crypto_curve25519.c
index de4e17a296..2a2589f07d 100644
--- a/src/lib/crypt_ops/crypto_curve25519.c
+++ b/src/lib/crypt_ops/crypto_curve25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_curve25519.h b/src/lib/crypt_ops/crypto_curve25519.h
index 061a7a3505..154a0b94bc 100644
--- a/src/lib/crypt_ops/crypto_curve25519.h
+++ b/src/lib/crypt_ops/crypto_curve25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -76,8 +76,8 @@ STATIC int curve25519_basepoint_impl(uint8_t *output, const uint8_t *secret);
 
 int curve25519_public_from_base64(curve25519_public_key_t *pkey,
                                   const char *input);
-int curve25519_public_to_base64(char *output,
-                                const curve25519_public_key_t *pkey);
+void curve25519_public_to_base64(char *output,
+                                 const curve25519_public_key_t *pkey);
 
 void curve25519_set_impl_params(int use_ed);
 void curve25519_init(void);
diff --git a/src/lib/crypt_ops/crypto_dh.c b/src/lib/crypt_ops/crypto_dh.c
index 4be7948761..086aceed6f 100644
--- a/src/lib/crypt_ops/crypto_dh.c
+++ b/src/lib/crypt_ops/crypto_dh.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_dh.h b/src/lib/crypt_ops/crypto_dh.h
index 850d50c7ae..2a0e1f6912 100644
--- a/src/lib/crypt_ops/crypto_dh.h
+++ b/src/lib/crypt_ops/crypto_dh.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_dh_nss.c b/src/lib/crypt_ops/crypto_dh_nss.c
index 379eb84a4f..018db8bf43 100644
--- a/src/lib/crypt_ops/crypto_dh_nss.c
+++ b/src/lib/crypt_ops/crypto_dh_nss.c
@@ -1,11 +1,11 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * \file crypto_dh_nss.h
+ * \file crypto_dh_nss.c
  *
  * \brief NSS implementation of Diffie-Hellman over Z_p.
  **/
diff --git a/src/lib/crypt_ops/crypto_dh_openssl.c b/src/lib/crypt_ops/crypto_dh_openssl.c
index 8c6388fd5d..c5f7271596 100644
--- a/src/lib/crypt_ops/crypto_dh_openssl.c
+++ b/src/lib/crypt_ops/crypto_dh_openssl.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,11 +17,11 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/dh.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/bn.h>
 #include <string.h>
@@ -34,7 +34,7 @@ static int tor_check_dh_key(int severity, const BIGNUM *bn);
 struct crypto_dh_t {
   DH *dh; /**< The openssl DH object */
 };
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 static DH *new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g);
 
@@ -68,7 +68,7 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     goto out;
   if (!DH_set0_pqg(dh, dh_p, NULL, dh_g))
     goto out;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (!(dh->p = BN_dup(p)))
     goto out;
   if (!(dh->g = BN_dup(g)))
@@ -100,10 +100,10 @@ crypto_validate_dh_params(const BIGNUM *p, const BIGNUM *g)
     DH_free(dh);
   return ret;
 }
-#endif
+#endif /* 0 */
 
 /**
- * Helper: convert <b>hex<b> to a bignum, and return it.  Assert that the
+ * Helper: convert <b>hex</b> to a bignum, and return it.  Assert that the
  * operation was successful.
  */
 static BIGNUM *
@@ -202,7 +202,7 @@ crypto_dh_new(int dh_type)
     tor_free(res); // sets res to NULL.
   return res;
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 /** Create and return a new openssl DH from a given prime and generator. */
 static DH *
@@ -231,7 +231,7 @@ new_openssl_dh_from_params(BIGNUM *p, BIGNUM *g)
 
   if (!DH_set_length(res_dh, DH_PRIVATE_KEY_BITS))
     goto err;
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   res_dh->p = dh_p;
   res_dh->g = dh_g;
   res_dh->length = DH_PRIVATE_KEY_BITS;
@@ -298,7 +298,7 @@ crypto_dh_generate_public(crypto_dh_t *dh)
              "the-universe chances really do happen.  Treating as a failure.");
     return -1;
   }
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   if (tor_check_dh_key(LOG_WARN, dh->dh->pub_key)<0) {
     /* LCOV_EXCL_START
      * If this happens, then openssl's DH implementation is busted. */
@@ -461,7 +461,7 @@ crypto_dh_free_(crypto_dh_t *dh)
   DH_free(dh->dh);
   tor_free(dh);
 }
-#endif
+#endif /* !defined(ENABLE_NSS) */
 
 void
 crypto_dh_free_all_openssl(void)
diff --git a/src/lib/crypt_ops/crypto_digest.c b/src/lib/crypt_ops/crypto_digest.c
index de81b87b7e..7775e69410 100644
--- a/src/lib/crypt_ops/crypto_digest.c
+++ b/src/lib/crypt_ops/crypto_digest.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,171 +23,6 @@
 
 #include "lib/arch/bytes.h"
 
-#ifdef ENABLE_NSS
-DISABLE_GCC_WARNING(strict-prototypes)
-#include <pk11pub.h>
-ENABLE_GCC_WARNING(strict-prototypes)
-#else
-
-#include "lib/crypt_ops/crypto_openssl_mgt.h"
-
-DISABLE_GCC_WARNING(redundant-decls)
-
-#include <openssl/hmac.h>
-#include <openssl/sha.h>
-
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
-
-#ifdef ENABLE_NSS
-/**
- * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
- * On failure, return SEC_OID_UNKNOWN. */
-static SECOidTag
-digest_alg_to_nss_oid(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: return SEC_OID_SHA1;
-    case DIGEST_SHA256: return SEC_OID_SHA256;
-    case DIGEST_SHA512: return SEC_OID_SHA512;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return SEC_OID_UNKNOWN;
-  }
-}
-
-/* Helper: get an unkeyed digest via pk11wrap */
-static int
-digest_nss_internal(SECOidTag alg,
-                    char *digest, unsigned len_out,
-                    const char *msg, size_t msg_len)
-{
-  if (alg == SEC_OID_UNKNOWN)
-    return -1;
-  tor_assert(msg_len <= UINT_MAX);
-
-  int rv = -1;
-  SECStatus s;
-  PK11Context *ctx = PK11_CreateDigestContext(alg);
-  if (!ctx)
-    return -1;
-
-  s = PK11_DigestBegin(ctx);
-  if (s != SECSuccess)
-    goto done;
-
-  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-
-  unsigned int len = 0;
-  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
-  if (s != SECSuccess)
-    goto done;
-
-  rv = 0;
- done:
-  PK11_DestroyContext(ctx, PR_TRUE);
-  return rv;
-}
-
-/** True iff alg is implemented in our crypto library, and we want to use that
- * implementation */
-static bool
-library_supports_digest(digest_algorithm_t alg)
-{
-  switch (alg) {
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return true;
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512: FALLTHROUGH;
-    default:
-      return false;
-  }
-}
-#endif
-
-/* Crypto digest functions */
-
-/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
- * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
- * Return 0 on success, -1 on failure.
- */
-MOCK_IMPL(int,
-crypto_digest,(char *digest, const char *m, size_t len))
-{
-  tor_assert(m);
-  tor_assert(digest);
-#ifdef ENABLE_NSS
-  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
-#else
-  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
-    return -1;
-  }
-#endif
-  return 0;
-}
-
-/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest256(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA256) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
-#else
-    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
-#endif
-  } else {
-    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
-/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
- * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
- * into <b>digest</b>.  Return 0 on success, -1 on failure. */
-int
-crypto_digest512(char *digest, const char *m, size_t len,
-                 digest_algorithm_t algorithm)
-{
-  tor_assert(m);
-  tor_assert(digest);
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-
-  int ret = 0;
-  if (algorithm == DIGEST_SHA512) {
-#ifdef ENABLE_NSS
-    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
-#else
-    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
-           != NULL);
-#endif
-  } else {
-    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
-           > -1);
-  }
-
-  if (!ret)
-    return -1;
-  return 0;
-}
-
 /** Set the common_digests_t in <b>ds_out</b> to contain every digest on the
  * <b>len</b> bytes in <b>m</b> that we know how to compute.  Return 0 on
  * success, -1 on failure. */
@@ -267,485 +102,6 @@ crypto_digest_algorithm_get_length(digest_algorithm_t alg)
   }
 }
 
-/** Intermediate information about the digest of a stream of data. */
-struct crypto_digest_t {
-  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
-   /** State for the digest we're using.  Only one member of the
-    * union is usable, depending on the value of <b>algorithm</b>. Note also
-    * that space for other members might not even be allocated!
-    */
-  union {
-#ifdef ENABLE_NSS
-    PK11Context *ctx;
-#else
-    SHA_CTX sha1; /**< state for SHA1 */
-    SHA256_CTX sha2; /**< state for SHA256 */
-    SHA512_CTX sha512; /**< state for SHA512 */
-#endif
-    keccak_state sha3; /**< state for SHA3-[256,512] */
-  } d;
-};
-
-#ifdef TOR_UNIT_TESTS
-
-digest_algorithm_t
-crypto_digest_get_algorithm(crypto_digest_t *digest)
-{
-  tor_assert(digest);
-
-  return digest->algorithm;
-}
-
-#endif /* defined(TOR_UNIT_TESTS) */
-
-/**
- * Return the number of bytes we need to malloc in order to get a
- * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
- * when we free one.
- */
-static size_t
-crypto_digest_alloc_bytes(digest_algorithm_t alg)
-{
-  /* Helper: returns the number of bytes in the 'f' field of 'st' */
-#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
-  /* Gives the length of crypto_digest_t through the end of the field 'd' */
-#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
-                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
-  switch (alg) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.ctx);
-#else
-    case DIGEST_SHA1:
-      return END_OF_FIELD(d.sha1);
-    case DIGEST_SHA256:
-      return END_OF_FIELD(d.sha2);
-    case DIGEST_SHA512:
-      return END_OF_FIELD(d.sha512);
-#endif
-    case DIGEST_SHA3_256:
-    case DIGEST_SHA3_512:
-      return END_OF_FIELD(d.sha3);
-    default:
-      tor_assert(0); // LCOV_EXCL_LINE
-      return 0;      // LCOV_EXCL_LINE
-  }
-#undef END_OF_FIELD
-#undef STRUCT_FIELD_SIZE
-}
-
-/**
- * Internal function: create and return a new digest object for 'algorithm'.
- * Does not typecheck the algorithm.
- */
-static crypto_digest_t *
-crypto_digest_new_internal(digest_algorithm_t algorithm)
-{
-  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
-  r->algorithm = algorithm;
-
-  switch (algorithm)
-    {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
-      if (BUG(!r->d.ctx)) {
-        tor_free(r);
-        return NULL;
-      }
-      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
-        crypto_digest_free(r);
-        return NULL;
-      }
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Init(&r->d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Init(&r->d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Init(&r->d.sha512);
-      break;
-#endif
-    case DIGEST_SHA3_256:
-      keccak_digest_init(&r->d.sha3, 256);
-      break;
-    case DIGEST_SHA3_512:
-      keccak_digest_init(&r->d.sha3, 512);
-      break;
-    default:
-      tor_assert_unreached();
-    }
-
-  return r;
-}
-
-/** Allocate and return a new digest object to compute SHA1 digests.
- */
-crypto_digest_t *
-crypto_digest_new(void)
-{
-  return crypto_digest_new_internal(DIGEST_SHA1);
-}
-
-/** Allocate and return a new digest object to compute 256-bit digests
- * using <b>algorithm</b>.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
- * C_RUST_COUPLED: `crypto::digest::Sha256::default`
- */
-crypto_digest_t *
-crypto_digest256_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Allocate and return a new digest object to compute 512-bit digests
- * using <b>algorithm</b>. */
-crypto_digest_t *
-crypto_digest512_new(digest_algorithm_t algorithm)
-{
-  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
-  return crypto_digest_new_internal(algorithm);
-}
-
-/** Deallocate a digest object.
- */
-void
-crypto_digest_free_(crypto_digest_t *digest)
-{
-  if (!digest)
-    return;
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
-  }
-#endif
-  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  memwipe(digest, 0, bytes);
-  tor_free(digest);
-}
-
-/** Add <b>len</b> bytes from <b>data</b> to the digest object.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
- * C_RUST_COUPLED: `crypto::digest::Sha256::process`
- */
-void
-crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
-                        size_t len)
-{
-  tor_assert(digest);
-  tor_assert(data);
-  /* Using the SHA*_*() calls directly means we don't support doing
-   * SHA in hardware. But so far the delay of getting the question
-   * to the hardware, and hearing the answer, is likely higher than
-   * just doing it ourselves. Hashes are fast.
-   */
-  switch (digest->algorithm) {
-#ifdef ENABLE_NSS
-    case DIGEST_SHA1: FALLTHROUGH;
-    case DIGEST_SHA256: FALLTHROUGH;
-    case DIGEST_SHA512:
-      tor_assert(len <= UINT_MAX);
-      SECStatus s = PK11_DigestOp(digest->d.ctx,
-                                  (const unsigned char *)data,
-                                  (unsigned int)len);
-      tor_assert(s == SECSuccess);
-      break;
-#else
-    case DIGEST_SHA1:
-      SHA1_Update(&digest->d.sha1, (void*)data, len);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Update(&digest->d.sha2, (void*)data, len);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Update(&digest->d.sha512, (void*)data, len);
-      break;
-#endif
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
-      break;
-    default:
-      /* LCOV_EXCL_START */
-      tor_fragile_assert();
-      break;
-      /* LCOV_EXCL_STOP */
-  }
-}
-
-/** Compute the hash of the data that has been passed to the digest
- * object; write the first out_len bytes of the result to <b>out</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN.
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
- * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
- */
-void
-crypto_digest_get_digest(crypto_digest_t *digest,
-                         char *out, size_t out_len)
-{
-  unsigned char r[DIGEST512_LEN];
-  tor_assert(digest);
-  tor_assert(out);
-  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
-
-  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
-   * short output buffers by truncating appropriately. */
-  if (digest->algorithm == DIGEST_SHA3_256 ||
-      digest->algorithm == DIGEST_SHA3_512) {
-    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
-    return;
-  }
-
-#ifdef ENABLE_NSS
-  /* Copy into a temporary buffer since DigestFinal (alters) the context */
-  unsigned char buf[1024];
-  unsigned int saved_len = 0;
-  unsigned rlen;
-  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
-                                               buf, sizeof(buf),
-                                               &saved_len);
-  tor_assert(saved);
-  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
-  tor_assert(s == SECSuccess);
-  tor_assert(rlen >= out_len);
-  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
-  tor_assert(s == SECSuccess);
-  if (saved != buf) {
-    PORT_ZFree(saved, saved_len);
-  }
-#else
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t tmpenv;
-  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
-  memcpy(&tmpenv, digest, alloc_bytes);
-  switch (digest->algorithm) {
-    case DIGEST_SHA1:
-      SHA1_Final(r, &tmpenv.d.sha1);
-      break;
-    case DIGEST_SHA256:
-      SHA256_Final(r, &tmpenv.d.sha2);
-      break;
-    case DIGEST_SHA512:
-      SHA512_Final(r, &tmpenv.d.sha512);
-      break;
-//LCOV_EXCL_START
-    case DIGEST_SHA3_256: FALLTHROUGH;
-    case DIGEST_SHA3_512:
-    default:
-      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
-      /* This is fatal, because it should never happen. */
-      tor_assert_unreached();
-      break;
-//LCOV_EXCL_STOP
-  }
-#endif
-  memcpy(out, r, out_len);
-  memwipe(r, 0, sizeof(r));
-}
-
-/** Allocate and return a new digest object with the same state as
- * <b>digest</b>
- *
- * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
- * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
- */
-crypto_digest_t *
-crypto_digest_dup(const crypto_digest_t *digest)
-{
-  tor_assert(digest);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    result->d.ctx = PK11_CloneContext(digest->d.ctx);
-  }
-#endif
-  return result;
-}
-
-/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object.
- */
-void
-crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
-                         const crypto_digest_t *digest)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-  tor_assert(bytes <= sizeof(checkpoint->mem));
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    unsigned char *allocated;
-    allocated = PK11_SaveContextAlloc(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      sizeof(checkpoint->mem),
-                                      &checkpoint->bytes_used);
-    /* No allocation is allowed here. */
-    tor_assert(allocated == checkpoint->mem);
-    return;
-  }
-#endif
-  memcpy(checkpoint->mem, digest, bytes);
-}
-
-/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
- * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
- * state was previously stored with crypto_digest_checkpoint() */
-void
-crypto_digest_restore(crypto_digest_t *digest,
-                      const crypto_digest_checkpoint_t *checkpoint)
-{
-  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(digest->algorithm)) {
-    SECStatus s = PK11_RestoreContext(digest->d.ctx,
-                                      (unsigned char *)checkpoint->mem,
-                                      checkpoint->bytes_used);
-    tor_assert(s == SECSuccess);
-    return;
-  }
-#endif
-  memcpy(digest, checkpoint->mem, bytes);
-}
-
-/** Replace the state of the digest object <b>into</b> with the state
- * of the digest object <b>from</b>.  Requires that 'into' and 'from'
- * have the same digest type.
- */
-void
-crypto_digest_assign(crypto_digest_t *into,
-                     const crypto_digest_t *from)
-{
-  tor_assert(into);
-  tor_assert(from);
-  tor_assert(into->algorithm == from->algorithm);
-  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
-#ifdef ENABLE_NSS
-  if (library_supports_digest(from->algorithm)) {
-    PK11_DestroyContext(into->d.ctx, PR_TRUE);
-    into->d.ctx = PK11_CloneContext(from->d.ctx);
-    return;
-  }
-#endif
-  memcpy(into,from,alloc_bytes);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of those strings,
- * plus the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>out_len</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist(char *digest_out, size_t len_out,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
-}
-
-/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
- * at <b>digest_out</b> to the hash of the concatenation of: the
- * optional string <b>prepend</b>, those strings,
- * and the optional string <b>append</b>, computed with the algorithm
- * <b>alg</b>.
- * <b>len_out</b> must be \<= DIGEST512_LEN. */
-void
-crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
-                        const char *prepend,
-                        const smartlist_t *lst,
-                        const char *append,
-                        digest_algorithm_t alg)
-{
-  crypto_digest_t *d = crypto_digest_new_internal(alg);
-  if (prepend)
-    crypto_digest_add_bytes(d, prepend, strlen(prepend));
-  SMARTLIST_FOREACH(lst, const char *, cp,
-                    crypto_digest_add_bytes(d, cp, strlen(cp)));
-  if (append)
-    crypto_digest_add_bytes(d, append, strlen(append));
-  crypto_digest_get_digest(d, digest_out, len_out);
-  crypto_digest_free(d);
-}
-
-/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
- * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
- * result in <b>hmac_out</b>. Asserts on failure.
- */
-void
-crypto_hmac_sha256(char *hmac_out,
-                   const char *key, size_t key_len,
-                   const char *msg, size_t msg_len)
-{
-  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
-  tor_assert(key_len < INT_MAX);
-  tor_assert(msg_len < INT_MAX);
-  tor_assert(hmac_out);
-#ifdef ENABLE_NSS
-  PK11SlotInfo *slot = NULL;
-  PK11SymKey *symKey = NULL;
-  PK11Context *hmac = NULL;
-
-  int ok = 0;
-  SECStatus s;
-  SECItem keyItem, paramItem;
-  keyItem.data = (unsigned char *)key;
-  keyItem.len = (unsigned)key_len;
-  paramItem.type = siBuffer;
-  paramItem.data = NULL;
-  paramItem.len = 0;
-
-  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
-  if (!slot)
-    goto done;
-  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
-                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
-  if (!symKey)
-    goto done;
-
-  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
-                                    &paramItem);
-  if (!hmac)
-    goto done;
-  s = PK11_DigestBegin(hmac);
-  if (s != SECSuccess)
-    goto done;
-  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
-  if (s != SECSuccess)
-    goto done;
-  unsigned int len=0;
-  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
-  if (s != SECSuccess || len != DIGEST256_LEN)
-    goto done;
-  ok = 1;
-
- done:
-  if (hmac)
-    PK11_DestroyContext(hmac, PR_TRUE);
-  if (symKey)
-    PK11_FreeSymKey(symKey);
-  if (slot)
-    PK11_FreeSlot(slot);
-
-  tor_assert(ok);
-#else
-  unsigned char *rv = NULL;
-  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
-            (unsigned char*)hmac_out, NULL);
-  tor_assert(rv);
-#endif
-}
-
 /** Compute a MAC using SHA3-256 of <b>msg_len</b> bytes in <b>msg</b> using a
  * <b>key</b> of length <b>key_len</b> and a <b>salt</b> of length
  * <b>salt_len</b>. Store the result of <b>len_out</b> bytes in in
@@ -779,7 +135,26 @@ crypto_mac_sha3_256(uint8_t *mac_out, size_t len_out,
 
 /** Internal state for a eXtendable-Output Function (XOF). */
 struct crypto_xof_t {
+#ifdef OPENSSL_HAS_SHAKE3_EVP
+  /* XXXX We can't enable this yet, because OpenSSL's
+   * DigestFinalXOF function can't be called repeatedly on the same
+   * XOF.
+   *
+   * We could in theory use the undocumented SHA3_absorb and SHA3_squeeze
+   * functions, but let's not mess with undocumented OpenSSL internals any
+   * more than we have to.
+   *
+   * We could also revise our XOF code so that it only allows a single
+   * squeeze operation; we don't require streaming squeeze operations
+   * outside the tests yet.
+   */
+  EVP_MD_CTX *ctx;
+#else /* !defined(OPENSSL_HAS_SHAKE3_EVP) */
+  /**
+   * State of the Keccak sponge for the SHAKE-256 computation.
+   **/
   keccak_state s;
+#endif /* defined(OPENSSL_HAS_SHAKE3_EVP) */
 };
 
 /** Allocate a new XOF object backed by SHAKE-256.  The security level
@@ -792,7 +167,14 @@ crypto_xof_new(void)
 {
   crypto_xof_t *xof;
   xof = tor_malloc(sizeof(crypto_xof_t));
+#ifdef OPENSSL_HAS_SHAKE256
+  xof->ctx = EVP_MD_CTX_new();
+  tor_assert(xof->ctx);
+  int r = EVP_DigestInit(xof->ctx, EVP_shake256());
+  tor_assert(r == 1);
+#else /* !defined(OPENSSL_HAS_SHAKE256) */
   keccak_xof_init(&xof->s, 256);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
   return xof;
 }
 
@@ -803,8 +185,13 @@ crypto_xof_new(void)
 void
 crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestUpdate(xof->ctx, data, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_absorb(&xof->s, data, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Squeeze bytes out of a XOF object.  Calling this routine will render
@@ -813,8 +200,13 @@ crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len)
 void
 crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len)
 {
+#ifdef OPENSSL_HAS_SHAKE256
+  int r = EVP_DigestFinalXOF(xof->ctx, out, len);
+  tor_assert(r == 1);
+#else
   int i = keccak_xof_squeeze(&xof->s, out, len);
   tor_assert(i == 0);
+#endif /* defined(OPENSSL_HAS_SHAKE256) */
 }
 
 /** Cleanse and deallocate a XOF object. */
@@ -823,6 +215,34 @@ crypto_xof_free_(crypto_xof_t *xof)
 {
   if (!xof)
     return;
+#ifdef OPENSSL_HAS_SHAKE256
+  if (xof->ctx)
+    EVP_MD_CTX_free(xof->ctx);
+#endif
   memwipe(xof, 0, sizeof(crypto_xof_t));
   tor_free(xof);
 }
+
+/** Compute the XOF (SHAKE256) of a <b>input_len</b> bytes at <b>input</b>,
+ * putting <b>output_len</b> bytes at <b>output</b>. */
+void
+crypto_xof(uint8_t *output, size_t output_len,
+           const uint8_t *input, size_t input_len)
+{
+#ifdef OPENSSL_HAS_SHA3
+  EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+  tor_assert(ctx);
+  int r = EVP_DigestInit(ctx, EVP_shake256());
+  tor_assert(r == 1);
+  r = EVP_DigestUpdate(ctx, input, input_len);
+  tor_assert(r == 1);
+  r = EVP_DigestFinalXOF(ctx, output, output_len);
+  tor_assert(r == 1);
+  EVP_MD_CTX_free(ctx);
+#else /* !defined(OPENSSL_HAS_SHA3) */
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, input, input_len);
+  crypto_xof_squeeze_bytes(xof, output, output_len);
+  crypto_xof_free(xof);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+}
diff --git a/src/lib/crypt_ops/crypto_digest.h b/src/lib/crypt_ops/crypto_digest.h
index 47e60ce617..eefd2e3f0a 100644
--- a/src/lib/crypt_ops/crypto_digest.h
+++ b/src/lib/crypt_ops/crypto_digest.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,6 +38,9 @@
 /** Length of hex encoding of SHA512 digest, not including final NUL. */
 #define HEX_DIGEST512_LEN 128
 
+/**
+ * An identifier for a cryptographic digest algorithm.
+ **/
 typedef enum {
   DIGEST_SHA1 = 0,
   DIGEST_SHA256 = 1,
@@ -45,16 +48,31 @@ typedef enum {
   DIGEST_SHA3_256 = 3,
   DIGEST_SHA3_512 = 4,
 } digest_algorithm_t;
+/** Number of digest algorithms that we know */
 #define  N_DIGEST_ALGORITHMS (DIGEST_SHA3_512+1)
+/** Number of digest algorithms to compute when computing "all the
+ * commonly used digests."
+ *
+ * (This is used in common_digests_t and related functions.)
+ */
 #define  N_COMMON_DIGEST_ALGORITHMS (DIGEST_SHA256+1)
 
+/**
+ * Bytes of storage needed to record the state of an in-progress SHA-1 digest.
+ *
+ * This is a deliberate overestimate.
+ **/
 #define DIGEST_CHECKPOINT_BYTES (SIZEOF_VOID_P + 512)
+
 /** Structure used to temporarily save the a digest object. Only implemented
  * for SHA1 digest for now. */
 typedef struct crypto_digest_checkpoint_t {
 #ifdef ENABLE_NSS
+  /** The number of bytes used in <b>mem</b>. */
   unsigned int bytes_used;
 #endif
+  /** A buffer to store the SHA1 state. Its contents are unspecified, and
+   * are managed by the underlying crypto library.*/
   uint8_t mem[DIGEST_CHECKPOINT_BYTES];
 } crypto_digest_checkpoint_t;
 
@@ -67,10 +85,19 @@ typedef struct crypto_digest_checkpoint_t {
  * once.
  **/
 typedef struct {
+  /** An array of digest outputs, one for each "common" digest algorithm. */
   char d[N_COMMON_DIGEST_ALGORITHMS][DIGEST256_LEN];
 } common_digests_t;
 
+/**
+ * State for computing a digest over a stream of data.
+ **/
 typedef struct crypto_digest_t crypto_digest_t;
+
+/**
+ * State for computing an "extendable-output function" (like SHAKE) over a
+ * stream of data, and/or streaming the output.
+ **/
 typedef struct crypto_xof_t crypto_xof_t;
 
 struct smartlist_t;
@@ -97,6 +124,9 @@ crypto_digest_t *crypto_digest_new(void);
 crypto_digest_t *crypto_digest256_new(digest_algorithm_t algorithm);
 crypto_digest_t *crypto_digest512_new(digest_algorithm_t algorithm);
 void crypto_digest_free_(crypto_digest_t *digest);
+/**
+ * Release all storage held in <b>d</b>, and set it to NULL.
+ **/
 #define crypto_digest_free(d) \
   FREE_AND_NULL(crypto_digest_t, crypto_digest_free_, (d))
 void crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
@@ -122,8 +152,13 @@ crypto_xof_t *crypto_xof_new(void);
 void crypto_xof_add_bytes(crypto_xof_t *xof, const uint8_t *data, size_t len);
 void crypto_xof_squeeze_bytes(crypto_xof_t *xof, uint8_t *out, size_t len);
 void crypto_xof_free_(crypto_xof_t *xof);
+/**
+ * Release all storage held in <b>xof</b>, and set it to NULL.
+ **/
 #define crypto_xof_free(xof) \
   FREE_AND_NULL(crypto_xof_t, crypto_xof_free_, (xof))
+void crypto_xof(uint8_t *output, size_t output_len,
+                const uint8_t *input, size_t input_len);
 
 #ifdef TOR_UNIT_TESTS
 digest_algorithm_t crypto_digest_get_algorithm(crypto_digest_t *digest);
diff --git a/src/lib/crypt_ops/crypto_digest_nss.c b/src/lib/crypt_ops/crypto_digest_nss.c
new file mode 100644
index 0000000000..92c20fe9e8
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_nss.c
@@ -0,0 +1,563 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_nss.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (NSS specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
+#include <pk11pub.h>
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
+
+/**
+ * Convert a digest_algorithm_t (used by tor) to a HashType (used by NSS).
+ * On failure, return SEC_OID_UNKNOWN. */
+static SECOidTag
+digest_alg_to_nss_oid(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: return SEC_OID_SHA1;
+    case DIGEST_SHA256: return SEC_OID_SHA256;
+    case DIGEST_SHA512: return SEC_OID_SHA512;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return SEC_OID_UNKNOWN;
+  }
+}
+
+/** Helper: Compute an unkeyed digest of the <b>msg_len</b> bytes at
+ * <b>msg</b>, using the digest algorithm specified by <b>alg</b>.
+ * Store the result in the <b>len_out</b>-byte buffer at <b>digest</b>.
+ * Return the number of bytes written on success, and -1 on failure.
+ **/
+static int
+digest_nss_internal(SECOidTag alg,
+                    char *digest, unsigned len_out,
+                    const char *msg, size_t msg_len)
+{
+  if (alg == SEC_OID_UNKNOWN)
+    return -1;
+  tor_assert(msg_len <= UINT_MAX);
+
+  int rv = -1;
+  SECStatus s;
+  PK11Context *ctx = PK11_CreateDigestContext(alg);
+  if (!ctx)
+    return -1;
+
+  s = PK11_DigestBegin(ctx);
+  if (s != SECSuccess)
+    goto done;
+
+  s = PK11_DigestOp(ctx, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+
+  unsigned int len = 0;
+  s = PK11_DigestFinal(ctx, (unsigned char *)digest, &len, len_out);
+  if (s != SECSuccess)
+    goto done;
+
+  rv = 0;
+ done:
+  PK11_DestroyContext(ctx, PR_TRUE);
+  return rv;
+}
+
+/** True iff alg is implemented in our crypto library, and we want to use that
+ * implementation */
+static bool
+library_supports_digest(digest_algorithm_t alg)
+{
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return true;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: FALLTHROUGH;
+    default:
+      return false;
+  }
+}
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  return digest_nss_internal(SEC_OID_SHA1, digest, DIGEST_LEN, m, len);
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    return digest_nss_internal(SEC_OID_SHA256, digest, DIGEST256_LEN, m, len);
+  } else {
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    return digest_nss_internal(SEC_OID_SHA512, digest, DIGEST512_LEN, m, len);
+  } else {
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    PK11Context *ctx;
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /* Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /* Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.ctx);
+    case DIGEST_SHA3_256:
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      r->d.ctx = PK11_CreateDigestContext(digest_alg_to_nss_oid(algorithm));
+      if (BUG(!r->d.ctx)) {
+        tor_free(r);
+        return NULL;
+      }
+      if (BUG(SECSuccess != PK11_DigestBegin(r->d.ctx))) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+  if (library_supports_digest(digest->algorithm)) {
+    PK11_DestroyContext(digest->d.ctx, PR_TRUE);
+  }
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1: FALLTHROUGH;
+    case DIGEST_SHA256: FALLTHROUGH;
+    case DIGEST_SHA512:
+      tor_assert(len <= UINT_MAX);
+      SECStatus s = PK11_DigestOp(digest->d.ctx,
+                                  (const unsigned char *)data,
+                                  (unsigned int)len);
+      tor_assert(s == SECSuccess);
+      break;
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+  }
+
+  /* Copy into a temporary buffer since DigestFinal (alters) the context */
+  unsigned char buf[1024];
+  unsigned int saved_len = 0;
+  unsigned rlen;
+  unsigned char *saved = PK11_SaveContextAlloc(digest->d.ctx,
+                                               buf, sizeof(buf),
+                                               &saved_len);
+  tor_assert(saved);
+  SECStatus s = PK11_DigestFinal(digest->d.ctx, r, &rlen, sizeof(r));
+  tor_assert(s == SECSuccess);
+  tor_assert(rlen >= out_len);
+  s = PK11_RestoreContext(digest->d.ctx, saved, saved_len);
+  tor_assert(s == SECSuccess);
+
+  if (saved != buf) {
+    PORT_ZFree(saved, saved_len);
+  }
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+  if (library_supports_digest(digest->algorithm)) {
+    result->d.ctx = PK11_CloneContext(digest->d.ctx);
+  }
+
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  if (library_supports_digest(digest->algorithm)) {
+    unsigned char *allocated;
+    allocated = PK11_SaveContextAlloc(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      sizeof(checkpoint->mem),
+                                      &checkpoint->bytes_used);
+    /* No allocation is allowed here. */
+    tor_assert(allocated == checkpoint->mem);
+    return;
+  }
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  if (library_supports_digest(digest->algorithm)) {
+    SECStatus s = PK11_RestoreContext(digest->d.ctx,
+                                      (unsigned char *)checkpoint->mem,
+                                      checkpoint->bytes_used);
+    tor_assert(s == SECSuccess);
+    return;
+  }
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+  if (library_supports_digest(from->algorithm)) {
+    PK11_DestroyContext(into->d.ctx, PR_TRUE);
+    into->d.ctx = PK11_CloneContext(from->d.ctx);
+    return;
+  }
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+
+  PK11SlotInfo *slot = NULL;
+  PK11SymKey *symKey = NULL;
+  PK11Context *hmac = NULL;
+
+  int ok = 0;
+  SECStatus s;
+  SECItem keyItem, paramItem;
+  keyItem.data = (unsigned char *)key;
+  keyItem.len = (unsigned)key_len;
+  paramItem.type = siBuffer;
+  paramItem.data = NULL;
+  paramItem.len = 0;
+
+  slot = PK11_GetBestSlot(CKM_SHA256_HMAC, NULL);
+  if (!slot)
+    goto done;
+  symKey = PK11_ImportSymKey(slot, CKM_SHA256_HMAC,
+                             PK11_OriginUnwrap, CKA_SIGN, &keyItem, NULL);
+  if (!symKey)
+    goto done;
+
+  hmac = PK11_CreateContextBySymKey(CKM_SHA256_HMAC, CKA_SIGN, symKey,
+                                    &paramItem);
+  if (!hmac)
+    goto done;
+  s = PK11_DigestBegin(hmac);
+  if (s != SECSuccess)
+    goto done;
+  s = PK11_DigestOp(hmac, (const unsigned char *)msg, (unsigned int)msg_len);
+  if (s != SECSuccess)
+    goto done;
+  unsigned int len=0;
+  s = PK11_DigestFinal(hmac, (unsigned char *)hmac_out, &len, DIGEST256_LEN);
+  if (s != SECSuccess || len != DIGEST256_LEN)
+    goto done;
+  ok = 1;
+
+ done:
+  if (hmac)
+    PK11_DestroyContext(hmac, PR_TRUE);
+  if (symKey)
+    PK11_FreeSymKey(symKey);
+  if (slot)
+    PK11_FreeSlot(slot);
+
+  tor_assert(ok);
+}
diff --git a/src/lib/crypt_ops/crypto_digest_openssl.c b/src/lib/crypt_ops/crypto_digest_openssl.c
new file mode 100644
index 0000000000..11189c7fb2
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_digest_openssl.c
@@ -0,0 +1,521 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_digest_openssl.c
+ * \brief Block of functions related with digest and xof utilities and
+ * operations (OpenSSL specific implementations).
+ **/
+
+#include "lib/container/smartlist.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+
+#include "keccak-tiny/keccak-tiny.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "lib/arch/bytes.h"
+
+#include "lib/crypt_ops/crypto_openssl_mgt.h"
+
+DISABLE_GCC_WARNING("-Wredundant-decls")
+
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+ENABLE_GCC_WARNING("-Wredundant-decls")
+
+/* Crypto digest functions */
+
+/** Compute the SHA1 digest of the <b>len</b> bytes on data stored in
+ * <b>m</b>.  Write the DIGEST_LEN byte result into <b>digest</b>.
+ * Return 0 on success, -1 on failure.
+ */
+MOCK_IMPL(int,
+crypto_digest,(char *digest, const char *m, size_t len))
+{
+  tor_assert(m);
+  tor_assert(digest);
+  if (SHA1((const unsigned char*)m,len,(unsigned char*)digest) == NULL) {
+    return -1;
+  }
+  return 0;
+}
+
+/** Compute a 256-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN256-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest256(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA256) {
+    ret = (SHA256((const uint8_t*)m,len,(uint8_t*)digest) != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST256_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_256(), NULL);
+#else
+    ret = (sha3_256((uint8_t *)digest, DIGEST256_LEN,(const uint8_t *)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Compute a 512-bit digest of <b>len</b> bytes in data stored in <b>m</b>,
+ * using the algorithm <b>algorithm</b>.  Write the DIGEST_LEN512-byte result
+ * into <b>digest</b>.  Return 0 on success, -1 on failure. */
+int
+crypto_digest512(char *digest, const char *m, size_t len,
+                 digest_algorithm_t algorithm)
+{
+  tor_assert(m);
+  tor_assert(digest);
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+
+  int ret = 0;
+  if (algorithm == DIGEST_SHA512) {
+    ret = (SHA512((const unsigned char*)m,len,(unsigned char*)digest)
+           != NULL);
+  } else {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned int dlen = DIGEST512_LEN;
+    ret = EVP_Digest(m, len, (uint8_t*)digest, &dlen, EVP_sha3_512(), NULL);
+#else
+    ret = (sha3_512((uint8_t*)digest, DIGEST512_LEN, (const uint8_t*)m, len)
+           > -1);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  if (!ret)
+    return -1;
+  return 0;
+}
+
+/** Intermediate information about the digest of a stream of data. */
+struct crypto_digest_t {
+  digest_algorithm_t algorithm; /**< Which algorithm is in use? */
+   /** State for the digest we're using.  Only one member of the
+    * union is usable, depending on the value of <b>algorithm</b>. Note also
+    * that space for other members might not even be allocated!
+    */
+  union {
+    SHA_CTX sha1; /**< state for SHA1 */
+    SHA256_CTX sha2; /**< state for SHA256 */
+    SHA512_CTX sha512; /**< state for SHA512 */
+#ifdef OPENSSL_HAS_SHA3
+    EVP_MD_CTX *md;
+#else
+    keccak_state sha3; /**< state for SHA3-[256,512] */
+#endif
+  } d;
+};
+
+#ifdef TOR_UNIT_TESTS
+
+digest_algorithm_t
+crypto_digest_get_algorithm(crypto_digest_t *digest)
+{
+  tor_assert(digest);
+
+  return digest->algorithm;
+}
+
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Return the number of bytes we need to malloc in order to get a
+ * crypto_digest_t for <b>alg</b>, or the number of bytes we need to wipe
+ * when we free one.
+ */
+static size_t
+crypto_digest_alloc_bytes(digest_algorithm_t alg)
+{
+  /** Helper: returns the number of bytes in the 'f' field of 'st' */
+#define STRUCT_FIELD_SIZE(st, f) (sizeof( ((st*)0)->f ))
+  /** Gives the length of crypto_digest_t through the end of the field 'd' */
+#define END_OF_FIELD(f) (offsetof(crypto_digest_t, f) + \
+                         STRUCT_FIELD_SIZE(crypto_digest_t, f))
+  switch (alg) {
+    case DIGEST_SHA1:
+      return END_OF_FIELD(d.sha1);
+    case DIGEST_SHA256:
+      return END_OF_FIELD(d.sha2);
+    case DIGEST_SHA512:
+      return END_OF_FIELD(d.sha512);
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.md);
+#else
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      return END_OF_FIELD(d.sha3);
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert(0); // LCOV_EXCL_LINE
+      return 0;      // LCOV_EXCL_LINE
+  }
+#undef END_OF_FIELD
+#undef STRUCT_FIELD_SIZE
+}
+
+/**
+ * Internal function: create and return a new digest object for 'algorithm'.
+ * Does not typecheck the algorithm.
+ */
+static crypto_digest_t *
+crypto_digest_new_internal(digest_algorithm_t algorithm)
+{
+  crypto_digest_t *r = tor_malloc(crypto_digest_alloc_bytes(algorithm));
+  r->algorithm = algorithm;
+
+  switch (algorithm)
+    {
+    case DIGEST_SHA1:
+      SHA1_Init(&r->d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Init(&r->d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Init(&r->d.sha512);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_256())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+    case DIGEST_SHA3_512:
+      r->d.md = EVP_MD_CTX_new();
+      if (!EVP_DigestInit(r->d.md, EVP_sha3_512())) {
+        crypto_digest_free(r);
+        return NULL;
+      }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256:
+      keccak_digest_init(&r->d.sha3, 256);
+      break;
+    case DIGEST_SHA3_512:
+      keccak_digest_init(&r->d.sha3, 512);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      tor_assert_unreached();
+    }
+
+  return r;
+}
+
+/** Allocate and return a new digest object to compute SHA1 digests.
+ */
+crypto_digest_t *
+crypto_digest_new(void)
+{
+  return crypto_digest_new_internal(DIGEST_SHA1);
+}
+
+/** Allocate and return a new digest object to compute 256-bit digests
+ * using <b>algorithm</b>.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest256_new`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::default`
+ */
+crypto_digest_t *
+crypto_digest256_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA256 || algorithm == DIGEST_SHA3_256);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Allocate and return a new digest object to compute 512-bit digests
+ * using <b>algorithm</b>. */
+crypto_digest_t *
+crypto_digest512_new(digest_algorithm_t algorithm)
+{
+  tor_assert(algorithm == DIGEST_SHA512 || algorithm == DIGEST_SHA3_512);
+  return crypto_digest_new_internal(algorithm);
+}
+
+/** Deallocate a digest object.
+ */
+void
+crypto_digest_free_(crypto_digest_t *digest)
+{
+  if (!digest)
+    return;
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    if (digest->d.md) {
+      EVP_MD_CTX_free(digest->d.md);
+    }
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memwipe(digest, 0, bytes);
+  tor_free(digest);
+}
+
+/** Add <b>len</b> bytes from <b>data</b> to the digest object.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_add_bytess`
+ * C_RUST_COUPLED: `crypto::digest::Sha256::process`
+ */
+void
+crypto_digest_add_bytes(crypto_digest_t *digest, const char *data,
+                        size_t len)
+{
+  tor_assert(digest);
+  tor_assert(data);
+  /* Using the SHA*_*() calls directly means we don't support doing
+   * SHA in hardware. But so far the delay of getting the question
+   * to the hardware, and hearing the answer, is likely higher than
+   * just doing it ourselves. Hashes are fast.
+   */
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Update(&digest->d.sha1, (void*)data, len);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Update(&digest->d.sha2, (void*)data, len);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Update(&digest->d.sha512, (void*)data, len);
+      break;
+#ifdef OPENSSL_HAS_SHA3
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512: {
+      int r = EVP_DigestUpdate(digest->d.md, data, len);
+      tor_assert(r);
+  }
+      break;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+      keccak_digest_update(&digest->d.sha3, (const uint8_t *)data, len);
+      break;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+    default:
+      /* LCOV_EXCL_START */
+      tor_fragile_assert();
+      break;
+      /* LCOV_EXCL_STOP */
+  }
+}
+
+/** Compute the hash of the data that has been passed to the digest
+ * object; write the first out_len bytes of the result to <b>out</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN.
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_get_digest`
+ * C_RUST_COUPLED: `impl digest::FixedOutput for Sha256`
+ */
+void
+crypto_digest_get_digest(crypto_digest_t *digest,
+                         char *out, size_t out_len)
+{
+  unsigned char r[DIGEST512_LEN];
+  tor_assert(digest);
+  tor_assert(out);
+  tor_assert(out_len <= crypto_digest_algorithm_get_length(digest->algorithm));
+
+  /* The SHA-3 code handles copying into a temporary ctx, and also can handle
+   * short output buffers by truncating appropriately. */
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+#ifdef OPENSSL_HAS_SHA3
+    unsigned dlen = (unsigned)
+      crypto_digest_algorithm_get_length(digest->algorithm);
+    EVP_MD_CTX *tmp = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(tmp, digest->d.md);
+    memset(r, 0xff, sizeof(r));
+    int res = EVP_DigestFinal(tmp, r, &dlen);
+    EVP_MD_CTX_free(tmp);
+    tor_assert(res == 1);
+    goto done;
+#else /* !defined(OPENSSL_HAS_SHA3) */
+    /* Tiny-Keccak handles copying into a temporary ctx, and also can handle
+     * short output buffers by truncating appropriately. */
+    keccak_digest_sum(&digest->d.sha3, (uint8_t *)out, out_len);
+    return;
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  }
+
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t tmpenv;
+  /* memcpy into a temporary ctx, since SHA*_Final clears the context */
+  memcpy(&tmpenv, digest, alloc_bytes);
+  switch (digest->algorithm) {
+    case DIGEST_SHA1:
+      SHA1_Final(r, &tmpenv.d.sha1);
+      break;
+    case DIGEST_SHA256:
+      SHA256_Final(r, &tmpenv.d.sha2);
+      break;
+    case DIGEST_SHA512:
+      SHA512_Final(r, &tmpenv.d.sha512);
+      break;
+//LCOV_EXCL_START
+    case DIGEST_SHA3_256: FALLTHROUGH;
+    case DIGEST_SHA3_512:
+    default:
+      log_warn(LD_BUG, "Handling unexpected algorithm %d", digest->algorithm);
+      /* This is fatal, because it should never happen. */
+      tor_assert_unreached();
+      break;
+//LCOV_EXCL_STOP
+  }
+#ifdef OPENSSL_HAS_SHA3
+ done:
+#endif
+  memcpy(out, r, out_len);
+  memwipe(r, 0, sizeof(r));
+}
+
+/** Allocate and return a new digest object with the same state as
+ * <b>digest</b>
+ *
+ * C_RUST_COUPLED: `external::crypto_digest::crypto_digest_dup`
+ * C_RUST_COUPLED: `impl Clone for crypto::digest::Sha256`
+ */
+crypto_digest_t *
+crypto_digest_dup(const crypto_digest_t *digest)
+{
+  tor_assert(digest);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  crypto_digest_t *result = tor_memdup(digest, alloc_bytes);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (digest->algorithm == DIGEST_SHA3_256 ||
+      digest->algorithm == DIGEST_SHA3_512) {
+    result->d.md = EVP_MD_CTX_new();
+    EVP_MD_CTX_copy(result->d.md, digest->d.md);
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+  return result;
+}
+
+/** Temporarily save the state of <b>digest</b> in <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object.
+ */
+void
+crypto_digest_checkpoint(crypto_digest_checkpoint_t *checkpoint,
+                         const crypto_digest_t *digest)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  tor_assert(bytes <= sizeof(checkpoint->mem));
+  memcpy(checkpoint->mem, digest, bytes);
+}
+
+/** Restore the state of  <b>digest</b> from <b>checkpoint</b>.
+ * Asserts that <b>digest</b> is a SHA1 digest object. Requires that the
+ * state was previously stored with crypto_digest_checkpoint() */
+void
+crypto_digest_restore(crypto_digest_t *digest,
+                      const crypto_digest_checkpoint_t *checkpoint)
+{
+  const size_t bytes = crypto_digest_alloc_bytes(digest->algorithm);
+  memcpy(digest, checkpoint->mem, bytes);
+}
+
+/** Replace the state of the digest object <b>into</b> with the state
+ * of the digest object <b>from</b>.  Requires that 'into' and 'from'
+ * have the same digest type.
+ */
+void
+crypto_digest_assign(crypto_digest_t *into,
+                     const crypto_digest_t *from)
+{
+  tor_assert(into);
+  tor_assert(from);
+  tor_assert(into->algorithm == from->algorithm);
+  const size_t alloc_bytes = crypto_digest_alloc_bytes(from->algorithm);
+
+#ifdef OPENSSL_HAS_SHA3
+  if (from->algorithm == DIGEST_SHA3_256 ||
+      from->algorithm == DIGEST_SHA3_512) {
+    EVP_MD_CTX_copy(into->d.md, from->d.md);
+    return;
+  }
+#endif /* defined(OPENSSL_HAS_SHA3) */
+
+  memcpy(into,from,alloc_bytes);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of those strings,
+ * plus the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>out_len</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist(char *digest_out, size_t len_out,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_smartlist_prefix(digest_out, len_out, NULL, lst, append, alg);
+}
+
+/** Given a list of strings in <b>lst</b>, set the <b>len_out</b>-byte digest
+ * at <b>digest_out</b> to the hash of the concatenation of: the
+ * optional string <b>prepend</b>, those strings,
+ * and the optional string <b>append</b>, computed with the algorithm
+ * <b>alg</b>.
+ * <b>len_out</b> must be \<= DIGEST512_LEN. */
+void
+crypto_digest_smartlist_prefix(char *digest_out, size_t len_out,
+                        const char *prepend,
+                        const smartlist_t *lst,
+                        const char *append,
+                        digest_algorithm_t alg)
+{
+  crypto_digest_t *d = crypto_digest_new_internal(alg);
+  if (prepend)
+    crypto_digest_add_bytes(d, prepend, strlen(prepend));
+  SMARTLIST_FOREACH(lst, const char *, cp,
+                    crypto_digest_add_bytes(d, cp, strlen(cp)));
+  if (append)
+    crypto_digest_add_bytes(d, append, strlen(append));
+  crypto_digest_get_digest(d, digest_out, len_out);
+  crypto_digest_free(d);
+}
+
+/** Compute the HMAC-SHA-256 of the <b>msg_len</b> bytes in <b>msg</b>, using
+ * the <b>key</b> of length <b>key_len</b>.  Store the DIGEST256_LEN-byte
+ * result in <b>hmac_out</b>. Asserts on failure.
+ */
+void
+crypto_hmac_sha256(char *hmac_out,
+                   const char *key, size_t key_len,
+                   const char *msg, size_t msg_len)
+{
+  /* If we've got OpenSSL >=0.9.8 we can use its hmac implementation. */
+  tor_assert(key_len < INT_MAX);
+  tor_assert(msg_len < INT_MAX);
+  tor_assert(hmac_out);
+  unsigned char *rv = NULL;
+  rv = HMAC(EVP_sha256(), key, (int)key_len, (unsigned char*)msg, (int)msg_len,
+            (unsigned char*)hmac_out, NULL);
+  tor_assert(rv);
+}
diff --git a/src/lib/crypt_ops/crypto_ed25519.c b/src/lib/crypt_ops/crypto_ed25519.c
index 0a442bb739..f242c7011e 100644
--- a/src/lib/crypt_ops/crypto_ed25519.c
+++ b/src/lib/crypt_ops/crypto_ed25519.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -226,7 +226,7 @@ ed25519_keypair_generate(ed25519_keypair_t *keypair_out, int extra_strong)
 int
 ed25519_public_key_is_zero(const ed25519_public_key_t *pubkey)
 {
-  return tor_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
+  return safe_mem_is_zero((char*)pubkey->pubkey, ED25519_PUBKEY_LEN);
 }
 
 /* Return a heap-allocated array that contains <b>msg</b> prefixed by the
diff --git a/src/lib/crypt_ops/crypto_ed25519.h b/src/lib/crypt_ops/crypto_ed25519.h
index 325b28244d..346de464e3 100644
--- a/src/lib/crypt_ops/crypto_ed25519.h
+++ b/src/lib/crypt_ops/crypto_ed25519.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_format.c b/src/lib/crypt_ops/crypto_format.c
index 84f73e5272..92b8b9372e 100644
--- a/src/lib/crypt_ops/crypto_format.c
+++ b/src/lib/crypt_ops/crypto_format.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -104,7 +104,7 @@ crypto_read_tagged_contents_from_file(const char *fname,
   prefix[32] = 0;
   /* Check type, extract tag. */
   if (strcmpstart(prefix, "== ") || strcmpend(prefix, " ==") ||
-      ! tor_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
+      ! fast_mem_is_zero(prefix+strlen(prefix), 32-strlen(prefix))) {
     saved_errno = EINVAL;
     goto end;
   }
@@ -131,20 +131,27 @@ crypto_read_tagged_contents_from_file(const char *fname,
   return r;
 }
 
-/** Encode <b>pkey</b> as a base64-encoded string, without trailing "="
+/** Encode <b>pkey</b> as a base64-encoded string, including trailing "="
  * characters, in the buffer <b>output</b>, which must have at least
- * CURVE25519_BASE64_PADDED_LEN+1 bytes available.  Return 0 on success, -1 on
- * failure. */
-int
+ * CURVE25519_BASE64_PADDED_LEN+1 bytes available.
+ * Can not fail.
+ *
+ * Careful! CURVE25519_BASE64_PADDED_LEN is one byte longer than
+ * ED25519_BASE64_LEN.
+ */
+void
 curve25519_public_to_base64(char *output,
                             const curve25519_public_key_t *pkey)
 {
   char buf[128];
-  base64_encode(buf, sizeof(buf),
-                (const char*)pkey->public_key, CURVE25519_PUBKEY_LEN, 0);
-  buf[CURVE25519_BASE64_PADDED_LEN] = '\0';
+  int n = base64_encode(buf, sizeof(buf),
+                        (const char*)pkey->public_key,
+                        CURVE25519_PUBKEY_LEN, 0);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode(). */
+  tor_assert(n == CURVE25519_BASE64_PADDED_LEN);
+  tor_assert(buf[CURVE25519_BASE64_PADDED_LEN] == '\0');
   memcpy(output, buf, CURVE25519_BASE64_PADDED_LEN+1);
-  return 0;
 }
 
 /** Try to decode a base64-encoded curve25519 public key from <b>input</b>
@@ -181,8 +188,7 @@ ed25519_fmt(const ed25519_public_key_t *pkey)
     if (ed25519_public_key_is_zero(pkey)) {
       strlcpy(formatted, "<unset>", sizeof(formatted));
     } else {
-      int r = ed25519_public_to_base64(formatted, pkey);
-      tor_assert(!r);
+      ed25519_public_to_base64(formatted, pkey);
     }
   } else {
     strlcpy(formatted, "<null>", sizeof(formatted));
@@ -202,28 +208,35 @@ ed25519_public_from_base64(ed25519_public_key_t *pkey,
 
 /** Encode the public key <b>pkey</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_BASE64_LEN bytes of encoded key,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
+ *
+ * Careful! ED25519_BASE64_LEN is one byte shorter than
+ * CURVE25519_BASE64_PADDED_LEN.
  */
-int
+void
 ed25519_public_to_base64(char *output,
                          const ed25519_public_key_t *pkey)
 {
-  return digest256_to_base64(output, (const char *)pkey->pubkey);
+  digest256_to_base64(output, (const char *)pkey->pubkey);
 }
 
 /** Encode the signature <b>sig</b> into the buffer at <b>output</b>,
  * which must have space for ED25519_SIG_BASE64_LEN bytes of encoded signature,
- * plus one byte for a terminating NUL.  Return 0 on success, -1 on failure.
+ * plus one byte for a terminating NUL.
+ * Can not fail.
  */
-int
+void
 ed25519_signature_to_base64(char *output,
                             const ed25519_signature_t *sig)
 {
   char buf[256];
   int n = base64_encode_nopad(buf, sizeof(buf), sig->sig, ED25519_SIG_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
   tor_assert(n == ED25519_SIG_BASE64_LEN);
+  tor_assert(buf[ED25519_SIG_BASE64_LEN] == '\0');
   memcpy(output, buf, ED25519_SIG_BASE64_LEN+1);
-  return 0;
 }
 
 /** Try to decode the string <b>input</b> into an ed25519 signature. On
@@ -233,16 +246,11 @@ int
 ed25519_signature_from_base64(ed25519_signature_t *sig,
                               const char *input)
 {
-
   if (strlen(input) != ED25519_SIG_BASE64_LEN)
     return -1;
-  char buf[ED25519_SIG_BASE64_LEN+3];
-  memcpy(buf, input, ED25519_SIG_BASE64_LEN);
-  buf[ED25519_SIG_BASE64_LEN+0] = '=';
-  buf[ED25519_SIG_BASE64_LEN+1] = '=';
-  buf[ED25519_SIG_BASE64_LEN+2] = 0;
   char decoded[128];
-  int n = base64_decode(decoded, sizeof(decoded), buf, strlen(buf));
+  int n = base64_decode(decoded, sizeof(decoded), input,
+                        ED25519_SIG_BASE64_LEN);
   if (n < 0 || n != ED25519_SIG_LEN)
     return -1;
   memcpy(sig->sig, decoded, ED25519_SIG_LEN);
@@ -250,24 +258,26 @@ ed25519_signature_from_base64(ed25519_signature_t *sig,
   return 0;
 }
 
-/** Base64 encode DIGEST_LINE bytes from <b>digest</b>, remove the trailing =
+/** Base64 encode DIGEST_LEN bytes from <b>digest</b>, remove the trailing =
  * characters, and store the nul-terminated result in the first
- * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.  */
-/* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST_LEN, 0);
-  buf[BASE64_DIGEST_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST_LEN);
+  tor_assert(buf[BASE64_DIGEST_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest_from_base64(char *digest, const char *d64)
 {
@@ -279,22 +289,24 @@ digest_from_base64(char *digest, const char *d64)
 
 /** Base64 encode DIGEST256_LINE bytes from <b>digest</b>, remove the
  * trailing = characters, and store the nul-terminated result in the first
- * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>. */
- /* XXXX unify with crypto_format.c code */
-int
+ * BASE64_DIGEST256_LEN+1 bytes of <b>d64</b>.
+ * Can not fail. */
+void
 digest256_to_base64(char *d64, const char *digest)
 {
   char buf[256];
-  base64_encode(buf, sizeof(buf), digest, DIGEST256_LEN, 0);
-  buf[BASE64_DIGEST256_LEN] = '\0';
+  int n = base64_encode_nopad(buf, sizeof(buf),
+                              (const uint8_t *)digest, DIGEST256_LEN);
+  /* These asserts should always succeed, unless there is a bug in
+   * base64_encode_nopad(). */
+  tor_assert(n == BASE64_DIGEST256_LEN);
+  tor_assert(buf[BASE64_DIGEST256_LEN] == '\0');
   memcpy(d64, buf, BASE64_DIGEST256_LEN+1);
-  return 0;
 }
 
 /** Given a base64 encoded, nul-terminated digest in <b>d64</b> (without
  * trailing newline or = characters), decode it and store the result in the
  * first DIGEST256_LEN bytes at <b>digest</b>. */
-/* XXXX unify with crypto_format.c code */
 int
 digest256_from_base64(char *digest, const char *d64)
 {
diff --git a/src/lib/crypt_ops/crypto_format.h b/src/lib/crypt_ops/crypto_format.h
index fe852e6a61..91da137e1c 100644
--- a/src/lib/crypt_ops/crypto_format.h
+++ b/src/lib/crypt_ops/crypto_format.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,18 +33,18 @@ ssize_t crypto_read_tagged_contents_from_file(const char *fname,
 
 int ed25519_public_from_base64(struct ed25519_public_key_t *pkey,
                                const char *input);
-int ed25519_public_to_base64(char *output,
-                             const struct ed25519_public_key_t *pkey);
+void ed25519_public_to_base64(char *output,
+                              const struct ed25519_public_key_t *pkey);
 const char *ed25519_fmt(const struct ed25519_public_key_t *pkey);
 
 int ed25519_signature_from_base64(struct ed25519_signature_t *sig,
                                   const char *input);
-int ed25519_signature_to_base64(char *output,
-                                const struct ed25519_signature_t *sig);
+void ed25519_signature_to_base64(char *output,
+                                 const struct ed25519_signature_t *sig);
 
-int digest_to_base64(char *d64, const char *digest);
+void digest_to_base64(char *d64, const char *digest);
 int digest_from_base64(char *digest, const char *d64);
-int digest256_to_base64(char *d64, const char *digest);
+void digest256_to_base64(char *d64, const char *digest);
 int digest256_from_base64(char *digest, const char *d64);
 
 #endif /* !defined(TOR_CRYPTO_FORMAT_H) */
diff --git a/src/lib/crypt_ops/crypto_hkdf.c b/src/lib/crypt_ops/crypto_hkdf.c
index fd2e701651..7b02820087 100644
--- a/src/lib/crypt_ops/crypto_hkdf.c
+++ b/src/lib/crypt_ops/crypto_hkdf.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
 #include <openssl/kdf.h>
 #define HAVE_OPENSSL_HKDF 1
 #endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <string.h>
 
@@ -109,7 +109,7 @@ crypto_expand_key_material_rfc5869_sha256_openssl(
   return 0;
 }
 
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
 
 /**
  * Perform RFC5869 HKDF computation using our own legacy implementation.
@@ -166,7 +166,7 @@ crypto_expand_key_material_rfc5869_sha256_legacy(
   memwipe(mac, 0, sizeof(mac));
   return 0;
 }
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 
 /** Expand some secret key material according to RFC5869, using SHA256 as the
  * underlying hash.  The <b>key_in_len</b> bytes at <b>key_in</b> are the
@@ -191,11 +191,11 @@ crypto_expand_key_material_rfc5869_sha256(
                                              salt_in_len, info_in,
                                              info_in_len,
                                              key_out, key_out_len);
-#else
+#else /* !defined(HAVE_OPENSSL_HKDF) */
   return crypto_expand_key_material_rfc5869_sha256_legacy(key_in,
                                                key_in_len, salt_in,
                                                salt_in_len, info_in,
                                                info_in_len,
                                                key_out, key_out_len);
-#endif
+#endif /* defined(HAVE_OPENSSL_HKDF) */
 }
diff --git a/src/lib/crypt_ops/crypto_hkdf.h b/src/lib/crypt_ops/crypto_hkdf.h
index 2994d18e3d..404f548774 100644
--- a/src/lib/crypt_ops/crypto_hkdf.h
+++ b/src/lib/crypt_ops/crypto_hkdf.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_init.c b/src/lib/crypt_ops/crypto_init.c
index 329c264af6..a836bd8645 100644
--- a/src/lib/crypt_ops/crypto_init.c
+++ b/src/lib/crypt_ops/crypto_init.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,8 @@
 
 #include "orconfig.h"
 
+#define CRYPTO_PRIVATE
+
 #include "lib/crypt_ops/crypto_init.h"
 
 #include "lib/crypt_ops/crypto_curve25519.h"
@@ -20,8 +22,14 @@
 #include "lib/crypt_ops/crypto_openssl_mgt.h"
 #include "lib/crypt_ops/crypto_nss_mgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_sys.h"
+#include "lib/crypt_ops/crypto_options_st.h"
+#include "lib/conf/conftypes.h"
+#include "lib/log/util_bug.h"
+
+#include "lib/subsys/subsys.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** Boolean: has our crypto library been initialized? (early phase) */
 static int crypto_early_initialized_ = 0;
@@ -66,6 +74,8 @@ crypto_early_init(void)
     if (crypto_init_siphash_key() < 0)
       return -1;
 
+    crypto_rand_fast_init();
+
     curve25519_init();
     ed25519_init();
   }
@@ -92,7 +102,7 @@ crypto_global_init(int useAccel, const char *accelName, const char *accelDir)
     (void)useAccel;
     (void)accelName;
     (void)accelDir;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
     if (crypto_nss_late_init() < 0)
       return -1;
@@ -108,6 +118,7 @@ crypto_thread_cleanup(void)
 #ifdef ENABLE_OPENSSL
   crypto_openssl_thread_cleanup();
 #endif
+  destroy_thread_fast_rng();
 }
 
 /**
@@ -126,6 +137,8 @@ crypto_global_cleanup(void)
   crypto_nss_global_cleanup();
 #endif
 
+  crypto_rand_fast_shutdown();
+
   crypto_early_initialized_ = 0;
   crypto_global_initialized_ = 0;
   have_seeded_siphash = 0;
@@ -142,6 +155,12 @@ crypto_prefork(void)
 #ifdef ENABLE_NSS
   crypto_nss_prefork();
 #endif
+  /* It is not safe to share a fast_rng object across a fork boundary unless
+   * we actually have zero-on-fork support in map_anon.c.  If we have
+   * drop-on-fork support, we will crash; if we have neither, we will yield
+   * a copy of the parent process's rng, which is scary and insecure.
+   */
+  destroy_thread_fast_rng();
 }
 
 /** Run operations that the crypto library requires to be happy again
@@ -202,3 +221,111 @@ tor_is_using_nss(void)
   return 0;
 #endif
 }
+
+static int
+subsys_crypto_initialize(void)
+{
+  if (crypto_early_init() < 0)
+    return -1;
+  crypto_dh_init();
+  return 0;
+}
+
+static void
+subsys_crypto_shutdown(void)
+{
+  crypto_global_cleanup();
+}
+
+static void
+subsys_crypto_prefork(void)
+{
+  crypto_prefork();
+}
+
+static void
+subsys_crypto_postfork(void)
+{
+  crypto_postfork();
+}
+
+static void
+subsys_crypto_thread_cleanup(void)
+{
+  crypto_thread_cleanup();
+}
+
+/** Magic number for crypto_options_t. */
+#define CRYPTO_OPTIONS_MAGIC 0x68757368
+
+/**
+ * Return 0 if <b>arg</b> is a valid crypto_options_t.  Otherwise return -1
+ * and set *<b>msg_out</b> to a freshly allocated error string.
+ **/
+static int
+crypto_options_validate(const void *arg, char **msg_out)
+{
+  const crypto_options_t *opt = arg;
+  tor_assert(opt->magic == CRYPTO_OPTIONS_MAGIC);
+  tor_assert(msg_out);
+
+  if (opt->AccelDir && !opt->AccelName) {
+    *msg_out = tor_strdup("Can't use hardware crypto accelerator dir "
+                          "without engine name.");
+    return -1;
+  }
+
+  return 0;
+}
+
+/* Declare the options field table for crypto_options */
+#define CONF_CONTEXT LL_TABLE
+#include "lib/crypt_ops/crypto_options.inc"
+#undef CONF_CONTEXT
+
+/**
+ * Declares the configuration options for this module.
+ **/
+static const config_format_t crypto_options_fmt = {
+  .size = sizeof(crypto_options_t),
+  .magic = { "crypto_options_t",
+             CRYPTO_OPTIONS_MAGIC,
+             offsetof(crypto_options_t, magic) },
+  .vars = crypto_options_t_vars,
+  .validate_fn = crypto_options_validate,
+};
+
+/**
+ * Invoked from subsysmgr.c when a new set of options arrives.
+ **/
+static int
+crypto_set_options(void *arg)
+{
+  const crypto_options_t *options = arg;
+  const bool hardware_accel = options->HardwareAccel || options->AccelName;
+
+  // This call already checks for crypto_global_initialized_, so it
+  // will only initialize the subsystem the first time it's called.
+  if (crypto_global_init(hardware_accel,
+                         options->AccelName,
+                         options->AccelDir)) {
+    log_err(LD_BUG, "Unable to initialize the crypto subsystem. Exiting.");
+    return -1;
+  }
+  return 0;
+}
+
+const struct subsys_fns_t sys_crypto = {
+  .name = "crypto",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -60,
+  .initialize = subsys_crypto_initialize,
+  .shutdown = subsys_crypto_shutdown,
+  .prefork = subsys_crypto_prefork,
+  .postfork = subsys_crypto_postfork,
+  .thread_cleanup = subsys_crypto_thread_cleanup,
+
+  .options_format = &crypto_options_fmt,
+  .set_options = crypto_set_options,
+};
diff --git a/src/lib/crypt_ops/crypto_init.h b/src/lib/crypt_ops/crypto_init.h
index 540d08eb56..b11e2e34bf 100644
--- a/src/lib/crypt_ops/crypto_init.h
+++ b/src/lib/crypt_ops/crypto_init.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -33,4 +33,4 @@ const char *crypto_get_header_version_string(void);
 
 int tor_is_using_nss(void);
 
-#endif /* !defined(TOR_CRYPTO_H) */
+#endif /* !defined(TOR_CRYPTO_INIT_H) */
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.c b/src/lib/crypt_ops/crypto_nss_mgt.c
index 0179126e38..d82e51249c 100644
--- a/src/lib/crypt_ops/crypto_nss_mgt.c
+++ b/src/lib/crypt_ops/crypto_nss_mgt.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/string/printf.h"
 
-DISABLE_GCC_WARNING(strict-prototypes)
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
 #include <nss.h>
 #include <pk11func.h>
 #include <ssl.h>
@@ -24,7 +24,7 @@ DISABLE_GCC_WARNING(strict-prototypes)
 #include <prerror.h>
 #include <prtypes.h>
 #include <prinit.h>
-ENABLE_GCC_WARNING(strict-prototypes)
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
 
 const char *
 crypto_nss_get_version_str(void)
diff --git a/src/lib/crypt_ops/crypto_nss_mgt.h b/src/lib/crypt_ops/crypto_nss_mgt.h
index 72fd2a1229..8686b1b8aa 100644
--- a/src/lib/crypt_ops/crypto_nss_mgt.h
+++ b/src/lib/crypt_ops/crypto_nss_mgt.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,6 @@ void crypto_nss_global_cleanup(void);
 
 void crypto_nss_prefork(void);
 void crypto_nss_postfork(void);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
-#endif /* !defined(TOR_CRYPTO_NSS_H) */
+#endif /* !defined(TOR_CRYPTO_NSS_MGT_H) */
diff --git a/src/lib/crypt_ops/crypto_ope.c b/src/lib/crypt_ops/crypto_ope.c
index 2186d2a939..4cacb3dd98 100644
--- a/src/lib/crypt_ops/crypto_ope.c
+++ b/src/lib/crypt_ops/crypto_ope.c
@@ -1,8 +1,9 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
- * A rudimentary order-preserving encryption scheme.
+ * @file crypto_ope.c
+ * @brief A rudimentary order-preserving encryption scheme.
  *
  * To compute the encryption of N, this scheme uses an AES-CTR stream to
  * generate M-byte values, and adds the first N of them together. (+1 each to
@@ -57,9 +58,9 @@ ope_val_from_le(ope_val_t x)
     ((x) >> 8) |
     (((x)&0xff) << 8);
 }
-#else
+#else /* !defined(WORDS_BIGENDIAN) */
 #define ope_val_from_le(x) (x)
-#endif
+#endif /* defined(WORDS_BIGENDIAN) */
 
 /**
  * Return a new AES256-CTR stream cipher object for <b>ope</b>, ready to yield
@@ -143,7 +144,7 @@ crypto_ope_new(const uint8_t *key)
   return ope;
 }
 
-/** Free all storage held in <>ope</b>. */
+/** Free all storage held in <b>ope</b>. */
 void
 crypto_ope_free_(crypto_ope_t *ope)
 {
diff --git a/src/lib/crypt_ops/crypto_ope.h b/src/lib/crypt_ops/crypto_ope.h
index 610d956335..7498ea6a2e 100644
--- a/src/lib/crypt_ops/crypto_ope.h
+++ b/src/lib/crypt_ops/crypto_ope.h
@@ -1,6 +1,11 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file crypto_ope.h
+ * @brief header for crypto_ope.c
+ **/
+
 #ifndef CRYPTO_OPE_H
 #define CRYPTO_OPE_H
 
@@ -37,10 +42,10 @@ void crypto_ope_free_(crypto_ope_t *ope);
 uint64_t crypto_ope_encrypt(const crypto_ope_t *ope, int plaintext);
 
 #ifdef CRYPTO_OPE_PRIVATE
-struct aes_cnt_cipher;
-STATIC struct aes_cnt_cipher *ope_get_cipher(const crypto_ope_t *ope,
+struct aes_cnt_cipher_t;
+STATIC struct aes_cnt_cipher_t *ope_get_cipher(const crypto_ope_t *ope,
                                               uint32_t initial_idx);
-STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher *c, size_t n);
-#endif
+STATIC uint64_t sum_values_from_cipher(struct aes_cnt_cipher_t *c, size_t n);
+#endif /* defined(CRYPTO_OPE_PRIVATE) */
 
-#endif
+#endif /* !defined(CRYPTO_OPE_H) */
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.c b/src/lib/crypt_ops/crypto_openssl_mgt.c
index e7d6084f6c..065cbca1cc 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.c
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,7 +21,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/thread/threads.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/err.h>
 #include <openssl/rsa.h>
@@ -36,7 +36,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/crypto.h>
 #include <openssl/ssl.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <string.h>
 
@@ -130,10 +130,12 @@ crypto_openssl_get_header_version_str(void)
   return crypto_openssl_header_version_str;
 }
 
+#ifndef COCCI
 #ifndef OPENSSL_THREADS
-#error OpenSSL has been built without thread support. Tor requires an \
- OpenSSL library with thread support enabled.
+#error "OpenSSL has been built without thread support. Tor requires an \
+ OpenSSL library with thread support enabled."
 #endif
+#endif /* !defined(COCCI) */
 
 #ifndef NEW_THREAD_API
 /** Helper: OpenSSL uses this callback to manipulate mutexes. */
@@ -185,6 +187,10 @@ crypto_openssl_free_all(void)
   tor_free(crypto_openssl_version_str);
   tor_free(crypto_openssl_header_version_str);
 
+  /* Destroying a locked mutex is undefined behaviour. This mutex may be
+   * locked, because multiple threads can access it. But we need to destroy
+   * it, otherwise re-initialisation will trigger undefined behaviour.
+   * See #31735 for details. */
 #ifndef NEW_THREAD_API
   if (n_openssl_mutexes_) {
     int n = n_openssl_mutexes_;
@@ -209,10 +215,10 @@ crypto_openssl_early_init(void)
                      OPENSSL_INIT_LOAD_CRYPTO_STRINGS |
                      OPENSSL_INIT_ADD_ALL_CIPHERS |
                      OPENSSL_INIT_ADD_ALL_DIGESTS, NULL);
-#else
+#else /* !defined(OPENSSL_1_1_API) */
     ERR_load_crypto_strings();
     OpenSSL_add_all_algorithms();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
     setup_openssl_threading();
 
@@ -278,8 +284,14 @@ log_engine(const char *fn, ENGINE *e)
 }
 #endif /* !defined(DISABLE_ENGINES) */
 
-/** Initialize engines for openssl (if enabled). */
-static void
+/** Initialize engines for openssl (if enabled).  Load all the built-in
+ * engines, along with the one called <b>accelName</b> (which may be NULL).
+ * If <b>accelName</b> is prefixed with "!", then it is required: return -1
+ * if it can't be loaded.  Otherwise return 0.
+ *
+ * If <b>accelDir</b> is not NULL, it is the path from which the engine should
+ * be loaded. */
+static int
 crypto_openssl_init_engines(const char *accelName,
                             const char *accelDir)
 {
@@ -287,7 +299,13 @@ crypto_openssl_init_engines(const char *accelName,
   (void)accelName;
   (void)accelDir;
   log_warn(LD_CRYPTO, "No OpenSSL hardware acceleration support enabled.");
-#else
+  if (accelName && accelName[0] == '!') {
+    log_warn(LD_CRYPTO, "Unable to load required dynamic OpenSSL engine "
+             "\"%s\".", accelName+1);
+    return -1;
+  }
+  return 0;
+#else /* !defined(DISABLE_ENGINES) */
   ENGINE *e = NULL;
 
   log_info(LD_CRYPTO, "Initializing OpenSSL engine support.");
@@ -295,6 +313,9 @@ crypto_openssl_init_engines(const char *accelName,
   ENGINE_register_all_complete();
 
   if (accelName) {
+    const bool required = accelName[0] == '!';
+    if (required)
+      ++accelName;
     if (accelDir) {
       log_info(LD_CRYPTO, "Trying to load dynamic OpenSSL engine \"%s\""
                " via path \"%s\".", accelName, accelDir);
@@ -305,8 +326,11 @@ crypto_openssl_init_engines(const char *accelName,
       e = ENGINE_by_id(accelName);
     }
     if (!e) {
-      log_warn(LD_CRYPTO, "Unable to load dynamic OpenSSL engine \"%s\".",
+      log_warn(LD_CRYPTO, "Unable to load %sdynamic OpenSSL engine \"%s\".",
+               required?"required ":"",
                accelName);
+      if (required)
+        return -1;
     } else {
       log_info(LD_CRYPTO, "Loaded dynamic OpenSSL engine \"%s\".",
                accelName);
@@ -343,6 +367,7 @@ crypto_openssl_init_engines(const char *accelName,
 #ifdef NID_aes_256_gcm
   log_engine("AES-256-GCM", ENGINE_get_cipher_engine(NID_aes_256_gcm));
 #endif
+  return 0;
 
 #endif /* defined(DISABLE_ENGINES) */
 }
@@ -353,7 +378,8 @@ crypto_openssl_late_init(int useAccel, const char *accelName,
                          const char *accelDir)
 {
   if (useAccel > 0) {
-    crypto_openssl_init_engines(accelName, accelDir);
+    if (crypto_openssl_init_engines(accelName, accelDir) < 0)
+      return -1;
   } else {
     log_info(LD_CRYPTO, "NOT using OpenSSL engine support.");
   }
@@ -382,7 +408,7 @@ crypto_openssl_thread_cleanup(void)
 void
 crypto_openssl_global_cleanup(void)
 {
-  #ifndef OPENSSL_1_1_API
+#ifndef OPENSSL_1_1_API
   EVP_cleanup();
 #endif
 #ifndef NEW_THREAD_API
diff --git a/src/lib/crypt_ops/crypto_openssl_mgt.h b/src/lib/crypt_ops/crypto_openssl_mgt.h
index 8dbadfc9d2..c67ab6467c 100644
--- a/src/lib/crypt_ops/crypto_openssl_mgt.h
+++ b/src/lib/crypt_ops/crypto_openssl_mgt.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -83,6 +83,6 @@ int crypto_openssl_late_init(int useAccel, const char *accelName,
 void crypto_openssl_thread_cleanup(void);
 void crypto_openssl_global_cleanup(void);
 
-#endif /* ENABLE_OPENSSL */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #endif /* !defined(TOR_CRYPTO_OPENSSL_H) */
diff --git a/src/lib/crypt_ops/crypto_options.inc b/src/lib/crypt_ops/crypto_options.inc
new file mode 100644
index 0000000000..5bee0daacd
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_options.inc
@@ -0,0 +1,19 @@
+
+/**
+ * @file crypto_options.inc
+ * @brief Declare configuration options for the crypto_ops module.
+ **/
+
+/** Holds configuration about our cryptography options. */
+BEGIN_CONF_STRUCT(crypto_options_t)
+
+/** Should we enable extra OpenSSL hardware acceleration (where available)?  */
+CONF_VAR(HardwareAccel, BOOL, CFLG_IMMUTABLE, "0")
+
+/** Optional OpenSSL hardware-acceleration engine name */
+CONF_VAR(AccelName, STRING, CFLG_IMMUTABLE, NULL)
+
+/** Optional OpenSSL hardware-acceleration engine search directory. */
+CONF_VAR(AccelDir, FILENAME, CFLG_IMMUTABLE, NULL)
+
+END_CONF_STRUCT(crypto_options_t)
diff --git a/src/lib/crypt_ops/crypto_options_st.h b/src/lib/crypt_ops/crypto_options_st.h
new file mode 100644
index 0000000000..a453c451fe
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_options_st.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file crypto_options_st.h
+ * @brief Header for lib/crypt_ops/crypto_options_st.c
+ **/
+
+#ifndef TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H
+#define TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H
+
+#include "lib/conf/confdecl.h"
+
+#define CONF_CONTEXT STRUCT
+#include "lib/crypt_ops/crypto_options.inc"
+#undef CONF_CONTEXT
+
+typedef struct crypto_options_t crypto_options_t;
+
+#endif /* !defined(TOR_LIB_CRYPT_OPS_CRYPTO_OPTIONS_ST_H) */
diff --git a/src/lib/crypt_ops/crypto_pwbox.c b/src/lib/crypt_ops/crypto_pwbox.c
index a8db08f7b7..bfad27d9fc 100644
--- a/src/lib/crypt_ops/crypto_pwbox.c
+++ b/src/lib/crypt_ops/crypto_pwbox.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_pwbox.h b/src/lib/crypt_ops/crypto_pwbox.h
index 5a26889fb2..0a85b1230a 100644
--- a/src/lib/crypt_ops/crypto_pwbox.h
+++ b/src/lib/crypt_ops/crypto_pwbox.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_rand.c b/src/lib/crypt_ops/crypto_rand.c
index 206929d6b3..ce6f21dbb4 100644
--- a/src/lib/crypt_ops/crypto_rand.c
+++ b/src/lib/crypt_ops/crypto_rand.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,6 @@
  * number generators, and working with randomness.
  **/
 
-#ifndef CRYPTO_RAND_PRIVATE
 #define CRYPTO_RAND_PRIVATE
 
 #include "lib/crypt_ops/crypto_rand.h"
@@ -37,17 +36,18 @@
 
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/crypto_digest.h"
+#include "lib/ctime/di_ops.h"
 
 #ifdef ENABLE_NSS
 #include "lib/crypt_ops/crypto_nss_mgt.h"
 #endif
 
 #ifdef ENABLE_OPENSSL
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 #include <openssl/rand.h>
 #include <openssl/sha.h>
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 #include <pk11pub.h>
@@ -248,7 +248,7 @@ crypto_strongest_rand_fallback(uint8_t *out, size_t out_len)
   (void)out;
   (void)out_len;
   return -1;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   static const char *filenames[] = {
     "/dev/srandom", "/dev/urandom", "/dev/random", NULL
   };
@@ -315,7 +315,7 @@ crypto_strongest_rand_raw(uint8_t *out, size_t out_len)
       }
     }
 
-    if ((out_len < sanity_min_size) || !tor_mem_is_zero((char*)out, out_len))
+    if ((out_len < sanity_min_size) || !safe_mem_is_zero((char*)out, out_len))
       return 0;
   }
 
@@ -419,7 +419,7 @@ crypto_seed_openssl_rng(void)
   else
     return -1;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 /**
@@ -442,7 +442,7 @@ crypto_seed_nss_rng(void)
 
   return load_entropy_ok ? 0 : -1;
 }
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 /**
  * Seed the RNG for any and all crypto libraries that we're using with bytes
@@ -520,7 +520,7 @@ crypto_rand_unmocked(char *to, size_t n)
 
 #undef BUFLEN
   }
-#else
+#else /* !defined(ENABLE_NSS) */
   int r = RAND_bytes((unsigned char*)to, (int)n);
   /* We consider a PRNG failure non-survivable. Let's assert so that we get a
    * stack trace about where it happened.
@@ -530,111 +530,14 @@ crypto_rand_unmocked(char *to, size_t n)
 }
 
 /**
- * Return a pseudorandom integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
- * INT_MAX+1, inclusive.
+ * Draw an unsigned 32-bit integer uniformly at random.
  */
-int
-crypto_rand_int(unsigned int max)
-{
-  unsigned int val;
-  unsigned int cutoff;
-  tor_assert(max <= ((unsigned int)INT_MAX)+1);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT_MAX - (UINT_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom integer, chosen uniformly from the values i such
- * that min <= i < max.
- *
- * <b>min</b> MUST be in range [0, <b>max</b>).
- * <b>max</b> MUST be in range (min, INT_MAX].
- **/
-int
-crypto_rand_int_range(unsigned int min, unsigned int max)
-{
-  tor_assert(min < max);
-  tor_assert(max <= INT_MAX);
-
-  /* The overflow is avoided here because crypto_rand_int() returns a value
-   * between 0 and (max - min) inclusive. */
-  return min + crypto_rand_int(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports uint64_t.
- **/
-uint64_t
-crypto_rand_uint64_range(uint64_t min, uint64_t max)
+uint32_t
+crypto_rand_u32(void)
 {
-  tor_assert(min < max);
-  return min + crypto_rand_uint64(max - min);
-}
-
-/**
- * As crypto_rand_int_range, but supports time_t.
- **/
-time_t
-crypto_rand_time_range(time_t min, time_t max)
-{
-  tor_assert(min < max);
-  return min + (time_t)crypto_rand_uint64(max - min);
-}
-
-/**
- * Return a pseudorandom 64-bit integer, chosen uniformly from the values
- * between 0 and <b>max</b>-1 inclusive.
- **/
-uint64_t
-crypto_rand_uint64(uint64_t max)
-{
-  uint64_t val;
-  uint64_t cutoff;
-  tor_assert(max < UINT64_MAX);
-  tor_assert(max > 0); /* don't div by 0 */
-
-  /* We ignore any values that are >= 'cutoff,' to avoid biasing the
-   * distribution with clipping at the upper end of unsigned int's
-   * range.
-   */
-  cutoff = UINT64_MAX - (UINT64_MAX%max);
-  while (1) {
-    crypto_rand((char*)&val, sizeof(val));
-    if (val < cutoff)
-      return val % max;
-  }
-}
-
-/**
- * Return a pseudorandom double d, chosen uniformly from the range
- * 0.0 <= d < 1.0.
- **/
-double
-crypto_rand_double(void)
-{
-  /* We just use an unsigned int here; we don't really care about getting
-   * more than 32 bits of resolution */
-  unsigned int u;
-  crypto_rand((char*)&u, sizeof(u));
-#if SIZEOF_INT == 4
-#define UINT_MAX_AS_DOUBLE 4294967296.0
-#elif SIZEOF_INT == 8
-#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
-#else
-#error SIZEOF_INT is neither 4 nor 8
-#endif /* SIZEOF_INT == 4 || ... */
-  return ((double)u) / UINT_MAX_AS_DOUBLE;
+  uint32_t rand;
+  crypto_rand((void*)&rand, sizeof(rand));
+  return rand;
 }
 
 /**
@@ -724,8 +627,6 @@ crypto_force_rand_ssleay(void)
     RAND_set_rand_method(default_method);
     return 1;
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   return 0;
 }
-
-#endif /* !defined(CRYPTO_RAND_PRIVATE) */
diff --git a/src/lib/crypt_ops/crypto_rand.h b/src/lib/crypt_ops/crypto_rand.h
index 86fa20faa3..99aff5d4a9 100644
--- a/src/lib/crypt_ops/crypto_rand.h
+++ b/src/lib/crypt_ops/crypto_rand.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/testsupport/testsupport.h"
+#include "lib/malloc/malloc.h"
 
 /* random numbers */
 int crypto_seed_rng(void) ATTR_WUR;
@@ -24,9 +25,11 @@ void crypto_rand_unmocked(char *to, size_t n);
 void crypto_strongest_rand(uint8_t *out, size_t out_len);
 MOCK_DECL(void,crypto_strongest_rand_,(uint8_t *out, size_t out_len));
 int crypto_rand_int(unsigned int max);
+unsigned crypto_rand_uint(unsigned limit);
 int crypto_rand_int_range(unsigned int min, unsigned int max);
 uint64_t crypto_rand_uint64_range(uint64_t min, uint64_t max);
 time_t crypto_rand_time_range(time_t min, time_t max);
+uint32_t crypto_rand_u32(void);
 uint64_t crypto_rand_uint64(uint64_t max);
 double crypto_rand_double(void);
 struct tor_weak_rng_t;
@@ -40,6 +43,61 @@ void *smartlist_choose(const struct smartlist_t *sl);
 void smartlist_shuffle(struct smartlist_t *sl);
 int crypto_force_rand_ssleay(void);
 
+/**
+ * A fast PRNG, for use when the PRNG provided by our crypto library isn't
+ * fast enough.  This one _should_ be cryptographically strong, but
+ * has seen less auditing than the PRNGs in OpenSSL and NSS.  Use with
+ * caution.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+typedef struct crypto_fast_rng_t crypto_fast_rng_t;
+/**
+ * Number of bytes used to seed a crypto_rand_fast_t.
+ **/
+crypto_fast_rng_t *crypto_fast_rng_new(void);
+#define CRYPTO_FAST_RNG_SEED_LEN 48
+crypto_fast_rng_t *crypto_fast_rng_new_from_seed(const uint8_t *seed);
+void crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n);
+void crypto_fast_rng_free_(crypto_fast_rng_t *);
+#define crypto_fast_rng_free(c)                                 \
+  FREE_AND_NULL(crypto_fast_rng_t, crypto_fast_rng_free_, (c))
+
+unsigned crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit);
+uint64_t crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit);
+uint32_t crypto_fast_rng_get_u32(crypto_fast_rng_t *rng);
+uint64_t crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                                      uint64_t min, uint64_t max);
+double crypto_fast_rng_get_double(crypto_fast_rng_t *rng);
+
+/**
+ * Using the fast_rng <b>rng</b>, yield true with probability
+ * 1/<b>n</b>. Otherwise yield false.
+ *
+ * <b>n</b> must not be zero.
+ **/
+#define crypto_fast_rng_one_in_n(rng, n)        \
+  (0 == (crypto_fast_rng_get_uint((rng), (n))))
+
+crypto_fast_rng_t *get_thread_fast_rng(void);
+
+#ifdef CRYPTO_PRIVATE
+/* These are only used from crypto_init.c */
+void destroy_thread_fast_rng(void);
+void crypto_rand_fast_init(void);
+void crypto_rand_fast_shutdown(void);
+#endif /* defined(CRYPTO_PRIVATE) */
+
+#if defined(TOR_UNIT_TESTS)
+/* Used for white-box testing */
+size_t crypto_fast_rng_get_bytes_used_per_stream(void);
+/* For deterministic prng implementations */
+void crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng);
+/* To override the prng for testing. */
+crypto_fast_rng_t *crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng);
+#endif /* defined(TOR_UNIT_TESTS) */
+
 #ifdef CRYPTO_RAND_PRIVATE
 
 STATIC int crypto_strongest_rand_raw(uint8_t *out, size_t out_len);
diff --git a/src/lib/crypt_ops/crypto_rand_fast.c b/src/lib/crypt_ops/crypto_rand_fast.c
new file mode 100644
index 0000000000..172ea48bdb
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_fast.c
@@ -0,0 +1,438 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_rand_fast.c
+ *
+ * \brief A fast strong PRNG for use when our underlying cryptographic
+ *   library's PRNG isn't fast enough.
+ **/
+
+/* This library is currently implemented to use the same implementation
+ * technique as libottery, using AES-CTR-256 as our underlying stream cipher.
+ * It's backtracking-resistant immediately, and prediction-resistant after
+ * a while.
+ *
+ * Here's how it works:
+ *
+ * We generate pseudorandom bytes using AES-CTR-256.  We generate BUFLEN bytes
+ * at a time.  When we do this, we keep the first SEED_LEN bytes as the key
+ * and the IV for our next invocation of AES_CTR, and yield the remaining
+ * BUFLEN - SEED_LEN bytes to the user as they invoke the PRNG.  As we yield
+ * bytes to the user, we clear them from the buffer.
+ *
+ * After we have refilled the buffer RESEED_AFTER times, we mix in an
+ * additional SEED_LEN bytes from our strong PRNG into the seed.
+ *
+ * If the user ever asks for a huge number of bytes at once, we pull SEED_LEN
+ * bytes from the PRNG and use them with our stream cipher to fill the user's
+ * request.
+ */
+
+#define CRYPTO_PRIVATE
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/crypt_ops/crypto_cipher.h"
+#include "lib/crypt_ops/crypto_digest.h"
+#include "lib/crypt_ops/crypto_util.h"
+#include "lib/intmath/cmp.h"
+#include "lib/cc/ctassert.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/thread/threads.h"
+
+#include "lib/log/util_bug.h"
+
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#include <string.h>
+
+#ifdef NOINHERIT_CAN_FAIL
+#define CHECK_PID
+#endif
+
+#ifdef CHECK_PID
+#define PID_FIELD_LEN sizeof(pid_t)
+#else
+#define PID_FIELD_LEN 0
+#endif
+
+/* Alias for CRYPTO_FAST_RNG_SEED_LEN to make our code shorter.
+ */
+#define SEED_LEN (CRYPTO_FAST_RNG_SEED_LEN)
+
+/* The amount of space that we mmap for a crypto_fast_rng_t.
+ */
+#define MAPLEN 4096
+
+/* The number of random bytes that we can yield to the user after each
+ * time we fill a crypto_fast_rng_t's buffer.
+ */
+#define BUFLEN (MAPLEN - 2*sizeof(uint16_t) - SEED_LEN - PID_FIELD_LEN)
+
+/* The number of buffer refills after which we should fetch more
+ * entropy from crypto_strongest_rand().
+ */
+#define RESEED_AFTER 16
+
+/* The length of the stream cipher key we will use for the PRNG, in bytes.
+ */
+#define KEY_LEN (CRYPTO_FAST_RNG_SEED_LEN - CIPHER_IV_LEN)
+/* The length of the stream cipher key we will use for the PRNG, in bits.
+ */
+#define KEY_BITS (KEY_LEN * 8)
+
+/* Make sure that we have a key length we can actually use with AES. */
+CTASSERT(KEY_BITS == 128 || KEY_BITS == 192 || KEY_BITS == 256);
+
+struct crypto_fast_rng_t {
+  /** How many more fills does this buffer have before we should mix
+   * in the output of crypto_strongest_rand()?
+   *
+   * This value may be negative if unit tests are enabled.  If so, it
+   * indicates that we should never mix in extra data from
+   * crypto_strongest_rand().
+   */
+  int16_t n_till_reseed;
+  /** How many bytes are remaining in cbuf_t.bytes? */
+  uint16_t bytes_left;
+#ifdef CHECK_PID
+  /** Which process owns this fast_rng?  If this value is zero, we do not
+   * need to test the owner. */
+  pid_t owner;
+#endif
+  struct cbuf_t {
+    /** The seed (key and IV) that we will use the next time that we refill
+     * cbuf_t. */
+    uint8_t seed[SEED_LEN];
+    /**
+     * Bytes that we are yielding to the user.  The next byte to be
+     * yielded is at bytes[BUFLEN-bytes_left]; all other bytes in this
+     * array are set to zero.
+     */
+    uint8_t bytes[BUFLEN];
+  } buf;
+};
+
+/* alignof(uint8_t) should be 1, so there shouldn't be any padding in cbuf_t.
+ */
+CTASSERT(sizeof(struct cbuf_t) == BUFLEN+SEED_LEN);
+/* We're trying to fit all of the RNG state into a nice mmapable chunk.
+ */
+CTASSERT(sizeof(crypto_fast_rng_t) <= MAPLEN);
+
+/**
+ * Initialize and return a new fast PRNG, using a strong random seed.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, use crypto_rand(), or wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new(void)
+{
+  uint8_t seed[SEED_LEN];
+  crypto_strongest_rand(seed, sizeof(seed));
+  crypto_fast_rng_t *result = crypto_fast_rng_new_from_seed(seed);
+  memwipe(seed, 0, sizeof(seed));
+  return result;
+}
+
+/**
+ * Initialize and return a new fast PRNG, using a seed value specified
+ * in <b>seed</b>.  This value must be CRYPTO_FAST_RNG_SEED_LEN bytes
+ * long.
+ *
+ * Note that this object is NOT thread-safe.  If you need a thread-safe
+ * prng, you should probably look at get_thread_fast_rng().  Alternatively,
+ * use crypto_rand(), wrap this in a mutex.
+ **/
+crypto_fast_rng_t *
+crypto_fast_rng_new_from_seed(const uint8_t *seed)
+{
+  unsigned inherit = INHERIT_RES_KEEP;
+  /* We try to allocate this object as securely as we can, to avoid
+   * having it get dumped, swapped, or shared after fork.
+   */
+  crypto_fast_rng_t *result = tor_mmap_anonymous(sizeof(*result),
+                                ANONMAP_PRIVATE | ANONMAP_NOINHERIT,
+                                &inherit);
+  memcpy(result->buf.seed, seed, SEED_LEN);
+  /* Causes an immediate refill once the user asks for data. */
+  result->bytes_left = 0;
+  result->n_till_reseed = RESEED_AFTER;
+#ifdef CHECK_PID
+  if (inherit == INHERIT_RES_KEEP) {
+    /* This value will neither be dropped nor zeroed after fork, so we need to
+     * check our pid to make sure we are not sharing it across a fork.  This
+     * can be expensive if the pid value isn't cached, sadly.
+     */
+    result->owner = getpid();
+  }
+#elif defined(_WIN32)
+  /* Windows can't fork(), so there's no need to noinherit. */
+#else
+  /* We decided above that noinherit would always do _something_. Assert here
+   * that we were correct. */
+  tor_assertf(inherit != INHERIT_RES_KEEP,
+              "We failed to create a non-inheritable memory region, even "
+              "though we believed such a failure to be impossible! This is "
+              "probably a bug in Tor support for your platform; please report "
+              "it.");
+#endif /* defined(CHECK_PID) || ... */
+  return result;
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Unit tests only: prevent a crypto_fast_rng_t from ever mixing in more
+ * entropy.
+ */
+void
+crypto_fast_rng_disable_reseed(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed = -1;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Helper: create a crypto_cipher_t object from SEED_LEN bytes of
+ * input.  The first KEY_LEN bytes are used as the stream cipher's key,
+ * and the remaining CIPHER_IV_LEN bytes are used as its IV.
+ **/
+static inline crypto_cipher_t *
+cipher_from_seed(const uint8_t *seed)
+{
+  return crypto_cipher_new_with_iv_and_bits(seed, seed+KEY_LEN, KEY_BITS);
+}
+
+/**
+ * Helper: mix additional entropy into <b>rng</b> by using our XOF to mix the
+ * old value for the seed with some additional bytes from
+ * crypto_strongest_rand().
+ **/
+static void
+crypto_fast_rng_add_entopy(crypto_fast_rng_t *rng)
+{
+  crypto_xof_t *xof = crypto_xof_new();
+  crypto_xof_add_bytes(xof, rng->buf.seed, SEED_LEN);
+  {
+    uint8_t seedbuf[SEED_LEN];
+    crypto_strongest_rand(seedbuf, SEED_LEN);
+    crypto_xof_add_bytes(xof, seedbuf, SEED_LEN);
+    memwipe(seedbuf, 0, SEED_LEN);
+  }
+  crypto_xof_squeeze_bytes(xof, rng->buf.seed, SEED_LEN);
+  crypto_xof_free(xof);
+}
+
+/**
+ * Helper: refill the seed bytes and output buffer of <b>rng</b>, using
+ * the input seed bytes as input (key and IV) for the stream cipher.
+ *
+ * If the n_till_reseed counter has reached zero, mix more random bytes into
+ * the seed before refilling the buffer.
+ **/
+static void
+crypto_fast_rng_refill(crypto_fast_rng_t *rng)
+{
+  rng->n_till_reseed--;
+  if (rng->n_till_reseed == 0) {
+    /* It's time to reseed the RNG. */
+    crypto_fast_rng_add_entopy(rng);
+    rng->n_till_reseed = RESEED_AFTER;
+  } else if (rng->n_till_reseed < 0) {
+#ifdef TOR_UNIT_TESTS
+    /* Reseeding is disabled for testing; never do it on this prng. */
+    rng->n_till_reseed = -1;
+#else
+    /* If testing is disabled, this shouldn't be able to become negative. */
+    tor_assert_unreached();
+#endif /* defined(TOR_UNIT_TESTS) */
+  }
+  /* Now fill rng->buf with output from our stream cipher, initialized from
+   * that seed value. */
+  crypto_cipher_t *c = cipher_from_seed(rng->buf.seed);
+  memset(&rng->buf, 0, sizeof(rng->buf));
+  crypto_cipher_crypt_inplace(c, (char*)&rng->buf, sizeof(rng->buf));
+  crypto_cipher_free(c);
+
+  rng->bytes_left = sizeof(rng->buf.bytes);
+}
+
+/**
+ * Release all storage held by <b>rng</b>.
+ **/
+void
+crypto_fast_rng_free_(crypto_fast_rng_t *rng)
+{
+  if (!rng)
+    return;
+  memwipe(rng, 0, sizeof(*rng));
+  tor_munmap_anonymous(rng, sizeof(*rng));
+}
+
+/**
+ * Helper: extract bytes from the PRNG, refilling it as necessary. Does not
+ * optimize the case when the user has asked for a huge output.
+ **/
+static void
+crypto_fast_rng_getbytes_impl(crypto_fast_rng_t *rng, uint8_t *out,
+                              const size_t n)
+{
+#ifdef CHECK_PID
+  if (rng->owner) {
+    /* Note that we only need to do this check when we have owner set: that
+     * is, when our attempt to block inheriting failed, and the result was
+     * INHERIT_RES_KEEP.
+     *
+     * If the result was INHERIT_RES_DROP, then any attempt to access the rng
+     * memory after forking will crash.
+     *
+     * If the result was INHERIT_RES_ZERO, then forking will set the bytes_left
+     * and n_till_reseed fields to zero.  This function will call
+     * crypto_fast_rng_refill(), which will in turn reseed the PRNG.
+     *
+     * So we only need to do this test in the case when mmap_anonymous()
+     * returned INHERIT_KEEP.  We avoid doing it needlessly, since getpid() is
+     * often a system call, and that can be slow.
+     */
+    tor_assert(rng->owner == getpid());
+  }
+#endif /* defined(CHECK_PID) */
+
+  size_t bytes_to_yield = n;
+
+  while (bytes_to_yield) {
+    if (rng->bytes_left == 0)
+      crypto_fast_rng_refill(rng);
+
+    const size_t to_copy = MIN(rng->bytes_left, bytes_to_yield);
+
+    tor_assert(sizeof(rng->buf.bytes) >= rng->bytes_left);
+    uint8_t *copy_from = rng->buf.bytes +
+      (sizeof(rng->buf.bytes) - rng->bytes_left);
+    memcpy(out, copy_from, to_copy);
+    memset(copy_from, 0, to_copy);
+
+    out += to_copy;
+    bytes_to_yield -= to_copy;
+    rng->bytes_left -= to_copy;
+  }
+}
+
+/**
+ * Extract <b>n</b> bytes from <b>rng</b> into the buffer at <b>out</b>.
+ **/
+void
+crypto_fast_rng_getbytes(crypto_fast_rng_t *rng, uint8_t *out, size_t n)
+{
+  if (PREDICT_UNLIKELY(n > BUFLEN)) {
+    /* The user has asked for a lot of output; generate it from a stream
+     * cipher seeded by the PRNG rather than by pulling it out of the PRNG
+     * directly.
+     */
+    uint8_t seed[SEED_LEN];
+    crypto_fast_rng_getbytes_impl(rng, seed, SEED_LEN);
+    crypto_cipher_t *c = cipher_from_seed(seed);
+    memset(out, 0, n);
+    crypto_cipher_crypt_inplace(c, (char*)out, n);
+    crypto_cipher_free(c);
+    memwipe(seed, 0, sizeof(seed));
+    return;
+  }
+
+  crypto_fast_rng_getbytes_impl(rng, out, n);
+}
+
+#if defined(TOR_UNIT_TESTS)
+/** for white-box testing: return the number of bytes that are returned from
+ * the user for each invocation of the stream cipher in this RNG. */
+size_t
+crypto_fast_rng_get_bytes_used_per_stream(void)
+{
+  return BUFLEN;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Thread-local instance for our fast RNG.
+ **/
+static tor_threadlocal_t thread_rng;
+
+/**
+ * Return a per-thread fast RNG, initializing it if necessary.
+ *
+ * You do not need to free this yourself.
+ *
+ * It is NOT safe to share this value across threads.
+ **/
+crypto_fast_rng_t *
+get_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+
+  if (PREDICT_UNLIKELY(rng == NULL)) {
+    rng = crypto_fast_rng_new();
+    tor_threadlocal_set(&thread_rng, rng);
+  }
+
+  return rng;
+}
+
+/**
+ * Used when a thread is exiting: free the per-thread fast RNG if needed.
+ * Invoked from the crypto subsystem's thread-cleanup code.
+ **/
+void
+destroy_thread_fast_rng(void)
+{
+  crypto_fast_rng_t *rng = tor_threadlocal_get(&thread_rng);
+  if (!rng)
+    return;
+  crypto_fast_rng_free(rng);
+  tor_threadlocal_set(&thread_rng, NULL);
+}
+
+#ifdef TOR_UNIT_TESTS
+/**
+ * Replace the current thread's rng with <b>rng</b>. For use by the
+ * unit tests only.  Returns the previous thread rng.
+ **/
+crypto_fast_rng_t *
+crypto_replace_thread_fast_rng(crypto_fast_rng_t *rng)
+{
+  crypto_fast_rng_t *old_rng =  tor_threadlocal_get(&thread_rng);
+  tor_threadlocal_set(&thread_rng, rng);
+  return old_rng;
+}
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * initialization code.
+ **/
+void
+crypto_rand_fast_init(void)
+{
+  tor_threadlocal_init(&thread_rng);
+}
+
+/**
+ * Initialize the global thread-local key that will be used to keep track
+ * of per-thread fast RNG instances.  Called from the crypto subsystem's
+ * shutdown code.
+ **/
+void
+crypto_rand_fast_shutdown(void)
+{
+  destroy_thread_fast_rng();
+  tor_threadlocal_destroy(&thread_rng);
+}
diff --git a/src/lib/crypt_ops/crypto_rand_numeric.c b/src/lib/crypt_ops/crypto_rand_numeric.c
new file mode 100644
index 0000000000..b2516c4bdc
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_rand_numeric.c
@@ -0,0 +1,194 @@
+/**
+ * \file crypto_rand_numeric.c
+ *
+ * \brief Functions for retrieving uniformly distributed numbers
+ *   from our PRNGs.
+ **/
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/log/util_bug.h"
+
+/**
+ * Implementation macro: yields code that returns a uniform unbiased
+ * random number between 0 and limit.  "type" is the type of the number to
+ * return; "maxval" is the largest possible value of "type"; and "fill_stmt"
+ * is a code snippet that fills an object named "val" with random bits.
+ **/
+#define IMPLEMENT_RAND_UNSIGNED(type, maxval, limit, fill_stmt)         \
+  do {                                                                  \
+    type val;                                                           \
+    type cutoff;                                                        \
+    tor_assert((limit) > 0);                                            \
+                                                                        \
+    /* We ignore any values that are >= 'cutoff,' to avoid biasing */   \
+    /* the distribution with clipping at the upper end of the type's */ \
+    /* range. */                                                        \
+    cutoff = (maxval) - ((maxval)%(limit));                             \
+    while (1) {                                                         \
+      fill_stmt;                                                        \
+      if (val < cutoff)                                                 \
+        return val % (limit);                                           \
+    }                                                                   \
+  } while (0)
+
+/**
+ * Return a pseudorandom integer chosen uniformly from the values between 0
+ * and <b>limit</b>-1 inclusive. limit must be strictly greater than 0, and
+ * less than UINT_MAX. */
+unsigned
+crypto_rand_uint(unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.  <b>max</b> must be between 1 and
+ * INT_MAX+1, inclusive.
+ */
+int
+crypto_rand_int(unsigned int max)
+{
+  /* We can't use IMPLEMENT_RAND_UNSIGNED directly, since we're trying
+   * to return a signed type. Instead we make sure that the range is
+   * reasonable for a nonnegative int, use crypto_rand_uint(), and cast.
+   */
+  tor_assert(max <= ((unsigned int)INT_MAX)+1);
+
+  return (int)crypto_rand_uint(max);
+}
+
+/**
+ * Return a pseudorandom integer, chosen uniformly from the values i such
+ * that min <= i < max.
+ *
+ * <b>min</b> MUST be in range [0, <b>max</b>).
+ * <b>max</b> MUST be in range (min, INT_MAX].
+ **/
+int
+crypto_rand_int_range(unsigned int min, unsigned int max)
+{
+  tor_assert(min < max);
+  tor_assert(max <= INT_MAX);
+
+  /* The overflow is avoided here because crypto_rand_int() returns a value
+   * between 0 and (max - min) inclusive. */
+  return min + crypto_rand_int(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports uint64_t.
+ **/
+uint64_t
+crypto_rand_uint64_range(uint64_t min, uint64_t max)
+{
+  tor_assert(min < max);
+  return min + crypto_rand_uint64(max - min);
+}
+
+/**
+ * As crypto_rand_int_range, but supports time_t.
+ **/
+time_t
+crypto_rand_time_range(time_t min, time_t max)
+{
+  tor_assert(min < max);
+  return min + (time_t)crypto_rand_uint64(max - min);
+}
+
+/**
+ * Return a pseudorandom 64-bit integer, chosen uniformly from the values
+ * between 0 and <b>max</b>-1 inclusive.
+ **/
+uint64_t
+crypto_rand_uint64(uint64_t max)
+{
+  tor_assert(max < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, max,
+                          crypto_rand((char*)&val, sizeof(val)));
+}
+
+#if SIZEOF_INT == 4
+#define UINT_MAX_AS_DOUBLE 4294967296.0
+#elif SIZEOF_INT == 8
+#define UINT_MAX_AS_DOUBLE 1.8446744073709552e+19
+#else
+#error SIZEOF_INT is neither 4 nor 8
+#endif /* SIZEOF_INT == 4 || ... */
+
+/**
+ * Return a pseudorandom double d, chosen uniformly from the range
+ * 0.0 <= d < 1.0.
+ **/
+double
+crypto_rand_double(void)
+{
+  /* We just use an unsigned int here; we don't really care about getting
+   * more than 32 bits of resolution */
+  unsigned int u;
+  crypto_rand((char*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
+/**
+ * As crypto_rand_uint, but extract the result from a crypto_fast_rng_t
+ */
+unsigned
+crypto_fast_rng_get_uint(crypto_fast_rng_t *rng, unsigned limit)
+{
+  tor_assert(limit < UINT_MAX);
+  IMPLEMENT_RAND_UNSIGNED(unsigned, UINT_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_uint64, but extract the result from a crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_get_uint64(crypto_fast_rng_t *rng, uint64_t limit)
+{
+  tor_assert(limit < UINT64_MAX);
+  IMPLEMENT_RAND_UNSIGNED(uint64_t, UINT64_MAX, limit,
+                  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val)));
+}
+
+/**
+ * As crypto_rand_u32, but extract the result from a crypto_fast_rng_t.
+ */
+uint32_t
+crypto_fast_rng_get_u32(crypto_fast_rng_t *rng)
+{
+  uint32_t val;
+  crypto_fast_rng_getbytes(rng, (void*)&val, sizeof(val));
+  return val;
+}
+
+/**
+ * As crypto_rand_uint64_range(), but extract the result from a
+ * crypto_fast_rng_t.
+ */
+uint64_t
+crypto_fast_rng_uint64_range(crypto_fast_rng_t *rng,
+                             uint64_t min, uint64_t max)
+{
+  /* Handle corrupted input */
+  if (BUG(min >= max)) {
+    return min;
+  }
+
+  return min + crypto_fast_rng_get_uint64(rng, max - min);
+}
+
+/**
+ * As crypto_rand_get_double() but extract the result from a crypto_fast_rng_t.
+ */
+double
+crypto_fast_rng_get_double(crypto_fast_rng_t *rng)
+{
+  unsigned int u;
+  crypto_fast_rng_getbytes(rng, (void*)&u, sizeof(u));
+  return ((double)u) / UINT_MAX_AS_DOUBLE;
+}
+
diff --git a/src/lib/crypt_ops/crypto_rsa.c b/src/lib/crypt_ops/crypto_rsa.c
index 8fd8a8aa7b..195e4bbaf9 100644
--- a/src/lib/crypt_ops/crypto_rsa.c
+++ b/src/lib/crypt_ops/crypto_rsa.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -59,7 +59,7 @@ crypto_get_rsa_padding(int padding)
     default: tor_assert(0); return -1; // LCOV_EXCL_LINE
     }
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Compare the public-key components of a and b.  Return non-zero iff
  * a==b.  A NULL key is considered to be distinct from all non-NULL
diff --git a/src/lib/crypt_ops/crypto_rsa.h b/src/lib/crypt_ops/crypto_rsa.h
index 6d9cc8d30e..ab2e9db80d 100644
--- a/src/lib/crypt_ops/crypto_rsa.h
+++ b/src/lib/crypt_ops/crypto_rsa.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -122,7 +122,7 @@ struct rsa_st *crypto_pk_get_openssl_rsa_(crypto_pk_t *env);
 crypto_pk_t *crypto_new_pk_from_openssl_rsa_(struct rsa_st *rsa);
 MOCK_DECL(struct evp_pkey_st *, crypto_pk_get_openssl_evp_pkey_,(
                                  crypto_pk_t *env,int private));
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef ENABLE_NSS
 struct SECKEYPublicKeyStr;
@@ -132,7 +132,7 @@ const struct SECKEYPublicKeyStr *crypto_pk_get_nss_pubkey(
                                            const crypto_pk_t *key);
 const struct SECKEYPrivateKeyStr *crypto_pk_get_nss_privkey(
                                            const crypto_pk_t *key);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
 void crypto_pk_assign_public(crypto_pk_t *dest, const crypto_pk_t *src);
 void crypto_pk_assign_private(crypto_pk_t *dest, const crypto_pk_t *src);
@@ -143,6 +143,6 @@ struct SECItemStr;
 STATIC int secitem_uint_cmp(const struct SECItemStr *a,
                             const struct SECItemStr *b);
 #endif
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
-#endif
+#endif /* !defined(TOR_CRYPTO_RSA_H) */
diff --git a/src/lib/crypt_ops/crypto_rsa_nss.c b/src/lib/crypt_ops/crypto_rsa_nss.c
index 7abf6716f0..66f325e868 100644
--- a/src/lib/crypt_ops/crypto_rsa_nss.c
+++ b/src/lib/crypt_ops/crypto_rsa_nss.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -156,7 +156,7 @@ crypto_pk_get_openssl_evp_pkey_,(crypto_pk_t *pk, int private))
   tor_free(buf);
   return result;
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Allocate and return storage for a public key.  The key itself will not yet
  * be set.
@@ -645,7 +645,7 @@ crypto_pk_asn1_decode(const char *str, size_t len)
   return result;
 }
 
-DISABLE_GCC_WARNING(unused-parameter)
+DISABLE_GCC_WARNING("-Wunused-parameter")
 
 /** Given a crypto_pk_t <b>pk</b>, allocate a new buffer containing the Base64
  * encoding of the DER representation of the private key into the
diff --git a/src/lib/crypt_ops/crypto_rsa_openssl.c b/src/lib/crypt_ops/crypto_rsa_openssl.c
index 17eae24cc2..c96ee81fd3 100644
--- a/src/lib/crypt_ops/crypto_rsa_openssl.c
+++ b/src/lib/crypt_ops/crypto_rsa_openssl.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/fs/files.h"
 
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/err.h>
 #include <openssl/rsa.h>
@@ -27,7 +27,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/bn.h>
 #include <openssl/conf.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "lib/log/log.h"
 #include "lib/encoding/binascii.h"
@@ -54,7 +54,7 @@ crypto_pk_key_is_private(const crypto_pk_t *k)
   const BIGNUM *p, *q;
   RSA_get0_factors(k->key, &p, &q);
   return p != NULL; /* XXX/yawning: Should we check q? */
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   return k && k->key && k->key->p;
 #endif /* defined(OPENSSL_1_1_API) */
 }
@@ -288,7 +288,7 @@ crypto_pk_num_bits(crypto_pk_t *env)
   tor_assert(n != NULL);
 
   return RSA_bits(env->key);
-#else /* !(defined(OPENSSL_1_1_API)) */
+#else /* !defined(OPENSSL_1_1_API) */
   tor_assert(env->key->n);
   return BN_num_bits(env->key->n);
 #endif /* defined(OPENSSL_1_1_API) */
@@ -583,15 +583,15 @@ rsa_private_key_too_long(RSA *rsa, int max_bits)
   dmp1 = RSA_get0_dmp1(rsa);
   dmq1 = RSA_get0_dmq1(rsa);
   iqmp = RSA_get0_iqmp(rsa);
-#else
+#else /* !(OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1)) */
   /* The accessors above did not exist in openssl 1.1.0. */
   p = q = dmp1 = dmq1 = iqmp = NULL;
   RSA_get0_key(rsa, &n, &e, &d);
-#endif
+#endif /* OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,1) */
 
   if (RSA_bits(rsa) > max_bits)
     return true;
-#else
+#else /* !defined(OPENSSL_1_1_API) */
   n = rsa->n;
   e = rsa->e;
   p = rsa->p;
@@ -600,7 +600,7 @@ rsa_private_key_too_long(RSA *rsa, int max_bits)
   dmp1 = rsa->dmp1;
   dmq1 = rsa->dmq1;
   iqmp = rsa->iqmp;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
   if (n && BN_num_bits(n) > max_bits)
     return true;
diff --git a/src/lib/crypt_ops/crypto_s2k.c b/src/lib/crypt_ops/crypto_s2k.c
index 42276597d4..3a9ed5ef58 100644
--- a/src/lib/crypt_ops/crypto_s2k.c
+++ b/src/lib/crypt_ops/crypto_s2k.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -285,7 +285,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv < 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
       SECItem passItem = { .type = siBuffer,
                            .data = (unsigned char *) secret,
                            .len = (int)secret_len };
@@ -325,7 +325,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (alg)
         SECOID_DestroyAlgorithmID(alg, PR_TRUE);
       return rv;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
     }
 
     case S2K_TYPE_SCRYPT: {
@@ -348,7 +348,7 @@ secret_to_key_compute_key(uint8_t *key_out, size_t key_out_len,
       if (rv != 0)
         return S2K_FAILED;
       return (int)key_out_len;
-#else /* !(defined(HAVE_SCRYPT)) */
+#else /* !defined(HAVE_SCRYPT) */
       return S2K_NO_SCRYPT_SUPPORT;
 #endif /* defined(HAVE_SCRYPT) */
     }
@@ -380,7 +380,7 @@ secret_to_key_derivekey(uint8_t *key_out, size_t key_out_len,
 #ifndef HAVE_SCRYPT
   if (type == S2K_TYPE_SCRYPT)
     return S2K_NO_SCRYPT_SUPPORT;
- #endif
+#endif
 
   if (! legacy_format) {
     ++spec;
diff --git a/src/lib/crypt_ops/crypto_s2k.h b/src/lib/crypt_ops/crypto_s2k.h
index a16a3d781e..181a17acb1 100644
--- a/src/lib/crypt_ops/crypto_s2k.h
+++ b/src/lib/crypt_ops/crypto_s2k.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/crypt_ops/crypto_sys.h b/src/lib/crypt_ops/crypto_sys.h
new file mode 100644
index 0000000000..2115d4fc99
--- /dev/null
+++ b/src/lib/crypt_ops/crypto_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file crypto_sys.h
+ * \brief Declare subsystem object for the crypto module.
+ **/
+
+#ifndef TOR_CRYPTO_SYS_H
+#define TOR_CRYPTO_SYS_H
+
+extern const struct subsys_fns_t sys_crypto;
+
+#endif /* !defined(TOR_CRYPTO_SYS_H) */
diff --git a/src/lib/crypt_ops/crypto_util.c b/src/lib/crypt_ops/crypto_util.c
index 67a1a9eb92..7ebb860d09 100644
--- a/src/lib/crypt_ops/crypto_util.c
+++ b/src/lib/crypt_ops/crypto_util.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,8 +10,6 @@
  * \brief Common cryptographic utilities.
  **/
 
-#define CRYPTO_UTIL_PRIVATE
-
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/cc/compat_compiler.h"
 
@@ -26,11 +24,11 @@
 #include <stdlib.h>
 
 #ifdef ENABLE_OPENSSL
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 #include <openssl/err.h>
 #include <openssl/crypto.h>
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -109,3 +107,17 @@ memwipe(void *mem, uint8_t byte, size_t sz)
    **/
   memset(mem, byte, sz);
 }
+
+/**
+ * Securely all memory in <b>str</b>, then free it.
+ *
+ * As tor_free(), tolerates null pointers.
+ **/
+void
+tor_str_wipe_and_free_(char *str)
+{
+  if (!str)
+    return;
+  memwipe(str, 0, strlen(str));
+  tor_free_(str);
+}
diff --git a/src/lib/crypt_ops/crypto_util.h b/src/lib/crypt_ops/crypto_util.h
index 613a1bd0dd..36ee230176 100644
--- a/src/lib/crypt_ops/crypto_util.h
+++ b/src/lib/crypt_ops/crypto_util.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,8 +14,18 @@
 #define TOR_CRYPTO_UTIL_H
 
 #include "lib/cc/torint.h"
+#include "lib/malloc/malloc.h"
 
 /** OpenSSL-based utility functions. */
 void memwipe(void *mem, uint8_t byte, size_t sz);
 
+void tor_str_wipe_and_free_(char *str);
+/**
+ * Securely all memory in <b>str</b>, then free it.
+ *
+ * As tor_free(), tolerates null pointers, and sets <b>str</b> to NULL.
+ **/
+#define tor_str_wipe_and_free(str)                      \
+  FREE_AND_NULL(char, tor_str_wipe_and_free_, (str))
+
 #endif /* !defined(TOR_CRYPTO_UTIL_H) */
diff --git a/src/lib/crypt_ops/digestset.c b/src/lib/crypt_ops/digestset.c
index 0dba64d595..5162d6d8bd 100644
--- a/src/lib/crypt_ops/digestset.c
+++ b/src/lib/crypt_ops/digestset.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/digestset.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /* Wrap our hash function to have the signature that the bloom filter
  * needs. */
diff --git a/src/lib/crypt_ops/digestset.h b/src/lib/crypt_ops/digestset.h
index 91d53a0542..505ac10395 100644
--- a/src/lib/crypt_ops/digestset.h
+++ b/src/lib/crypt_ops/digestset.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -26,4 +26,4 @@ void digestset_add(digestset_t *set, const char *addr);
 int digestset_probably_contains(const digestset_t *set,
                                 const char *addr);
 
-#endif
+#endif /* !defined(TOR_DIGESTSET_H) */
diff --git a/src/lib/crypt_ops/include.am b/src/lib/crypt_ops/include.am
index 1022096fdc..7644cab412 100644
--- a/src/lib/crypt_ops/include.am
+++ b/src/lib/crypt_ops/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-crypt-ops-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_cipher.c		\
 	src/lib/crypt_ops/crypto_curve25519.c		\
@@ -17,6 +18,8 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 	src/lib/crypt_ops/crypto_ope.c          	\
 	src/lib/crypt_ops/crypto_pwbox.c		\
 	src/lib/crypt_ops/crypto_rand.c			\
+	src/lib/crypt_ops/crypto_rand_fast.c		\
+	src/lib/crypt_ops/crypto_rand_numeric.c		\
 	src/lib/crypt_ops/crypto_rsa.c			\
 	src/lib/crypt_ops/crypto_s2k.c			\
 	src/lib/crypt_ops/crypto_util.c                 \
@@ -25,12 +28,14 @@ src_lib_libtor_crypt_ops_a_SOURCES =			\
 if USE_NSS
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_nss.c			\
+	src/lib/crypt_ops/crypto_digest_nss.c		\
 	src/lib/crypt_ops/crypto_dh_nss.c		\
 	src/lib/crypt_ops/crypto_nss_mgt.c		\
 	src/lib/crypt_ops/crypto_rsa_nss.c
 else
 src_lib_libtor_crypt_ops_a_SOURCES +=			\
 	src/lib/crypt_ops/aes_openssl.c			\
+	src/lib/crypt_ops/crypto_digest_openssl.c	\
 	src/lib/crypt_ops/crypto_rsa_openssl.c
 endif
 
@@ -48,6 +53,7 @@ src_lib_libtor_crypt_ops_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_crypt_ops_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/crypt_ops/aes.h				\
 	src/lib/crypt_ops/compat_openssl.h		\
@@ -62,9 +68,12 @@ noinst_HEADERS +=					\
 	src/lib/crypt_ops/crypto_nss_mgt.h		\
 	src/lib/crypt_ops/crypto_openssl_mgt.h		\
 	src/lib/crypt_ops/crypto_ope.h          	\
+	src/lib/crypt_ops/crypto_options.inc		\
+	src/lib/crypt_ops/crypto_options_st.h		\
 	src/lib/crypt_ops/crypto_pwbox.h		\
 	src/lib/crypt_ops/crypto_rand.h			\
 	src/lib/crypt_ops/crypto_rsa.h			\
 	src/lib/crypt_ops/crypto_s2k.h			\
+	src/lib/crypt_ops/crypto_sys.h			\
 	src/lib/crypt_ops/crypto_util.h                 \
 	src/lib/crypt_ops/digestset.h
diff --git a/src/lib/crypt_ops/lib_crypt_ops.md b/src/lib/crypt_ops/lib_crypt_ops.md
new file mode 100644
index 0000000000..4e675e4871
--- /dev/null
+++ b/src/lib/crypt_ops/lib_crypt_ops.md
@@ -0,0 +1,137 @@
+@dir /lib/crypt_ops
+@brief lib/crypt_ops: Cryptographic operations.
+
+This module contains wrappers around the cryptographic libraries that we
+support, and implementations for some higher-level cryptographic
+constructions that we use.
+
+It wraps our two major cryptographic backends (OpenSSL or NSS, as configured
+by the user), and also wraps other cryptographic code in src/ext.
+
+Generally speaking, Tor code shouldn't be calling OpenSSL or NSS
+(or any other crypto library) directly.  Instead, we should indirect through
+one of the functions in this directory, or through \refdir{lib/tls}.
+
+Cryptography functionality that's available is described below.
+
+### RNG facilities ###
+
+The most basic RNG capability in Tor is the crypto_rand() family of
+functions. These currently use OpenSSL's RAND_() backend, but may use
+something faster in the future.
+
+In addition to crypto_rand(), which fills in a buffer with random
+bytes, we also have functions to produce random integers in certain
+ranges; to produce random hostnames; to produce random doubles, etc.
+
+When you're creating a long-term cryptographic secret, you might want
+to use crypto_strongest_rand() instead of crypto_rand().  It takes the
+operating system's entropy source and combines it with output from
+crypto_rand().  This is a pure paranoia measure, but it might help us
+someday.
+
+You can use smartlist_choose() to pick a random element from a smartlist
+and smartlist_shuffle() to randomize the order of a smartlist.  Both are
+potentially a bit slow.
+
+### Cryptographic digests and related functions ###
+
+We treat digests as separate types based on the length of their
+outputs.  We support one 160-bit digest (SHA1), two 256-bit digests
+(SHA256 and SHA3-256), and two 512-bit digests (SHA512 and SHA3-512).
+
+You should not use SHA1 for anything new.
+
+The crypto_digest\*() family of functions manipulates digests.  You
+can either compute a digest of a chunk of memory all at once using
+crypto_digest(), crypto_digest256(), or crypto_digest512().  Or you
+can create a crypto_digest_t object with
+crypto_digest{,256,512}_new(), feed information to it in chunks using
+crypto_digest_add_bytes(), and then extract the final digest using
+crypto_digest_get_digest().  You can copy the state of one of these
+objects using crypto_digest_dup() or crypto_digest_assign().
+
+We support the HMAC hash-based message authentication code
+instantiated using SHA256. See crypto_hmac_sha256.  (You should not
+add any HMAC users with SHA1, and HMAC is not necessary with SHA3.)
+
+We also support the SHA3 cousins, SHAKE128 and SHAKE256.  Unlike
+digests, these are extendable output functions (or XOFs) where you can
+get any amount of output.  Use the crypto_xof_\*() functions to access
+these.
+
+We have several ways to derive keys from cryptographically strong secret
+inputs (like diffie-hellman outputs). The old
+crypto_expand_key_material_TAP() performs an ad-hoc KDF based on SHA1 -- you
+shouldn't use it for implementing anything but old versions of the Tor
+protocol.  You can use HKDF-SHA256 (as defined in RFC5869) for more modern
+protocols.  Also consider SHAKE256.
+
+If your input is potentially weak, like a password or passphrase, use a salt
+along with the secret_to_key() functions as defined in crypto_s2k.c.  Prefer
+scrypt over other hashing methods when possible.  If you're using a password
+to encrypt something, see the "boxed file storage" section below.
+
+Finally, in order to store objects in hash tables, Tor includes the
+randomized SipHash 2-4 function.  Call it via the siphash24g() function in
+src/ext/siphash.h whenever you're creating a hashtable whose keys may be
+manipulated by an attacker in order to DoS you with collisions.
+
+
+### Stream ciphers ###
+
+You can create instances of a stream cipher using crypto_cipher_new().
+These are stateful objects of type crypto_cipher_t.  Note that these
+objects only support AES-128 right now; a future version should add
+support for AES-128 and/or ChaCha20.
+
+You can encrypt/decrypt with crypto_cipher_encrypt or
+crypto_cipher_decrypt. The crypto_cipher_crypt_inplace function performs
+an encryption without a copy.
+
+Note that sensible people should not use raw stream ciphers; they should
+probably be using some kind of AEAD. Sorry.
+
+### Public key functionality ###
+
+We support four public key algorithms: DH1024, RSA, Curve25519, and
+Ed25519.
+
+We support DH1024 over two prime groups.  You access these via the
+crypto_dh_\*() family of functions.
+
+We support RSA in many bit sizes for signing and encryption.  You access
+it via the crypto_pk_*() family of functions.  Note that a crypto_pk_t
+may or may not include a private key.  See the crypto_pk_* functions in
+crypto.c for a full list of functions here.
+
+For Curve25519 functionality, see the functions and types in
+crypto_curve25519.c. Curve25519 is generally suitable for when you need
+a secure fast elliptic-curve diffie hellman implementation. When
+designing new protocols, prefer it over DH in Z_p.
+
+For Ed25519 functionality, see the functions and types in
+crypto_ed25519.c. Ed25519 is a generally suitable as a secure fast
+elliptic curve signature method. For new protocols, prefer it over RSA
+signatures.
+
+### Metaformats for storage ###
+
+When OpenSSL manages the storage of some object, we use whatever format
+OpenSSL provides -- typically, some kind of PEM-wrapped base 64 encoding
+that starts with "----- BEGIN CRYPTOGRAPHIC OBJECT ----".
+
+When we manage the storage of some cryptographic object, we prefix the
+object with 32-byte NUL-padded prefix in order to avoid accidental
+object confusion; see the crypto_read_tagged_contents_from_file() and
+crypto_write_tagged_contents_to_file() functions for manipulating
+these. The prefix is "== type: tag ==", where type describes the object
+and its encoding, and tag indicates which one it is.
+
+### Boxed-file storage ###
+
+When managing keys, you frequently want to have some way to write a
+secret object to disk, encrypted with a passphrase.  The crypto_pwbox
+and crypto_unpwbox functions do so in a way that's likely to be
+readable by future versions of Tor.
+
diff --git a/src/lib/ctime/di_ops.c b/src/lib/ctime/di_ops.c
index 89e0837ae9..d57d286990 100644
--- a/src/lib/ctime/di_ops.c
+++ b/src/lib/ctime/di_ops.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -72,10 +72,10 @@ tor_memcmp(const void *a, const void *b, size_t len)
      * actually implementation-defined in standard C.  So how do we
      * get away with assuming it?  Easy.  We check.) */
 #if ((-60 >> 8) != -1)
-#error "According to cpp, right-shift doesn't perform sign-extension."
+#error "cpp says right-shift doesn't perform sign-extension."
 #endif
 #ifndef RSHIFT_DOES_SIGN_EXTEND
-#error "According to configure, right-shift doesn't perform sign-extension."
+#error "configure says right-shift doesn't perform sign-extension."
 #endif
 
     /* If v1 == v2, equal_p is ~0, so this will leave retval
@@ -145,8 +145,11 @@ tor_memeq(const void *a, const void *b, size_t sz)
 
 /* Implement di_digest256_map_t as a linked list of entries. */
 struct di_digest256_map_t {
+  /** Pointer to the next entry in the list. */
   struct di_digest256_map_t *next;
+  /** Key for this entry. */
   uint8_t key[32];
+  /** Value for this entry. */
   void *val;
 };
 
@@ -276,3 +279,30 @@ select_array_member_cumulative_timei(const uint64_t *entries, int n_entries,
 
   return i_chosen;
 }
+
+/**
+ * If <b>s</b> is true, then copy <b>n</b> bytes from <b>src</b> to
+ * <b>dest</b>.  Otherwise leave <b>dest</b> alone.
+ *
+ * This function behaves the same as
+ *
+ *      if (s)
+ *         memcpy(dest, src, n);
+ *
+ * except that it tries to run in the same amount of time whether <b>s</b> is
+ * true or not.
+ **/
+void
+memcpy_if_true_timei(bool s, void *dest, const void *src, size_t n)
+{
+  // If s is true, mask will be ~0.  If s is false, mask will be 0.
+  const char mask = (char) -(signed char)s;
+
+  char *destp = dest;
+  const char *srcp = src;
+  for (size_t i = 0; i < n; ++i) {
+    *destp = (*destp & ~mask) | (*srcp & mask);
+    ++destp;
+    ++srcp;
+  }
+}
diff --git a/src/lib/ctime/di_ops.h b/src/lib/ctime/di_ops.h
index 264b56a8c1..9fe2884ecc 100644
--- a/src/lib/ctime/di_ops.h
+++ b/src/lib/ctime/di_ops.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,8 @@
 
 int tor_memcmp(const void *a, const void *b, size_t sz);
 int tor_memeq(const void *a, const void *b, size_t sz);
+/** Perform a constant-time comparison of the <b>sz</b> bytes at <b>a</b> and
+ * <b>b</b>, yielding true if they are different, and false otherwise. */
 #define tor_memneq(a,b,sz) (!tor_memeq((a),(b),(sz)))
 
 /** Alias for the platform's memcmp() function.  This function is
@@ -24,7 +26,19 @@ int tor_memeq(const void *a, const void *b, size_t sz);
  * implementation.
  */
 #define fast_memcmp(a,b,c) (memcmp((a),(b),(c)))
+/** Alias for the platform's memcmp() function, for use in testing equality.
+ *
+ * This function is <em>not</em> data-independent: we define this alias so
+ * that we can mark cases where we are deliberately using a data-dependent
+ * memcmp() implementation.
+ */
 #define fast_memeq(a,b,c)  (0==memcmp((a),(b),(c)))
+/** Alias for the platform's memcmp() function, for use in testing inequality.
+ *
+ * This function is <em>not</em> data-independent: we define this alias so
+ * that we can mark cases where we are deliberately using a data-dependent
+ * memcmp() implementation.
+ */
 #define fast_memneq(a,b,c) (0!=memcmp((a),(b),(c)))
 
 int safe_mem_is_zero(const void *mem, size_t sz);
@@ -35,9 +49,17 @@ int safe_mem_is_zero(const void *mem, size_t sz);
  *
  * Not efficient for large maps! */
 typedef struct di_digest256_map_t di_digest256_map_t;
+/**
+ * Type for a function used to free members of a di_digest256_map_t.
+ **/
 typedef void (*dimap_free_fn)(void *);
 
 void dimap_free_(di_digest256_map_t *map, dimap_free_fn free_fn);
+/**
+ * @copydoc dimap_free_
+ *
+ * Additionally, set the pointer <b>map</b> to NULL.
+ **/
 #define dimap_free(map, free_fn)                \
   do {                                          \
     dimap_free_((map), (free_fn));              \
@@ -51,5 +73,6 @@ int select_array_member_cumulative_timei(const uint64_t *entries,
                                          int n_entries,
                                          uint64_t total, uint64_t rand_val);
 
-#endif /* !defined(TOR_DI_OPS_H) */
+void memcpy_if_true_timei(bool s, void *dest, const void *src, size_t n);
 
+#endif /* !defined(TOR_DI_OPS_H) */
diff --git a/src/lib/ctime/include.am b/src/lib/ctime/include.am
index b46c43ba0c..83942ca4e0 100644
--- a/src/lib/ctime/include.am
+++ b/src/lib/ctime/include.am
@@ -11,6 +11,7 @@ else
 mulodi4_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_ctime_a_SOURCES =			\
 	$(mulodi4_source) 				\
 	src/ext/csiphash.c   				\
@@ -21,5 +22,6 @@ src_lib_libtor_ctime_testing_a_SOURCES = \
 src_lib_libtor_ctime_a_CFLAGS = @CFLAGS_CONSTTIME@
 src_lib_libtor_ctime_testing_a_CFLAGS = @CFLAGS_CONSTTIME@ $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/ctime/di_ops.h
diff --git a/src/lib/ctime/lib_ctime.md b/src/lib/ctime/lib_ctime.md
new file mode 100644
index 0000000000..913199f6a5
--- /dev/null
+++ b/src/lib/ctime/lib_ctime.md
@@ -0,0 +1,14 @@
+@dir /lib/ctime
+@brief lib/ctime: Constant-time code to avoid side-channels.
+
+This module contains constant-time implementations of various
+data comparison and table lookup functions.  We use these in preference to
+memcmp() and so forth, since memcmp() can leak information about its inputs
+based on how fast it returns.  In general, your code should call tor_memeq()
+and tor_memneq(), not memcmp().
+
+We also define some _non_-constant-time wrappers for memcmp() here: Since we
+consider calls to memcmp() to be in error, we require that code that actually
+doesn't need to be constant-time to use the fast_memeq() / fast_memneq() /
+fast_memcmp() aliases instead.
+
diff --git a/src/lib/defs/dh_sizes.h b/src/lib/defs/dh_sizes.h
index a2ffbc51c2..bc2707b36f 100644
--- a/src/lib/defs/dh_sizes.h
+++ b/src/lib/defs/dh_sizes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@
 /** Length of our legacy DH keys. */
 #define DH1024_KEY_LEN (1024/8)
 
-#endif
+#endif /* !defined(TOR_DH_SIZES_H) */
diff --git a/src/lib/defs/digest_sizes.h b/src/lib/defs/digest_sizes.h
index 525e5209d6..7eef1747db 100644
--- a/src/lib/defs/digest_sizes.h
+++ b/src/lib/defs/digest_sizes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_DIGEST_SIZES_H
@@ -24,4 +24,4 @@
 /** Length of the output of our 64-bit optimized message digests (SHA512). */
 #define DIGEST512_LEN 64
 
-#endif
+#endif /* !defined(TOR_DIGEST_SIZES_H) */
diff --git a/src/lib/defs/include.am b/src/lib/defs/include.am
index 48ee7f29fc..84ee403771 100644
--- a/src/lib/defs/include.am
+++ b/src/lib/defs/include.am
@@ -1,5 +1,8 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += 			\
 	src/lib/defs/dh_sizes.h 	\
 	src/lib/defs/digest_sizes.h	\
+	src/lib/defs/logging_types.h	\
+	src/lib/defs/time.h      	\
 	src/lib/defs/x25519_sizes.h
diff --git a/src/lib/defs/lib_defs.md b/src/lib/defs/lib_defs.md
new file mode 100644
index 0000000000..5762e4550b
--- /dev/null
+++ b/src/lib/defs/lib_defs.md
@@ -0,0 +1,2 @@
+@dir /lib/defs
+@brief lib/defs: Lowest-level constants, used in many places.
diff --git a/src/lib/defs/logging_types.h b/src/lib/defs/logging_types.h
new file mode 100644
index 0000000000..33aa46186b
--- /dev/null
+++ b/src/lib/defs/logging_types.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file logging_types.h
+ *
+ * \brief Global definition for types used by logging systems.
+ **/
+
+#ifndef TOR_LOGGING_TYPES_H
+#define TOR_LOGGING_TYPES_H
+
+/* We define this here so that it can be used both by backtrace.h and
+ * log.h.
+ */
+
+/** Mask of zero or more log domains, OR'd together. */
+typedef uint64_t log_domain_mask_t;
+
+#endif /* !defined(TOR_LOGGING_TYPES_H) */
diff --git a/src/lib/defs/time.h b/src/lib/defs/time.h
new file mode 100644
index 0000000000..5707330795
--- /dev/null
+++ b/src/lib/defs/time.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_TIME_DEFS_H
+#define TOR_TIME_DEFS_H
+
+/**
+ * \file time.h
+ *
+ * \brief Definitions for timing-related constants.
+ **/
+
+/** How many microseconds per second */
+#define TOR_USEC_PER_SEC (1000000)
+/** How many nanoseconds per microsecond */
+#define TOR_NSEC_PER_USEC (1000)
+/** How many nanoseconds per millisecond */
+#define TOR_NSEC_PER_MSEC (1000*1000)
+
+#endif /* !defined(TOR_TIME_DEFS_H) */
diff --git a/src/lib/defs/x25519_sizes.h b/src/lib/defs/x25519_sizes.h
index 8933a8866b..acb08c5e6a 100644
--- a/src/lib/defs/x25519_sizes.h
+++ b/src/lib/defs/x25519_sizes.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,14 +23,22 @@
 /** Length of the result of a curve25519 handshake. */
 #define CURVE25519_OUTPUT_LEN 32
 
+/** Length of an Ed25519 public key */
 #define ED25519_PUBKEY_LEN 32
+/** Length of an Ed25519 secret key */
 #define ED25519_SECKEY_LEN 64
+/** Length of the seed that is ordinarily expanded to an Ed25519 secret
+ * key. */
 #define ED25519_SECKEY_SEED_LEN 32
+/** Length of an Ed25519 signature. */
 #define ED25519_SIG_LEN 64
 
+/** Length of a Curve25519 key when encoded in base 64, with padding. */
 #define CURVE25519_BASE64_PADDED_LEN 44
 
+/** Length of a Ed25519 key when encoded in base 64, without padding. */
 #define ED25519_BASE64_LEN 43
+/** Length of a Ed25519 signature when encoded in base 64, without padding. */
 #define ED25519_SIG_BASE64_LEN 86
 
-#endif
+#endif /* !defined(TOR_X25519_SIZES_H) */
diff --git a/src/lib/dispatch/.may_include b/src/lib/dispatch/.may_include
new file mode 100644
index 0000000000..884f4c0dbc
--- /dev/null
+++ b/src/lib/dispatch/.may_include
@@ -0,0 +1,11 @@
+orconfig.h
+
+ext/tor_queue.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
\ No newline at end of file
diff --git a/src/lib/dispatch/dispatch.h b/src/lib/dispatch/dispatch.h
new file mode 100644
index 0000000000..9c7c4833c2
--- /dev/null
+++ b/src/lib/dispatch/dispatch.h
@@ -0,0 +1,114 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_H
+#define TOR_DISPATCH_H
+
+#include "lib/dispatch/msgtypes.h"
+
+/**
+ * \file dispatch.h
+ * \brief Low-level APIs for message-passing system.
+ *
+ * This module implements message dispatch based on a set of short integer
+ * identifiers.  For a higher-level interface, see pubsub.h.
+ *
+ * Each message is represented as a generic msg_t object, and is discriminated
+ * by its message_id_t.  Messages are delivered by a dispatch_t object, which
+ * delivers each message to its recipients by a configured "channel".
+ *
+ * A "channel" is a means of delivering messages.  Every message_id_t must
+ * be associated with exactly one channel, identified by channel_id_t.
+ * When a channel receives messages, a callback is invoked to either process
+ * the messages immediately, or to cause them to be processed later.
+ *
+ * Every message_id_t has zero or more associated receiver functions set up in
+ * the dispatch_t object.  Once the dispatch_t object is created, receivers
+ * can be enabled or disabled [TODO], but not added or removed.
+ *
+ * Every message_id_t has an associated datatype, identified by a
+ * msg_type_id_t.  These datatypes can be associated with functions to
+ * (for example) free them, or format them for debugging.
+ *
+ * To setup a dispatch_t object, first create a dispatch_cfg_t object, and
+ * configure messages with their types, channels, and receivers.  Then, use
+ * dispatch_new() with that dispatch_cfg_t to create the dispatch_t object.
+ *
+ * (We use a two-phase contruction procedure here to enable better static
+ * reasoning about publish/subscribe relationships.)
+ *
+ * Once you have a dispatch_t, you can queue messages on it with
+ * dispatch_send*(), and cause those messages to be delivered with
+ * dispatch_flush().
+ **/
+
+/**
+ * A "dispatcher" is the highest-level object; it handles making sure that
+ * messages are received and delivered properly.  Only the mainloop
+ * should handle this type directly.
+ */
+typedef struct dispatch_t dispatch_t;
+
+struct dispatch_cfg_t;
+
+dispatch_t *dispatch_new(const struct dispatch_cfg_t *cfg);
+
+/**
+ * Free a dispatcher.  Tor does this at exit.
+ */
+#define dispatch_free(d) \
+  FREE_AND_NULL(dispatch_t, dispatch_free_, (d))
+
+void dispatch_free_(dispatch_t *);
+
+int dispatch_send(dispatch_t *d,
+                  subsys_id_t sender,
+                  channel_id_t channel,
+                  message_id_t msg,
+                  msg_type_id_t type,
+                  msg_aux_data_t auxdata);
+
+int dispatch_send_msg(dispatch_t *d, msg_t *m);
+
+int dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m);
+
+/* Flush up to <b>max_msgs</b> currently pending messages from the
+ * dispatcher.  Messages that are not pending when this function are
+ * called, are not flushed by this call.  Return 0 on success, -1 on
+ * unrecoverable error.
+ */
+int dispatch_flush(dispatch_t *, channel_id_t chan, int max_msgs);
+
+/**
+ * Function callback type used to alert some other module when a channel's
+ * queue changes from empty to nonempty.
+ *
+ * Ex 1: To cause messages to be processed immediately on-stack, this callback
+ * should invoke dispatch_flush() directly.
+ *
+ * Ex 2: To cause messages to be processed very soon, from the event queue,
+ * this callback should schedule an event callback to run dispatch_flush().
+ *
+ * Ex 3: To cause messages to be processed periodically, this function should
+ * do nothing, and a periodic event should invoke dispatch_flush().
+ **/
+typedef void (*dispatch_alertfn_t)(struct dispatch_t *,
+                                   channel_id_t, void *);
+
+int dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                          dispatch_alertfn_t fn, void *userdata);
+
+#define dispatch_free_msg(d,msg)                                \
+  STMT_BEGIN {                                                  \
+    msg_t **msg_tmp_ptr__ = &(msg);                             \
+    dispatch_free_msg_((d), *msg_tmp_ptr__);                    \
+    *msg_tmp_ptr__= NULL;                                       \
+  } STMT_END
+void dispatch_free_msg_(const dispatch_t *d, msg_t *msg);
+
+char *dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg);
+
+#endif /* !defined(TOR_DISPATCH_H) */
diff --git a/src/lib/dispatch/dispatch_cfg.c b/src/lib/dispatch/dispatch_cfg.c
new file mode 100644
index 0000000000..a54188dcaa
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.c
@@ -0,0 +1,141 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_cfg.c
+ * \brief Create and configure a dispatch_cfg_t.
+ *
+ * A dispatch_cfg_t object is used to configure a set of messages and
+ * associated information before creating a dispatch_t.
+ */
+
+#define DISPATCH_PRIVATE
+
+#include "orconfig.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/malloc/malloc.h"
+
+/**
+ * Create and return a new dispatch_cfg_t.
+ **/
+dispatch_cfg_t *
+dcfg_new(void)
+{
+  dispatch_cfg_t *cfg = tor_malloc(sizeof(dispatch_cfg_t));
+  cfg->type_by_msg = smartlist_new();
+  cfg->chan_by_msg = smartlist_new();
+  cfg->fns_by_type = smartlist_new();
+  cfg->recv_by_msg = smartlist_new();
+  return cfg;
+}
+
+/**
+ * Associate a message with a datatype.  Return 0 on success, -1 if a
+ * different type was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                  msg_type_id_t type)
+{
+  smartlist_grow(cfg->type_by_msg, msg+1);
+  msg_type_id_t *oldval = smartlist_get(cfg->type_by_msg, msg);
+  if (oldval != NULL && *oldval != type) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->type_by_msg, msg, tor_memdup(&type, sizeof(type)));
+  return 0;
+}
+
+/**
+ * Associate a message with a channel.  Return 0 on success, -1 if a
+ * different channel was previously associated with the message ID.
+ **/
+int
+dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                  channel_id_t chan)
+{
+  smartlist_grow(cfg->chan_by_msg, msg+1);
+  channel_id_t *oldval = smartlist_get(cfg->chan_by_msg, msg);
+  if (oldval != NULL && *oldval != chan) {
+    return -1;
+  }
+  if (!oldval)
+    smartlist_set(cfg->chan_by_msg, msg, tor_memdup(&chan, sizeof(chan)));
+  return 0;
+}
+
+/**
+ * Associate a set of functions with a datatype. Return 0 on success, -1 if
+ * different functions were previously associated with the type.
+ **/
+int
+dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                  const dispatch_typefns_t *fns)
+{
+  smartlist_grow(cfg->fns_by_type, type+1);
+  dispatch_typefns_t *oldfns = smartlist_get(cfg->fns_by_type, type);
+  if (oldfns && (oldfns->free_fn != fns->free_fn ||
+                 oldfns->fmt_fn != fns->fmt_fn))
+    return -1;
+  if (!oldfns)
+    smartlist_set(cfg->fns_by_type, type, tor_memdup(fns, sizeof(*fns)));
+  return 0;
+}
+
+/**
+ * Associate a receiver with a message ID.  Multiple receivers may be
+ * associated with a single messasge ID.
+ *
+ * Return 0 on success, on failure.
+ **/
+int
+dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+              subsys_id_t sys, recv_fn_t fn)
+{
+  smartlist_grow(cfg->recv_by_msg, msg+1);
+  smartlist_t *receivers = smartlist_get(cfg->recv_by_msg, msg);
+  if (!receivers) {
+    receivers = smartlist_new();
+    smartlist_set(cfg->recv_by_msg, msg, receivers);
+  }
+
+  dispatch_rcv_t *rcv = tor_malloc(sizeof(dispatch_rcv_t));
+  rcv->sys = sys;
+  rcv->enabled = true;
+  rcv->fn = fn;
+  smartlist_add(receivers, (void*)rcv);
+  return 0;
+}
+
+/** Helper: release all storage held by <b>cfg</b>. */
+void
+dcfg_free_(dispatch_cfg_t *cfg)
+{
+  if (!cfg)
+    return;
+
+  SMARTLIST_FOREACH(cfg->type_by_msg, msg_type_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->chan_by_msg, channel_id_t *, id, tor_free(id));
+  SMARTLIST_FOREACH(cfg->fns_by_type, dispatch_typefns_t *, f, tor_free(f));
+  smartlist_free(cfg->type_by_msg);
+  smartlist_free(cfg->chan_by_msg);
+  smartlist_free(cfg->fns_by_type);
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, receivers) {
+    if (!receivers)
+      continue;
+    SMARTLIST_FOREACH(receivers, dispatch_rcv_t *, rcv, tor_free(rcv));
+    smartlist_free(receivers);
+  } SMARTLIST_FOREACH_END(receivers);
+  smartlist_free(cfg->recv_by_msg);
+
+  tor_free(cfg);
+}
diff --git a/src/lib/dispatch/dispatch_cfg.h b/src/lib/dispatch/dispatch_cfg.h
new file mode 100644
index 0000000000..a4f1948eac
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg.h
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_DISPATCH_CFG_H
+#define TOR_DISPATCH_CFG_H
+
+/**
+ * @file dispatch_cfg.h
+ * @brief Header for distpach_cfg.c
+ **/
+
+#include "lib/dispatch/msgtypes.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * A "dispatch_cfg" is the configuration used to set up a dispatcher.
+ * It is created and accessed with a set of dcfg_* functions, and then
+ * used with dispatcher_new() to make the dispatcher.
+ */
+typedef struct dispatch_cfg_t dispatch_cfg_t;
+
+dispatch_cfg_t *dcfg_new(void);
+
+int dcfg_msg_set_type(dispatch_cfg_t *cfg, message_id_t msg,
+                      msg_type_id_t type);
+
+int dcfg_msg_set_chan(dispatch_cfg_t *cfg, message_id_t msg,
+                      channel_id_t chan);
+
+int dcfg_type_set_fns(dispatch_cfg_t *cfg, msg_type_id_t type,
+                      const dispatch_typefns_t *fns);
+
+int dcfg_add_recv(dispatch_cfg_t *cfg, message_id_t msg,
+                  subsys_id_t sys, recv_fn_t fn);
+
+/** Free a dispatch_cfg_t. */
+#define dcfg_free(cfg) \
+  FREE_AND_NULL(dispatch_cfg_t, dcfg_free_, (cfg))
+
+void dcfg_free_(dispatch_cfg_t *cfg);
+
+#ifdef DISPATCH_NEW_PRIVATE
+struct smartlist_t;
+STATIC int max_in_u16_sl(const struct smartlist_t *sl, int dflt);
+#endif
+
+#endif /* !defined(TOR_DISPATCH_CFG_H) */
diff --git a/src/lib/dispatch/dispatch_cfg_st.h b/src/lib/dispatch/dispatch_cfg_st.h
new file mode 100644
index 0000000000..3c99adf2f7
--- /dev/null
+++ b/src/lib/dispatch/dispatch_cfg_st.h
@@ -0,0 +1,33 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dispatch_cfg_st.h
+ * @brief Declarations for dispatch-configuration types.
+ **/
+
+#ifndef TOR_DISPATCH_CFG_ST_H
+#define TOR_DISPATCH_CFG_ST_H
+
+struct smartlist_t;
+
+/** Information needed to create a dispatcher, but in a less efficient, more
+ * mutable format.
+ *
+ * Nearly everybody should use the \refdir{lib/pubsub} module to configure
+ * dispatchers, instead of using this. */
+struct dispatch_cfg_t {
+  /** A list of msg_type_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *type_by_msg;
+  /** A list of channel_id_t (cast to void*), indexed by msg_t. */
+  struct smartlist_t *chan_by_msg;
+  /** A list of dispatch_rcv_t, indexed by msg_type_id_t. */
+  struct smartlist_t *fns_by_type;
+  /** A list of dispatch_typefns_t, indexed by msg_t. */
+  struct smartlist_t *recv_by_msg;
+};
+
+#endif /* !defined(TOR_DISPATCH_CFG_ST_H) */
diff --git a/src/lib/dispatch/dispatch_core.c b/src/lib/dispatch/dispatch_core.c
new file mode 100644
index 0000000000..3d51c876a7
--- /dev/null
+++ b/src/lib/dispatch/dispatch_core.c
@@ -0,0 +1,260 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_core.c
+ * \brief Core module for sending and receiving messages.
+ */
+
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_naming.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Use <b>d</b> to drop all storage held for <b>msg</b>.
+ *
+ * (We need the dispatcher so we know how to free the auxiliary data.)
+ **/
+void
+dispatch_free_msg_(const dispatch_t *d, msg_t *msg)
+{
+  if (!msg)
+    return;
+
+  d->typefns[msg->type].free_fn(msg->aux_data__);
+  tor_free(msg);
+}
+
+/**
+ * Format the auxiliary data held by msg.
+ **/
+char *
+dispatch_fmt_msg_data(const dispatch_t *d, const msg_t *msg)
+{
+  if (!msg)
+    return NULL;
+
+  return d->typefns[msg->type].fmt_fn(msg->aux_data__);
+}
+
+/**
+ * Release all storage held by <b>d</b>.
+ **/
+void
+dispatch_free_(dispatch_t *d)
+{
+  if (d == NULL)
+    return;
+
+  size_t n_queues = d->n_queues;
+  for (size_t i = 0; i < n_queues; ++i) {
+    msg_t *m, *mtmp;
+    TOR_SIMPLEQ_FOREACH_SAFE(m, &d->queues[i].queue, next, mtmp) {
+      dispatch_free_msg(d, m);
+    }
+  }
+
+  size_t n_msgs = d->n_msgs;
+
+  for (size_t i = 0; i < n_msgs; ++i) {
+    tor_free(d->table[i]);
+  }
+  tor_free(d->table);
+  tor_free(d->typefns);
+  tor_free(d->queues);
+
+  // This is the only time we will treat d->cfg as non-const.
+  //dispatch_cfg_free_((dispatch_items_t *) d->cfg);
+
+  tor_free(d);
+}
+
+/**
+ * Tell the dispatcher to call <b>fn</b> with <b>userdata</b> whenever
+ * <b>chan</b> becomes nonempty.  Return 0 on success, -1 on error.
+ **/
+int
+dispatch_set_alert_fn(dispatch_t *d, channel_id_t chan,
+                      dispatch_alertfn_t fn, void *userdata)
+{
+  if (BUG(chan >= d->n_queues))
+    return -1;
+
+  dqueue_t *q = &d->queues[chan];
+  q->alert_fn = fn;
+  q->alert_fn_arg = userdata;
+  return 0;
+}
+
+/**
+ * Send a message on the appropriate channel notifying that channel if
+ * necessary.
+ *
+ * This function takes ownership of the auxiliary data; it can't be static or
+ * stack-allocated, and the caller is not allowed to use it afterwards.
+ *
+ * This function does not check the various vields of the message object for
+ * consistency.
+ **/
+int
+dispatch_send(dispatch_t *d,
+              subsys_id_t sender,
+              channel_id_t channel,
+              message_id_t msg,
+              msg_type_id_t type,
+              msg_aux_data_t auxdata)
+{
+  if (!d->table[msg]) {
+    /* Fast path: nobody wants this data. */
+
+    d->typefns[type].free_fn(auxdata);
+    return 0;
+  }
+
+  msg_t *m = tor_malloc(sizeof(msg_t));
+
+  m->sender = sender;
+  m->channel = channel;
+  m->msg = msg;
+  m->type = type;
+  memcpy(&m->aux_data__, &auxdata, sizeof(msg_aux_data_t));
+
+  return dispatch_send_msg(d, m);
+}
+
+int
+dispatch_send_msg(dispatch_t *d, msg_t *m)
+{
+  if (BUG(!d))
+    goto err;
+  if (BUG(!m))
+    goto err;
+  if (BUG(m->channel >= d->n_queues))
+    goto err;
+  if (BUG(m->msg >= d->n_msgs))
+    goto err;
+
+  dtbl_entry_t *ent = d->table[m->msg];
+  if (ent) {
+    if (BUG(m->type != ent->type))
+      goto err;
+    if (BUG(m->channel != ent->channel))
+      goto err;
+  }
+
+  return dispatch_send_msg_unchecked(d, m);
+ err:
+  /* Probably it isn't safe to free m, since type could be wrong. */
+  return -1;
+}
+
+/**
+ * Send a message on the appropriate queue, notifying that queue if necessary.
+ *
+ * This function takes ownership of the message object and its auxiliary data;
+ * it can't be static or stack-allocated, and the caller isn't allowed to use
+ * it afterwards.
+ *
+ * This function does not check the various fields of the message object for
+ * consistency, and can crash if they are out of range.  Only functions that
+ * have already constructed the message in a safe way, or checked it for
+ * correctness themselves, should call this function.
+ **/
+int
+dispatch_send_msg_unchecked(dispatch_t *d, msg_t *m)
+{
+  /* Find the right queue. */
+  dqueue_t *q = &d->queues[m->channel];
+  bool was_empty = TOR_SIMPLEQ_EMPTY(&q->queue);
+
+  /* Append the message. */
+  TOR_SIMPLEQ_INSERT_TAIL(&q->queue, m, next);
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Queued: %s (%s) from %s, on %s.",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  /* If we just made the queue nonempty for the first time, call the alert
+   * function. */
+  if (was_empty) {
+    q->alert_fn(d, m->channel, q->alert_fn_arg);
+  }
+
+  return 0;
+}
+
+/**
+ * Run all of the callbacks on <b>d</b> associated with <b>m</b>.
+ **/
+static void
+dispatcher_run_msg_cbs(const dispatch_t *d, msg_t *m)
+{
+  tor_assert(m->msg <= d->n_msgs);
+  dtbl_entry_t *ent = d->table[m->msg];
+  int n_fns = ent->n_fns;
+
+  if (debug_logging_enabled()) {
+    char *arg = dispatch_fmt_msg_data(d, m);
+    log_debug(LD_MESG,
+              "Delivering: %s (%s) from %s, on %s:",
+              get_message_id_name(m->msg),
+              arg,
+              get_subsys_id_name(m->sender),
+              get_channel_id_name(m->channel));
+    tor_free(arg);
+  }
+
+  int i;
+  for (i=0; i < n_fns; ++i) {
+    if (ent->rcv[i].enabled) {
+      log_debug(LD_MESG, "  Delivering to %s.",
+                get_subsys_id_name(ent->rcv[i].sys));
+      ent->rcv[i].fn(m);
+    }
+  }
+}
+
+/**
+ * Run up to <b>max_msgs</b> callbacks for messages on the channel <b>ch</b>
+ * on the given dispatcher.  Return 0 on success or recoverable failure,
+ * -1 on unrecoverable error.
+ **/
+int
+dispatch_flush(dispatch_t *d, channel_id_t ch, int max_msgs)
+{
+  if (BUG(ch >= d->n_queues))
+    return 0;
+
+  int n_flushed = 0;
+  dqueue_t *q = &d->queues[ch];
+
+  while (n_flushed < max_msgs) {
+    msg_t *m = TOR_SIMPLEQ_FIRST(&q->queue);
+    if (!m)
+      break;
+    TOR_SIMPLEQ_REMOVE_HEAD(&q->queue, next);
+    dispatcher_run_msg_cbs(d, m);
+    dispatch_free_msg(d, m);
+    ++n_flushed;
+  }
+
+  return 0;
+}
diff --git a/src/lib/dispatch/dispatch_naming.c b/src/lib/dispatch/dispatch_naming.c
new file mode 100644
index 0000000000..bb49343712
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dispatch_naming.c
+ * @brief Name-to-ID maps for our message dispatch system.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/cc/compat_compiler.h"
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+
+#include "lib/log/util_bug.h"
+#include "lib/log/log.h"
+
+#include <stdlib.h>
+
+/** Global namemap for message IDs. */
+static namemap_t message_id_map = NAMEMAP_INIT();
+/** Global namemap for subsystem IDs. */
+static namemap_t subsys_id_map = NAMEMAP_INIT();
+/** Global namemap for channel IDs. */
+static namemap_t channel_id_map = NAMEMAP_INIT();
+/** Global namemap for message type IDs. */
+static namemap_t msg_type_id_map = NAMEMAP_INIT();
+
+void
+dispatch_naming_init(void)
+{
+}
+
+#ifndef COCCI
+/* Helper macro: declare functions to map IDs to and from names for a given
+ * type in a namemap_t.
+ */
+#define DECLARE_ID_MAP_FNS(type)                                        \
+  type##_id_t                                                           \
+  get_##type##_id(const char *name)                                     \
+  {                                                                     \
+    unsigned u = namemap_get_or_create_id(&type##_id_map, name);        \
+    tor_assert(u != NAMEMAP_ERR);                                       \
+    tor_assert(u != ERROR_ID);                                          \
+    return (type##_id_t) u;                                             \
+  }                                                                     \
+  const char *                                                          \
+  get_##type##_id_name(type##_id_t id)                                  \
+  {                                                                     \
+    return namemap_fmt_name(&type##_id_map, id);                        \
+  }                                                                     \
+  size_t                                                                \
+  get_num_##type##_ids(void)                                            \
+  {                                                                     \
+    return namemap_get_size(&type##_id_map);                            \
+  }                                                                     \
+  EAT_SEMICOLON
+#endif /* !defined(COCCI) */
+
+DECLARE_ID_MAP_FNS(message);
+DECLARE_ID_MAP_FNS(channel);
+DECLARE_ID_MAP_FNS(subsys);
+DECLARE_ID_MAP_FNS(msg_type);
diff --git a/src/lib/dispatch/dispatch_naming.h b/src/lib/dispatch/dispatch_naming.h
new file mode 100644
index 0000000000..72206d3ed5
--- /dev/null
+++ b/src/lib/dispatch/dispatch_naming.h
@@ -0,0 +1,51 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file dispatch_naming.h
+ * @brief Header for dispatch_naming.c
+ **/
+
+#ifndef TOR_DISPATCH_NAMING_H
+#define TOR_DISPATCH_NAMING_H
+
+#include "lib/dispatch/msgtypes.h"
+#include <stddef.h>
+
+/**
+ * Return an existing channel ID by name, allocating the channel ID if
+ * if necessary.  Returns ERROR_ID if we have run out of
+ * channels
+ */
+channel_id_t get_channel_id(const char *);
+/**
+ * Return the name corresponding to a given channel ID.
+ **/
+const char *get_channel_id_name(channel_id_t);
+/**
+ * Return the total number of _named_ channel IDs.
+ **/
+size_t get_num_channel_ids(void);
+
+/* As above, but for messages. */
+message_id_t get_message_id(const char *);
+const char *get_message_id_name(message_id_t);
+size_t get_num_message_ids(void);
+
+/* As above, but for subsystems */
+subsys_id_t get_subsys_id(const char *);
+const char *get_subsys_id_name(subsys_id_t);
+size_t get_num_subsys_ids(void);
+
+/* As above, but for types. Note that types additionally must be
+ * "defined", if any message is to use them. */
+msg_type_id_t get_msg_type_id(const char *);
+const char *get_msg_type_id_name(msg_type_id_t);
+size_t get_num_msg_type_ids(void);
+
+void dispatch_naming_init(void);
+
+#endif /* !defined(TOR_DISPATCH_NAMING_H) */
diff --git a/src/lib/dispatch/dispatch_new.c b/src/lib/dispatch/dispatch_new.c
new file mode 100644
index 0000000000..e1dbb1c4b8
--- /dev/null
+++ b/src/lib/dispatch/dispatch_new.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_new.c
+ * \brief Code to construct a dispatch_t from a dispatch_cfg_t.
+ **/
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_cfg_st.h"
+
+#include "lib/cc/ctassert.h"
+#include "lib/intmath/cmp.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/** Given a smartlist full of (possibly NULL) pointers to uint16_t values,
+ * return the largest value, or dflt if the list is empty. */
+STATIC int
+max_in_u16_sl(const smartlist_t *sl, int dflt)
+{
+  uint16_t *maxptr = NULL;
+  SMARTLIST_FOREACH_BEGIN(sl, uint16_t *, u) {
+    if (!maxptr)
+      maxptr = u;
+    else if (u && *u > *maxptr)
+      maxptr = u;
+  } SMARTLIST_FOREACH_END(u);
+
+  return maxptr ? *maxptr : dflt;
+}
+
+/* The above function is only safe to call if we are sure that channel_id_t
+ * and msg_type_id_t are really uint16_t.  They should be so defined in
+ * msgtypes.h, but let's be extra cautious.
+ */
+CTASSERT(sizeof(uint16_t) == sizeof(msg_type_id_t));
+CTASSERT(sizeof(uint16_t) == sizeof(channel_id_t));
+
+/** Helper: Format an unformattable message auxiliary data item: just return a
+* copy of the string <>. */
+static char *
+type_fmt_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+  return tor_strdup("<>");
+}
+
+/** Helper: Free an unfreeable message auxiliary data item: do nothing. */
+static void
+type_free_nop(msg_aux_data_t arg)
+{
+  (void)arg;
+}
+
+/** Type functions to use when no type functions are provided. */
+static dispatch_typefns_t nop_typefns = {
+  .free_fn = type_free_nop,
+  .fmt_fn = type_fmt_nop
+};
+
+/**
+ * Alert function to use when none is configured: do nothing.
+ **/
+static void
+alert_fn_nop(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)d;
+  (void)ch;
+  (void)arg;
+}
+
+/**
+ * Given a list of recvfn_t, create and return a new dtbl_entry_t mapping
+ * to each of those functions.
+ **/
+static dtbl_entry_t *
+dtbl_entry_from_lst(smartlist_t *receivers)
+{
+  if (!receivers)
+    return NULL;
+
+  size_t n_recv = smartlist_len(receivers);
+  dtbl_entry_t *ent;
+  ent = tor_malloc_zero(offsetof(dtbl_entry_t, rcv) +
+                        sizeof(dispatch_rcv_t) * n_recv);
+
+  ent->n_fns = n_recv;
+
+  SMARTLIST_FOREACH_BEGIN(receivers, const dispatch_rcv_t *, rcv) {
+    memcpy(&ent->rcv[rcv_sl_idx], rcv, sizeof(*rcv));
+    if (rcv->enabled) {
+      ++ent->n_enabled;
+    }
+  } SMARTLIST_FOREACH_END(rcv);
+
+  return ent;
+}
+
+/** Create and return a new dispatcher from a given dispatch_cfg_t. */
+dispatch_t *
+dispatch_new(const dispatch_cfg_t *cfg)
+{
+  dispatch_t *d = tor_malloc_zero(sizeof(dispatch_t));
+
+  /* Any message that has a type or a receiver counts towards our messages */
+  const size_t n_msgs = MAX(smartlist_len(cfg->type_by_msg),
+                            smartlist_len(cfg->recv_by_msg)) + 1;
+
+  /* Any channel that any message has counts towards the number of channels. */
+  const size_t n_chans = (size_t)
+    MAX(1, max_in_u16_sl(cfg->chan_by_msg,0)) + 1;
+
+  /* Any type that a message has, or that has functions, counts towards
+   * the number of types. */
+  const size_t n_types = (size_t) MAX(max_in_u16_sl(cfg->type_by_msg,0),
+                                      smartlist_len(cfg->fns_by_type)) + 1;
+
+  d->n_msgs = n_msgs;
+  d->n_queues = n_chans;
+  d->n_types = n_types;
+
+  /* Initialize the array of type-functions. */
+  d->typefns = tor_calloc(n_types, sizeof(dispatch_typefns_t));
+  for (size_t i = 0; i < n_types; ++i) {
+    /* Default to no-op for everything... */
+    memcpy(&d->typefns[i], &nop_typefns, sizeof(dispatch_typefns_t));
+  }
+  SMARTLIST_FOREACH_BEGIN(cfg->fns_by_type, dispatch_typefns_t *, fns) {
+    /* Set the functions if they are provided. */
+    if (fns) {
+      if (fns->free_fn)
+        d->typefns[fns_sl_idx].free_fn = fns->free_fn;
+      if (fns->fmt_fn)
+        d->typefns[fns_sl_idx].fmt_fn = fns->fmt_fn;
+    }
+  } SMARTLIST_FOREACH_END(fns);
+
+  /* Initialize the message queues: one for each channel. */
+  d->queues = tor_calloc(d->n_queues, sizeof(dqueue_t));
+  for (size_t i = 0; i < d->n_queues; ++i) {
+    TOR_SIMPLEQ_INIT(&d->queues[i].queue);
+    d->queues[i].alert_fn = alert_fn_nop;
+  }
+
+  /* Build the dispatch tables mapping message IDs to receivers. */
+  d->table = tor_calloc(d->n_msgs, sizeof(dtbl_entry_t *));
+  SMARTLIST_FOREACH_BEGIN(cfg->recv_by_msg, smartlist_t *, rcv) {
+    d->table[rcv_sl_idx] = dtbl_entry_from_lst(rcv);
+  } SMARTLIST_FOREACH_END(rcv);
+
+  /* Fill in the empty entries in the dispatch tables:
+   * types and channels for each message. */
+  SMARTLIST_FOREACH_BEGIN(cfg->type_by_msg, msg_type_id_t *, type) {
+    if (d->table[type_sl_idx])
+      d->table[type_sl_idx]->type = *type;
+  } SMARTLIST_FOREACH_END(type);
+
+  SMARTLIST_FOREACH_BEGIN(cfg->chan_by_msg, channel_id_t *, chan) {
+    if (d->table[chan_sl_idx])
+      d->table[chan_sl_idx]->channel = *chan;
+  } SMARTLIST_FOREACH_END(chan);
+
+  return d;
+}
diff --git a/src/lib/dispatch/dispatch_st.h b/src/lib/dispatch/dispatch_st.h
new file mode 100644
index 0000000000..ad5b4efc40
--- /dev/null
+++ b/src/lib/dispatch/dispatch_st.h
@@ -0,0 +1,108 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file dispatch_st.h
+ *
+ * \brief private structures used for the dispatcher module
+ */
+
+#ifndef TOR_DISPATCH_ST_H
+#define TOR_DISPATCH_ST_H
+
+#ifdef DISPATCH_PRIVATE
+
+#include "lib/container/smartlist.h"
+
+/**
+ * Information about the recipient of a message.
+ **/
+typedef struct dispatch_rcv_t {
+  /** The subsystem receiving a message. */
+  subsys_id_t sys;
+  /** True iff this recipient is enabled. */
+  bool enabled;
+  /** The function that will handle the message. */
+  recv_fn_t fn;
+} dispatch_rcv_t;
+
+/**
+ * Information used by a dispatcher to handle and dispatch a single message
+ * ID.  It maps that message ID to its type, channel, and list of receiver
+ * functions.
+ *
+ * This structure is used when the dispatcher is running.
+ **/
+typedef struct dtbl_entry_t {
+  /** The number of enabled non-stub subscribers for this message.
+   *
+   * Note that for now, this will be the same as <b>n_fns</b>, since there is
+   * no way to turn these subscribers on an off yet. */
+  uint16_t n_enabled;
+  /** The channel that handles this message. */
+  channel_id_t channel;
+  /** The associated C type for this message. */
+  msg_type_id_t type;
+  /**
+   * The number of functions pointers for subscribers that receive this
+   * message, in rcv. */
+  uint16_t n_fns;
+  /**
+   * The recipients for this message.
+   */
+  dispatch_rcv_t rcv[FLEXIBLE_ARRAY_MEMBER];
+} dtbl_entry_t;
+
+/**
+ * A queue of messages for a given channel, used by a live dispatcher.
+ */
+typedef struct dqueue_t {
+  /** The queue of messages itself. */
+  TOR_SIMPLEQ_HEAD( , msg_t) queue;
+  /** A function to be called when the queue becomes nonempty. */
+  dispatch_alertfn_t alert_fn;
+  /** An argument for the alert_fn. */
+  void *alert_fn_arg;
+} dqueue_t ;
+
+/**
+ * A single dispatcher for cross-module messages.
+ */
+struct dispatch_t {
+  /**
+   * The length of <b>table</b>: the number of message IDs that this
+   * dispatcher can handle.
+   */
+  size_t n_msgs;
+  /**
+   * The length of <b>queues</b>: the number of channels that this dispatcher
+   * has configured.
+   */
+  size_t n_queues;
+  /**
+   * The length of <b>typefns</b>: the number of C type IDs that this
+   * dispatcher has configured.
+   */
+  size_t n_types;
+  /**
+   * An array of message queues, indexed by channel ID.
+   */
+  dqueue_t *queues;
+  /**
+   * An array of entries about how to handle particular message types, indexed
+   * by message ID.
+   */
+  dtbl_entry_t **table;
+  /**
+   * An array of function tables for manipulating types, index by message
+   * type ID.
+   **/
+  dispatch_typefns_t *typefns;
+};
+
+#endif /* defined(DISPATCH_PRIVATE) */
+
+#endif /* !defined(TOR_DISPATCH_ST_H) */
diff --git a/src/lib/dispatch/include.am b/src/lib/dispatch/include.am
new file mode 100644
index 0000000000..4a0e0dfd90
--- /dev/null
+++ b/src/lib/dispatch/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-dispatch.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-dispatch-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_dispatch_a_SOURCES =			\
+	src/lib/dispatch/dispatch_cfg.c			\
+	src/lib/dispatch/dispatch_core.c		\
+	src/lib/dispatch/dispatch_naming.c		\
+	src/lib/dispatch/dispatch_new.c
+
+src_lib_libtor_dispatch_testing_a_SOURCES = \
+	$(src_lib_libtor_dispatch_a_SOURCES)
+src_lib_libtor_dispatch_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_dispatch_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/dispatch/dispatch.h			\
+	src/lib/dispatch/dispatch_cfg.h 		\
+	src/lib/dispatch/dispatch_cfg_st.h 		\
+	src/lib/dispatch/dispatch_naming.h		\
+	src/lib/dispatch/dispatch_st.h			\
+	src/lib/dispatch/msgtypes.h
diff --git a/src/lib/dispatch/lib_dispatch.md b/src/lib/dispatch/lib_dispatch.md
new file mode 100644
index 0000000000..153ca50080
--- /dev/null
+++ b/src/lib/dispatch/lib_dispatch.md
@@ -0,0 +1,14 @@
+@dir /lib/dispatch
+@brief lib/dispatch: In-process message delivery.
+
+This module provides a general in-process "message dispatch" system in which
+typed messages are sent on channels. The dispatch.h header has far more
+information.
+
+It is used by by \refdir{lib/pubsub} to implement our general
+inter-module publish/subscribe system.
+
+This is not a fancy multi-threaded many-to-many dispatcher as you may be used
+to from more sophisticated architectures: this dispatcher is intended only
+for use in improving Tor's architecture.
+
diff --git a/src/lib/dispatch/msgtypes.h b/src/lib/dispatch/msgtypes.h
new file mode 100644
index 0000000000..01d969dcb5
--- /dev/null
+++ b/src/lib/dispatch/msgtypes.h
@@ -0,0 +1,80 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file msgtypes.h
+ * \brief Types used for messages in the dispatcher code.
+ **/
+
+#ifndef TOR_DISPATCH_MSGTYPES_H
+#define TOR_DISPATCH_MSGTYPES_H
+
+#include <stdint.h>
+
+#include "ext/tor_queue.h"
+
+/**
+ * These types are aliases for subsystems, channels, and message IDs.
+ **/
+typedef uint16_t subsys_id_t;
+typedef uint16_t channel_id_t;
+typedef uint16_t message_id_t;
+
+/**
+ * This identifies a C type that can be sent along with a message.
+ **/
+typedef uint16_t msg_type_id_t;
+
+/**
+ * An ID value returned for *_type_t when none exists.
+ */
+#define ERROR_ID 65535
+
+/**
+ * Auxiliary (untyped) data sent along with a message.
+ *
+ * We define this as a union of a pointer and a u64, so that the integer
+ * types will have the same range across platforms.
+ **/
+typedef union {
+  void *ptr;
+  uint64_t u64;
+} msg_aux_data_t;
+
+/**
+ * Structure of a received message.
+ **/
+typedef struct msg_t {
+  TOR_SIMPLEQ_ENTRY(msg_t) next;
+  subsys_id_t sender;
+  channel_id_t channel;
+  message_id_t msg;
+  /** We could omit this field, since it is implicit in the message type, but
+   * IMO let's leave it in for safety. */
+  msg_type_id_t type;
+  /** Untyped auxiliary data. You shouldn't have to mess with this
+   * directly. */
+  msg_aux_data_t aux_data__;
+} msg_t;
+
+/**
+ * A function that a subscriber uses to receive a message.
+ **/
+typedef void (*recv_fn_t)(const msg_t *m);
+
+/**
+ * Table of functions to use for a given C type.  Any omitted (NULL) functions
+ * will be treated as no-ops.
+ **/
+typedef struct dispatch_typefns_t {
+  /** Release storage held for the auxiliary data of this type. */
+  void (*free_fn)(msg_aux_data_t);
+  /** Format and return a newly allocated string describing the contents
+   * of this data element. */
+  char *(*fmt_fn)(msg_aux_data_t);
+} dispatch_typefns_t;
+
+#endif /* !defined(TOR_DISPATCH_MSGTYPES_H) */
diff --git a/src/lib/encoding/.may_include b/src/lib/encoding/.may_include
index 7c2ef36929..c9bf4b1786 100644
--- a/src/lib/encoding/.may_include
+++ b/src/lib/encoding/.may_include
@@ -1,5 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/container/*.h
 lib/ctime/*.h
 lib/encoding/*.h
 lib/intmath/*.h
diff --git a/src/lib/encoding/binascii.c b/src/lib/encoding/binascii.c
index bd063440d6..3e549eb8e3 100644
--- a/src/lib/encoding/binascii.c
+++ b/src/lib/encoding/binascii.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -84,7 +84,7 @@ base32_encode(char *dest, size_t destlen, const char *src, size_t srclen)
 }
 
 /** Implements base32 decoding as in RFC 4648.
- * Returns 0 if successful, -1 otherwise.
+ * Return the number of bytes decoded if successful; -1 otherwise.
  */
 int
 base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
@@ -147,7 +147,7 @@ base32_decode(char *dest, size_t destlen, const char *src, size_t srclen)
   memset(tmp, 0, srclen); /* on the heap, this should be safe */
   tor_free(tmp);
   tmp = NULL;
-  return 0;
+  return i;
 }
 
 #define BASE64_OPENSSL_LINELEN 64
@@ -179,6 +179,18 @@ base64_encode_size(size_t srclen, int flags)
   return enclen;
 }
 
+/** Return an upper bound on the number of bytes that might be needed to hold
+ * the data from decoding the base64 string <b>srclen</b>.  This is only an
+ * upper bound, since some part of the base64 string might be padding or
+ * space. */
+size_t
+base64_decode_maxsize(size_t srclen)
+{
+  tor_assert(srclen < INT_MAX / 3);
+
+  return CEIL_DIV(srclen * 3, 4);
+}
+
 /** Internal table mapping 6 bit values to the Base64 alphabet. */
 static const char base64_encode_table[64] = {
   'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
@@ -309,8 +321,10 @@ base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
   return (int) enclen;
 }
 
-/** As base64_encode, but do not add any internal spaces or external padding
- * to the output stream. */
+/** As base64_encode, but do not add any internal spaces, and remove external
+ * padding from the output stream.
+ * dest must be at least base64_encode_size(srclen, 0), including space for
+ * the removed external padding. */
 int
 base64_encode_nopad(char *dest, size_t destlen,
                     const uint8_t *src, size_t srclen)
diff --git a/src/lib/encoding/binascii.h b/src/lib/encoding/binascii.h
index 7e3cc04f09..9cb03bab62 100644
--- a/src/lib/encoding/binascii.h
+++ b/src/lib/encoding/binascii.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,6 +42,7 @@ const char *hex_str(const char *from, size_t fromlen);
 
 #define BASE64_ENCODE_MULTILINE 1
 size_t base64_encode_size(size_t srclen, int flags);
+size_t base64_decode_maxsize(size_t srclen);
 int base64_encode(char *dest, size_t destlen, const char *src, size_t srclen,
                   int flags);
 int base64_decode(char *dest, size_t destlen, const char *src, size_t srclen);
@@ -57,4 +58,4 @@ size_t base32_encoded_size(size_t srclen);
 void base16_encode(char *dest, size_t destlen, const char *src, size_t srclen);
 int base16_decode(char *dest, size_t destlen, const char *src, size_t srclen);
 
-#endif /* !defined(TOR_UTIL_FORMAT_H) */
+#endif /* !defined(TOR_BINASCII_H) */
diff --git a/src/lib/encoding/confline.c b/src/lib/encoding/confline.c
index 8110f3dd9c..613e4a00c6 100644
--- a/src/lib/encoding/confline.c
+++ b/src/lib/encoding/confline.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -82,6 +82,19 @@ config_line_find(const config_line_t *lines,
   return NULL;
 }
 
+/** As config_line_find(), but perform a case-insensitive comparison. */
+const config_line_t *
+config_line_find_case(const config_line_t *lines,
+                      const char *key)
+{
+  const config_line_t *cl;
+  for (cl = lines; cl; cl = cl->next) {
+    if (!strcasecmp(cl->key, key))
+      return cl;
+  }
+  return NULL;
+}
+
 /** Auxiliary function that does all the work of config_get_lines.
  * <b>recursion_level</b> is the count of how many nested %includes we have.
  * <b>opened_lst</b> will have a list of opened files if provided.
@@ -138,6 +151,8 @@ config_get_lines_aux(const char *string, config_line_t **result, int extended,
       if (allow_include && !strcmp(k, "%include") && handle_include) {
         tor_free(k);
         include_used = 1;
+        log_notice(LD_CONFIG, "Processing configuration path \"%s\" at "
+                   "recursion level %d.", v, recursion_level);
 
         config_line_t *include_list;
         if (handle_include(v, recursion_level, extended, &include_list,
@@ -148,9 +163,6 @@ config_get_lines_aux(const char *string, config_line_t **result, int extended,
           tor_free(v);
           return -1;
         }
-        log_notice(LD_CONFIG, "Included configuration file or "
-                   "directory at recursion level %d: \"%s\".",
-                   recursion_level, v);
         *next = include_list;
         if (list_last)
           next = &list_last->next;
@@ -240,10 +252,39 @@ config_lines_dup_and_filter(const config_line_t *inp,
   return result;
 }
 
+/**
+ * Given a linelist <b>inp</b> beginning with the key <b>header</b>, find the
+ * next line with that key, and remove that instance and all following lines
+ * from the list.  Return the lines that were removed.  Operate
+ * case-insensitively.
+ *
+ * For example, if the header is "H", and <b>inp</b> contains "H, A, B, H, C,
+ * H, D", this function will alter <b>inp</b> to contain only "H, A, B", and
+ * return the elements "H, C, H, D" as a separate list.
+ **/
+config_line_t *
+config_lines_partition(config_line_t *inp, const char *header)
+{
+  if (BUG(inp == NULL))
+    return NULL;
+  if (BUG(strcasecmp(inp->key, header)))
+    return NULL;
+
+  /* Advance ptr until it points to the link to the next segment of this
+     list. */
+  config_line_t **ptr = &inp->next;
+  while (*ptr && strcasecmp((*ptr)->key, header)) {
+    ptr = &(*ptr)->next;
+  }
+  config_line_t *remainder = *ptr;
+  *ptr = NULL;
+  return remainder;
+}
+
 /** Return true iff a and b contain identical keys and values in identical
  * order. */
 int
-config_lines_eq(config_line_t *a, config_line_t *b)
+config_lines_eq(const config_line_t *a, const config_line_t *b)
 {
   while (a && b) {
     if (strcasecmp(a->key, b->key) || strcmp(a->value, b->value))
diff --git a/src/lib/encoding/confline.h b/src/lib/encoding/confline.h
index 3d9ae8a662..ce0d6c6e17 100644
--- a/src/lib/encoding/confline.h
+++ b/src/lib/encoding/confline.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,7 +48,10 @@ config_line_t *config_lines_dup_and_filter(const config_line_t *inp,
                                            const char *key);
 const config_line_t *config_line_find(const config_line_t *lines,
                                       const char *key);
-int config_lines_eq(config_line_t *a, config_line_t *b);
+const config_line_t *config_line_find_case(const config_line_t *lines,
+                                           const char *key);
+config_line_t *config_lines_partition(config_line_t *inp, const char *header);
+int config_lines_eq(const config_line_t *a, const config_line_t *b);
 int config_count_key(const config_line_t *a, const char *key);
 void config_free_lines_(config_line_t *front);
 #define config_free_lines(front) \
diff --git a/src/lib/encoding/cstring.c b/src/lib/encoding/cstring.c
index 29d3714126..54c330fca3 100644
--- a/src/lib/encoding/cstring.c
+++ b/src/lib/encoding/cstring.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/cstring.h b/src/lib/encoding/cstring.h
index 904a2c9c1c..2a3f6d0fc4 100644
--- a/src/lib/encoding/cstring.h
+++ b/src/lib/encoding/cstring.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/include.am b/src/lib/encoding/include.am
index 2d2aa3988a..48d0120bfc 100644
--- a/src/lib/encoding/include.am
+++ b/src/lib/encoding/include.am
@@ -4,12 +4,15 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-encoding-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_encoding_a_SOURCES =			\
 	src/lib/encoding/binascii.c			\
 	src/lib/encoding/confline.c			\
 	src/lib/encoding/cstring.c			\
 	src/lib/encoding/keyval.c			\
+	src/lib/encoding/kvline.c			\
 	src/lib/encoding/pem.c				\
+	src/lib/encoding/qstring.c			\
 	src/lib/encoding/time_fmt.c
 
 src_lib_libtor_encoding_testing_a_SOURCES = \
@@ -17,10 +20,13 @@ src_lib_libtor_encoding_testing_a_SOURCES = \
 src_lib_libtor_encoding_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_encoding_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/encoding/binascii.h			\
 	src/lib/encoding/confline.h			\
 	src/lib/encoding/cstring.h			\
 	src/lib/encoding/keyval.h			\
+	src/lib/encoding/kvline.h			\
 	src/lib/encoding/pem.h				\
+	src/lib/encoding/qstring.h			\
 	src/lib/encoding/time_fmt.h
diff --git a/src/lib/encoding/keyval.c b/src/lib/encoding/keyval.c
index c5da5a0bfc..0eb1219d43 100644
--- a/src/lib/encoding/keyval.c
+++ b/src/lib/encoding/keyval.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/keyval.h b/src/lib/encoding/keyval.h
index cd327b7a82..b4966b01de 100644
--- a/src/lib/encoding/keyval.h
+++ b/src/lib/encoding/keyval.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,4 +14,4 @@
 
 int string_is_key_value(int severity, const char *string);
 
-#endif
+#endif /* !defined(TOR_KEYVAL_H) */
diff --git a/src/lib/encoding/kvline.c b/src/lib/encoding/kvline.c
new file mode 100644
index 0000000000..5b220605d6
--- /dev/null
+++ b/src/lib/encoding/kvline.c
@@ -0,0 +1,297 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.c
+ *
+ * \brief Manipulating lines of key-value pairs.
+ **/
+
+#include "orconfig.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_ctype.h"
+#include "lib/string/printf.h"
+#include "lib/string/util_string.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <string.h>
+
+/** Return true iff we need to quote and escape the string <b>s</b> to encode
+ * it.
+ *
+ * kvline_can_encode_lines() also uses this (with
+ * <b>as_keyless_val</b> true) to check whether a key would require
+ * quoting.
+ */
+static bool
+needs_escape(const char *s, bool as_keyless_val)
+{
+  if (as_keyless_val && *s == 0)
+    return true;
+  /* Keyless values containing '=' need to be escaped. */
+  if (as_keyless_val && strchr(s, '='))
+    return true;
+
+  for (; *s; ++s) {
+    if (*s >= 127 || TOR_ISSPACE(*s) || ! TOR_ISPRINT(*s) ||
+        *s == '\'' || *s == '\"') {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Return true iff the key in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_key(const config_line_t *line)
+{
+  return line->key == NULL || strlen(line->key) == 0;
+}
+
+/**
+ * Return true iff the value in <b>line</b> is not set.
+ **/
+static bool
+line_has_no_val(const config_line_t *line)
+{
+  return line->value == NULL || strlen(line->value) == 0;
+}
+
+/**
+ * Return true iff the all the lines in <b>line</b> can be encoded
+ * using <b>flags</b>.
+ **/
+static bool
+kvline_can_encode_lines(const config_line_t *line, unsigned flags)
+{
+  for ( ; line; line = line->next) {
+    const bool keyless = line_has_no_key(line);
+    if (keyless && ! (flags & KV_OMIT_KEYS)) {
+      /* If KV_OMIT_KEYS is not set, we can't encode a line with no key. */
+      return false;
+    }
+
+    if (needs_escape(line->value, keyless) && ! (flags & (KV_QUOTED|KV_RAW))) {
+      /* If both KV_QUOTED and KV_RAW are false, we can't encode a
+         value that needs quotes. */
+      return false;
+    }
+    if (!keyless && needs_escape(line->key, true)) {
+      /* We can't handle keys that need quoting. */
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Encode a linked list of lines in <b>line</b> as a series of 'Key=Value'
+ * pairs, using the provided <b>flags</b> to encode it.  Return a newly
+ * allocated string on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then all values that contain
+ * spaces or unusual characters are escaped and quoted.  Otherwise, such
+ * values are not allowed.  Mutually exclusive with KV_RAW.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then pairs with empty keys are
+ * allowed, and are encoded as 'Value'.  Otherwise, such pairs are not
+ * allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then an empty value is
+ * encoded as 'Key', not as 'Key=' or 'Key=""'.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * If KV_RAW is set in <b>flags</b>, then don't apply any quoting to
+ * the value, and assume that the caller has adequately quoted it.
+ * (The control protocol has some quirks that make this necessary.)
+ * Mutually exclusive with KV_QUOTED.
+ *
+ * KV_QUOTED_QSTRING is not supported.
+ */
+char *
+kvline_encode(const config_line_t *line,
+              unsigned flags)
+{
+  tor_assert(! (flags & KV_QUOTED_QSTRING));
+
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+  tor_assert((flags & (KV_QUOTED|KV_RAW)) != (KV_QUOTED|KV_RAW));
+
+  if (!kvline_can_encode_lines(line, flags))
+    return NULL;
+
+  smartlist_t *elements = smartlist_new();
+
+  for (; line; line = line->next) {
+
+    const char *k = "";
+    const char *eq = "=";
+    const char *v = "";
+    const bool keyless = line_has_no_key(line);
+    bool esc = needs_escape(line->value, keyless);
+    char *tmp = NULL;
+
+    if (! keyless) {
+      k = line->key;
+    } else {
+      eq = "";
+    }
+
+    if ((flags & KV_OMIT_VALS) && line_has_no_val(line)) {
+      eq = "";
+      v = "";
+    } else if (!(flags & KV_RAW) && esc) {
+      tmp = esc_for_log(line->value);
+      v = tmp;
+    } else {
+      v = line->value;
+    }
+
+    smartlist_add_asprintf(elements, "%s%s%s", k, eq, v);
+    tor_free(tmp);
+  }
+
+  char *result = smartlist_join_strings(elements, " ", 0, NULL);
+
+  SMARTLIST_FOREACH(elements, char *, cp, tor_free(cp));
+  smartlist_free(elements);
+
+  return result;
+}
+
+/**
+ * Decode a <b>line</b> containing a series of space-separated 'Key=Value'
+ * pairs, using the provided <b>flags</b> to decode it.  Return a newly
+ * allocated list of pairs on success, or NULL on failure.
+ *
+ * If KV_QUOTED is set in <b>flags</b>, then (double-)quoted values are
+ * allowed and handled as C strings. Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_KEYS is set in <b>flags</b>, then values without keys are
+ * allowed.  Otherwise, such values are not allowed.
+ *
+ * If KV_OMIT_VALS is set in <b>flags</b>, then keys without values are
+ * allowed.  Otherwise, such keys are not allowed.  Mutually exclusive with
+ * KV_OMIT_KEYS.
+ *
+ * If KV_QUOTED_QSTRING is set in <b>flags</b>, then double-quoted values
+ * are allowed and handled as QuotedStrings per qstring.c.  Do not add
+ * new users of this flag.
+ *
+ * KV_RAW is not supported.
+ */
+config_line_t *
+kvline_parse(const char *line, unsigned flags)
+{
+  tor_assert((flags & (KV_OMIT_KEYS|KV_OMIT_VALS)) !=
+             (KV_OMIT_KEYS|KV_OMIT_VALS));
+  tor_assert(!(flags & KV_RAW));
+
+  const char *cp = line, *cplast = NULL;
+  const bool omit_keys = (flags & KV_OMIT_KEYS) != 0;
+  const bool omit_vals = (flags & KV_OMIT_VALS) != 0;
+  const bool quoted = (flags & (KV_QUOTED|KV_QUOTED_QSTRING)) != 0;
+  const bool c_quoted = (flags & (KV_QUOTED)) != 0;
+
+  config_line_t *result = NULL;
+  config_line_t **next_line = &result;
+
+  char *key = NULL;
+  char *val = NULL;
+
+  while (*cp) {
+    key = val = NULL;
+    /* skip all spaces */
+    {
+      size_t idx = strspn(cp, " \t\r\v\n");
+      cp += idx;
+    }
+    if (BUG(cp == cplast)) {
+      /* If we didn't parse anything since the last loop, this code is
+       * broken. */
+      goto err; // LCOV_EXCL_LINE
+    }
+    cplast = cp;
+    if (! *cp)
+      break; /* End of string; we're done. */
+
+    /* Possible formats are K=V, K="V", K, V, and "V", depending on flags. */
+
+    /* Find where the key ends */
+    if (*cp != '\"') {
+      size_t idx = strcspn(cp, " \t\r\v\n=");
+
+      if (cp[idx] == '=') {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx + 1;
+      } else if (omit_vals) {
+        key = tor_memdup_nulterm(cp, idx);
+        cp += idx;
+        goto commit;
+      } else {
+        if (!omit_keys)
+          goto err;
+      }
+    }
+
+    if (*cp == '\"') {
+      /* The type is "V". */
+      if (!quoted)
+        goto err;
+      size_t len=0;
+      if (c_quoted) {
+        cp = unescape_string(cp, &val, &len);
+      } else {
+        cp = decode_qstring(cp, strlen(cp), &val, &len);
+      }
+      if (cp == NULL || len != strlen(val)) {
+        // The string contains a NUL or is badly coded.
+        goto err;
+      }
+    } else {
+      size_t idx = strcspn(cp, " \t\r\v\n");
+      val = tor_memdup_nulterm(cp, idx);
+      cp += idx;
+    }
+
+  commit:
+    if (key && strlen(key) == 0) {
+      /* We don't allow empty keys. */
+      goto err;
+    }
+
+    *next_line = tor_malloc_zero(sizeof(config_line_t));
+    (*next_line)->key = key ? key : tor_strdup("");
+    (*next_line)->value = val ? val : tor_strdup("");
+    next_line = &(*next_line)->next;
+    key = val = NULL;
+  }
+
+  if (! (flags & KV_QUOTED_QSTRING)) {
+    if (!kvline_can_encode_lines(result, flags)) {
+      goto err;
+    }
+  }
+  return result;
+
+ err:
+  tor_free(key);
+  tor_free(val);
+  config_free_lines(result);
+  return NULL;
+}
diff --git a/src/lib/encoding/kvline.h b/src/lib/encoding/kvline.h
new file mode 100644
index 0000000000..34c52908e3
--- /dev/null
+++ b/src/lib/encoding/kvline.h
@@ -0,0 +1,27 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file kvline.h
+ *
+ * \brief Header for kvline.c
+ **/
+
+#ifndef TOR_KVLINE_H
+#define TOR_KVLINE_H
+
+struct config_line_t;
+
+#define KV_QUOTED    (1u<<0)
+#define KV_OMIT_KEYS (1u<<1)
+#define KV_OMIT_VALS (1u<<2)
+#define KV_QUOTED_QSTRING (1u<<3)
+#define KV_RAW       (1u<<4)
+
+struct config_line_t *kvline_parse(const char *line, unsigned flags);
+char *kvline_encode(const struct config_line_t *line, unsigned flags);
+
+#endif /* !defined(TOR_KVLINE_H) */
diff --git a/src/lib/encoding/lib_encoding.md b/src/lib/encoding/lib_encoding.md
new file mode 100644
index 0000000000..66dd9d8caf
--- /dev/null
+++ b/src/lib/encoding/lib_encoding.md
@@ -0,0 +1,6 @@
+@dir /lib/encoding
+@brief lib/encoding: Encoding data in various forms, types, and transformations
+
+Here we have time formats (timefmt.c), quoted strings (qstring.c), C strings
+(string.c) base-16/32/64 (binascii.c), and more.
+
diff --git a/src/lib/encoding/pem.c b/src/lib/encoding/pem.c
index 51f37d0840..6c9f10e085 100644
--- a/src/lib/encoding/pem.c
+++ b/src/lib/encoding/pem.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,7 +42,7 @@ pem_encoded_size(size_t src_len, const char *objtype)
 
 /**
  * PEM-encode the <b>srclen</b>-byte object at <b>src</b> into the
- * <b>destlen<\b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>.
+ * <b>destlen</b>-byte buffer at <b>dest</b>, tagging it with <b>objtype</b>.
  * Return 0 on success and -1 on failure.
  */
 int
diff --git a/src/lib/encoding/pem.h b/src/lib/encoding/pem.h
index 0bbb06a794..027c31c315 100644
--- a/src/lib/encoding/pem.h
+++ b/src/lib/encoding/pem.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,4 +23,4 @@ int pem_encode(char *dest, size_t destlen, const uint8_t *src, size_t srclen,
 int pem_decode(uint8_t *dest, size_t destlen, const char *src, size_t srclen,
                const char *objtype);
 
-#endif
+#endif /* !defined(TOR_PEM_H) */
diff --git a/src/lib/encoding/qstring.c b/src/lib/encoding/qstring.c
new file mode 100644
index 0000000000..5a34924eab
--- /dev/null
+++ b/src/lib/encoding/qstring.c
@@ -0,0 +1,90 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.c
+ * \brief Implement QuotedString parsing.
+ *
+ * Note that this is only used for controller authentication; do not
+ * create new users for this.  Instead, prefer the cstring.c functions.
+ **/
+
+#include "orconfig.h"
+#include "lib/encoding/qstring.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+/** If the first <b>in_len_max</b> characters in <b>start</b> contain a
+ * QuotedString, return the length of that
+ * string (as encoded, including quotes).  Otherwise return -1. */
+static inline int
+get_qstring_length(const char *start, size_t in_len_max,
+                          int *chars_out)
+{
+  const char *cp, *end;
+  int chars = 0;
+
+  if (*start != '\"')
+    return -1;
+
+  cp = start+1;
+  end = start+in_len_max;
+
+  /* Calculate length. */
+  while (1) {
+    if (cp >= end) {
+      return -1; /* Too long. */
+    } else if (*cp == '\\') {
+      if (++cp == end)
+        return -1; /* Can't escape EOS. */
+      ++cp;
+      ++chars;
+    } else if (*cp == '\"') {
+      break;
+    } else {
+      ++cp;
+      ++chars;
+    }
+  }
+  if (chars_out)
+    *chars_out = chars;
+  return (int)(cp - start+1);
+}
+
+/** Given a pointer to a string starting at <b>start</b> containing
+ * <b>in_len_max</b> characters, decode a string beginning with one double
+ * quote, containing any number of non-quote characters or characters escaped
+ * with a backslash, and ending with a final double quote.  Place the resulting
+ * string (unquoted, unescaped) into a newly allocated string in *<b>out</b>;
+ * store its length in <b>out_len</b>.  On success, return a pointer to the
+ * character immediately following the escaped string.  On failure, return
+ * NULL. */
+const char *
+decode_qstring(const char *start, size_t in_len_max,
+               char **out, size_t *out_len)
+{
+  const char *cp, *end;
+  char *outp;
+  int len, n_chars = 0;
+
+  len = get_qstring_length(start, in_len_max, &n_chars);
+  if (len<0)
+    return NULL;
+
+  end = start+len-1; /* Index of last quote. */
+  tor_assert(*end == '\"');
+  outp = *out = tor_malloc(len+1);
+  *out_len = n_chars;
+
+  cp = start+1;
+  while (cp < end) {
+    if (*cp == '\\')
+      ++cp;
+    *outp++ = *cp++;
+  }
+  *outp = '\0';
+  tor_assert((outp - *out) == (int)*out_len);
+
+  return end+1;
+}
diff --git a/src/lib/encoding/qstring.h b/src/lib/encoding/qstring.h
new file mode 100644
index 0000000000..f19a7dad87
--- /dev/null
+++ b/src/lib/encoding/qstring.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file qstring.h
+ * \brief Header for qstring.c
+ */
+
+#ifndef TOR_ENCODING_QSTRING_H
+#define TOR_ENCODING_QSTRING_H
+
+#include <stddef.h>
+
+const char *decode_qstring(const char *start, size_t in_len_max,
+                           char **out, size_t *out_len);
+
+#endif /* !defined(TOR_ENCODING_QSTRING_H) */
diff --git a/src/lib/encoding/time_fmt.c b/src/lib/encoding/time_fmt.c
index 40543d41e0..573dfaad82 100644
--- a/src/lib/encoding/time_fmt.c
+++ b/src/lib/encoding/time_fmt.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/encoding/time_fmt.h b/src/lib/encoding/time_fmt.h
index 0ddeca57fc..80e47c5332 100644
--- a/src/lib/encoding/time_fmt.h
+++ b/src/lib/encoding/time_fmt.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -41,4 +41,4 @@ int parse_iso_time_nospace(const char *cp, time_t *t);
 int parse_http_time(const char *buf, struct tm *tm);
 int format_time_interval(char *out, size_t out_len, long interval);
 
-#endif
+#endif /* !defined(TOR_TIME_FMT_H) */
diff --git a/src/lib/err/.may_include b/src/lib/err/.may_include
index 48cc0ef088..314424545e 100644
--- a/src/lib/err/.may_include
+++ b/src/lib/err/.may_include
@@ -1,3 +1,6 @@
 orconfig.h
 lib/cc/*.h
+lib/defs/*.h
 lib/err/*.h
+lib/subsys/*.h
+lib/version/*.h
diff --git a/src/lib/err/backtrace.c b/src/lib/err/backtrace.c
index 8606f42177..afb6b9503f 100644
--- a/src/lib/err/backtrace.c
+++ b/src/lib/err/backtrace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -52,12 +52,14 @@
 #include <pthread.h>
 #endif
 
-#define EXPOSE_CLEAN_BACKTRACE
+#include "lib/cc/ctassert.h"
+
+#define BACKTRACE_PRIVATE
 #include "lib/err/backtrace.h"
-#include "lib/err/torerr.h"
 
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
-  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
+  defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION) && \
+  defined(HAVE_PTHREAD_H)
 #define USE_BACKTRACE
 #endif
 
@@ -72,15 +74,40 @@
 static char bt_version[128] = "";
 
 #ifdef USE_BACKTRACE
+
 /** Largest stack depth to try to dump. */
 #define MAX_DEPTH 256
-/** Static allocation of stack to dump. This is static so we avoid stack
- * pressure. */
-static void *cb_buf[MAX_DEPTH];
+/** The size of the callback buffer, so we can clear it in unlock_cb_buf(). */
+#define SIZEOF_CB_BUF (MAX_DEPTH * sizeof(void *))
 /** Protects cb_buf from concurrent access. Pthreads, since this code
  * is Unix-only, and since this code needs to be lowest-level. */
 static pthread_mutex_t cb_buf_mutex = PTHREAD_MUTEX_INITIALIZER;
 
+/** Lock and return a static stack pointer buffer that can hold up to
+ *  MAX_DEPTH function pointers. */
+static void **
+lock_cb_buf(void)
+{
+  /* Lock the mutex first, before even declaring the buffer. */
+  pthread_mutex_lock(&cb_buf_mutex);
+
+  /** Static allocation of stack to dump. This is static so we avoid stack
+   * pressure. */
+  static void *cb_buf[MAX_DEPTH];
+  CTASSERT(SIZEOF_CB_BUF == sizeof(cb_buf));
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+
+  return cb_buf;
+}
+
+/** Unlock the static stack pointer buffer. */
+static void
+unlock_cb_buf(void **cb_buf)
+{
+  memset(cb_buf, 0, SIZEOF_CB_BUF);
+  pthread_mutex_unlock(&cb_buf_mutex);
+}
+
 /** Change a stacktrace in <b>stack</b> of depth <b>depth</b> so that it will
  * log the correct function from which a signal was received with context
  * <b>ctx</b>.  (When we get a signal, the current function will not have
@@ -104,7 +131,7 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
     return;
 
   stack[n] = (void*) ctx->PC_FROM_UCONTEXT;
-#else /* !(defined(PC_FROM_UCONTEXT)) */
+#else /* !defined(PC_FROM_UCONTEXT) */
   (void) depth;
   (void) ctx;
   (void) stack;
@@ -115,14 +142,14 @@ clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx)
  * that with a backtrace log.  Send messages via the tor_log function at
  * logger". */
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   size_t depth;
   char **symbols;
   size_t i;
 
-  pthread_mutex_lock(&cb_buf_mutex);
+  void **cb_buf = lock_cb_buf();
 
   depth = backtrace(cb_buf, MAX_DEPTH);
   symbols = backtrace_symbols(cb_buf, (int)depth);
@@ -140,7 +167,7 @@ log_backtrace_impl(int severity, int domain, const char *msg,
   raw_free(symbols);
 
  done:
-  pthread_mutex_unlock(&cb_buf_mutex);
+  unlock_cb_buf(cb_buf);
 }
 
 static void crash_handler(int sig, siginfo_t *si, void *ctx_)
@@ -156,6 +183,8 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   int n_fds, i;
   const int *fds = NULL;
 
+  void **cb_buf = lock_cb_buf();
+
   (void) si;
 
   depth = backtrace(cb_buf, MAX_DEPTH);
@@ -172,7 +201,9 @@ crash_handler(int sig, siginfo_t *si, void *ctx_)
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
 
-  abort();
+  unlock_cb_buf(cb_buf);
+
+  tor_raw_abort_();
 }
 
 /** Write a backtrace to all of the emergency-error fds. */
@@ -183,20 +214,26 @@ dump_stack_symbols_to_error_fds(void)
   const int *fds = NULL;
   size_t depth;
 
+  void **cb_buf = lock_cb_buf();
+
   depth = backtrace(cb_buf, MAX_DEPTH);
 
   n_fds = tor_log_get_sigsafe_err_fds(&fds);
   for (i=0; i < n_fds; ++i)
     backtrace_symbols_fd(cb_buf, (int)depth, fds[i]);
+
+  unlock_cb_buf(cb_buf);
 }
 
+/* The signals that we want our backtrace handler to trap */
+static int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
+  SIGIO, -1 };
+
 /** Install signal handlers as needed so that when we crash, we produce a
  * useful stack trace. Return 0 on success, -errno on failure. */
 static int
 install_bt_handler(void)
 {
-  int trap_signals[] = { SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS,
-                         SIGIO, -1 };
   int i, rv=0;
 
   struct sigaction sa;
@@ -219,10 +256,12 @@ install_bt_handler(void)
      * libc has pre-loaded the symbols we need to dump things, so that later
      * reads won't be denied by the sandbox code */
     char **symbols;
+    void **cb_buf = lock_cb_buf();
     size_t depth = backtrace(cb_buf, MAX_DEPTH);
     symbols = backtrace_symbols(cb_buf, (int) depth);
     if (symbols)
       raw_free(symbols);
+    unlock_cb_buf(cb_buf);
   }
 
   return rv;
@@ -232,12 +271,29 @@ install_bt_handler(void)
 static void
 remove_bt_handler(void)
 {
+  int i;
+
+  struct sigaction sa;
+
+  memset(&sa, 0, sizeof(sa));
+  sa.sa_handler = SIG_DFL;
+  sigfillset(&sa.sa_mask);
+
+  for (i = 0; trap_signals[i] >= 0; ++i) {
+    /* remove_bt_handler() is called on shutdown, from low-level code.
+     * It's not a fatal error, so we just ignore it. */
+    (void)sigaction(trap_signals[i], &sa, NULL);
+  }
+
+  /* cb_buf_mutex is statically initialised, so we can not destroy it.
+   * If we destroy it, and then re-initialise tor, all our backtraces will
+   * fail. */
 }
 #endif /* defined(USE_BACKTRACE) */
 
 #ifdef NO_BACKTRACE_IMPL
 void
-log_backtrace_impl(int severity, int domain, const char *msg,
+log_backtrace_impl(int severity, log_domain_mask_t domain, const char *msg,
                    tor_log_fn logger)
 {
   logger(severity, domain, "%s: %s. (Stack trace not available)",
diff --git a/src/lib/err/backtrace.h b/src/lib/err/backtrace.h
index 48b41fca02..d02e6960b5 100644
--- a/src/lib/err/backtrace.h
+++ b/src/lib/err/backtrace.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_BACKTRACE_H
@@ -12,11 +12,14 @@
 
 #include "orconfig.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/defs/logging_types.h"
 
-typedef void (*tor_log_fn)(int, unsigned, const char *fmt, ...)
+typedef void (*tor_log_fn)(int, log_domain_mask_t, const char *fmt, ...)
   CHECK_PRINTF(3,4);
 
-void log_backtrace_impl(int severity, int domain, const char *msg,
+void log_backtrace_impl(int severity, log_domain_mask_t domain,
+                        const char *msg,
                         tor_log_fn logger);
 int configure_backtrace_handler(const char *tor_version);
 void clean_up_backtrace_handler(void);
@@ -26,11 +29,11 @@ const char *get_tor_backtrace_version(void);
 #define log_backtrace(sev, dom, msg) \
   log_backtrace_impl((sev), (dom), (msg), tor_log)
 
-#ifdef EXPOSE_CLEAN_BACKTRACE
+#ifdef BACKTRACE_PRIVATE
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 void clean_backtrace(void **stack, size_t depth, const ucontext_t *ctx);
 #endif
-#endif /* defined(EXPOSE_CLEAN_BACKTRACE) */
+#endif /* defined(BACKTRACE_PRIVATE) */
 
 #endif /* !defined(TOR_BACKTRACE_H) */
diff --git a/src/lib/err/include.am b/src/lib/err/include.am
index f2a409c51e..883ac91511 100644
--- a/src/lib/err/include.am
+++ b/src/lib/err/include.am
@@ -5,15 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-err-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_err_a_SOURCES =			\
-	src/lib/err/backtrace.c				\
-	src/lib/err/torerr.c
+	src/lib/err/backtrace.c			\
+	src/lib/err/torerr.c			\
+	src/lib/err/torerr_sys.c
 
 src_lib_libtor_err_testing_a_SOURCES = \
 	$(src_lib_libtor_err_a_SOURCES)
 src_lib_libtor_err_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_err_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/err/backtrace.h				\
-	src/lib/err/torerr.h
+	src/lib/err/torerr.h				\
+	src/lib/err/torerr_sys.h
diff --git a/src/lib/err/lib_err.md b/src/lib/err/lib_err.md
new file mode 100644
index 0000000000..cb4eba2e0d
--- /dev/null
+++ b/src/lib/err/lib_err.md
@@ -0,0 +1,13 @@
+@dir /lib/err
+@brief lib/err: Lowest-level error handling code.
+
+This module is responsible for generating stack traces, handling raw
+assertion failures, and otherwise reporting problems that might not be
+safe to report via the regular logging module.
+
+There are three kinds of users for the functions in this module:
+  * Code that needs a way to assert(), but which cannot use the regular
+    `tor_assert()` macros in logging module.
+  * Code that needs signal-safe error reporting.
+  * Higher-level error handling code.
+
diff --git a/src/lib/err/torerr.c b/src/lib/err/torerr.c
index 6b5224273a..2de75c0be4 100644
--- a/src/lib/err/torerr.c
+++ b/src/lib/err/torerr.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -110,6 +110,14 @@ tor_log_get_sigsafe_err_fds(const int **out)
  * Update the list of fds that get errors from inside a signal handler or
  * other emergency condition. Ignore any beyond the first
  * TOR_SIGSAFE_LOG_MAX_FDS.
+ *
+ * These fds must remain open even after the log module has shut down. (And
+ * they should remain open even while logs are being reconfigured.) Therefore,
+ * any fds closed by the log module should be dup()ed, and the duplicate fd
+ * should be given to the err module in fds. In particular, the log module
+ * closes the file log fds, but does not close the stdio log fds.
+ *
+ * If fds is NULL or n is 0, clears the list of error fds.
  */
 void
 tor_log_set_sigsafe_err_fds(const int *fds, int n)
@@ -118,8 +126,52 @@ tor_log_set_sigsafe_err_fds(const int *fds, int n)
     n = TOR_SIGSAFE_LOG_MAX_FDS;
   }
 
-  memcpy(sigsafe_log_fds, fds, n * sizeof(int));
-  n_sigsafe_log_fds = n;
+  /* Clear the entire array. This code mitigates against some race conditions,
+   * but there are still some races here:
+   * - err logs are disabled while the array is cleared, and
+   * - a thread can read the old value of n_sigsafe_log_fds, then read a
+   *   partially written array.
+   * We could fix these races using atomics, but atomics use the err module. */
+  n_sigsafe_log_fds = 0;
+  memset(sigsafe_log_fds, 0, sizeof(sigsafe_log_fds));
+  if (fds && n > 0) {
+    memcpy(sigsafe_log_fds, fds, n * sizeof(int));
+    n_sigsafe_log_fds = n;
+  }
+}
+
+/**
+ * Reset the list of emergency error fds to its default.
+ */
+void
+tor_log_reset_sigsafe_err_fds(void)
+{
+  int fds[] = { STDERR_FILENO };
+  tor_log_set_sigsafe_err_fds(fds, 1);
+}
+
+/**
+ * Flush the list of fds that get errors from inside a signal handler or
+ * other emergency condition. These fds are shared with the logging code:
+ * flushing them also flushes the log buffers.
+ *
+ * This function is safe to call during signal handlers.
+ */
+void
+tor_log_flush_sigsafe_err_fds(void)
+{
+  /* If we don't have fsync() in unistd.h, we can't flush the logs. */
+#ifdef HAVE_FSYNC
+  int n_fds, i;
+  const int *fds = NULL;
+
+  n_fds = tor_log_get_sigsafe_err_fds(&fds);
+  for (i = 0; i < n_fds; ++i) {
+    /* This function is called on error and on shutdown, so we don't log, or
+     * take any other action, if fsync() fails. */
+    (void)fsync(fds[i]);
+  }
+#endif /* defined(HAVE_FSYNC) */
 }
 
 /**
@@ -161,6 +213,18 @@ tor_raw_assertion_failed_msg_(const char *file, int line, const char *expr,
   tor_log_err_sigsafe_write("\n");
 }
 
+/**
+ * Call the abort() function to kill the current process with a fatal
+ * error. But first, flush the raw error file descriptors, so error messages
+ * are written before process termination.
+ **/
+void
+tor_raw_abort_(void)
+{
+  tor_log_flush_sigsafe_err_fds();
+  abort();
+}
+
 /* As format_{hex,dex}_number_sigsafe, but takes a <b>radix</b> argument
  * in range 2..16 inclusive. */
 static int
@@ -195,7 +259,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
     unsigned digit = (unsigned) (x % radix);
     if (cp <= buf) {
       /* Not tor_assert(); see above. */
-      abort();
+      tor_raw_abort_();
     }
     --cp;
     *cp = "0123456789ABCDEF"[digit];
@@ -204,7 +268,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
 
   /* NOT tor_assert; see above. */
   if (cp != buf) {
-    abort(); // LCOV_EXCL_LINE
+    tor_raw_abort_(); // LCOV_EXCL_LINE
   }
 
   return len;
@@ -224,8 +288,7 @@ format_number_sigsafe(unsigned long x, char *buf, int buf_len,
  * does not guarantee that an int is wider than a char (an int must be at
  * least 16 bits but it is permitted for a char to be that wide as well), we
  * can't assume a signed int is sufficient to accommodate an unsigned char.
- * Thus, format_helper_exit_status() will still need to emit any require '-'
- * on its own.
+ * Thus, callers will still need to add any required '-' to the final string.
  *
  * For most purposes, you'd want to use tor_snprintf("%x") instead of this
  * function; it's designed to be used in code paths where you can't call
diff --git a/src/lib/err/torerr.h b/src/lib/err/torerr.h
index 6ae91fbe85..ce1b049c47 100644
--- a/src/lib/err/torerr.h
+++ b/src/lib/err/torerr.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,13 +20,13 @@
 #define raw_assert(expr) STMT_BEGIN                                     \
     if (!(expr)) {                                                      \
       tor_raw_assertion_failed_msg_(__FILE__, __LINE__, #expr, NULL);   \
-      abort();                                                          \
+      tor_raw_abort_();                                                 \
     }                                                                   \
   STMT_END
 #define raw_assert_unreached(expr) raw_assert(0)
 #define raw_assert_unreached_msg(msg) STMT_BEGIN                    \
     tor_raw_assertion_failed_msg_(__FILE__, __LINE__, "0", (msg));  \
-    abort();                                                        \
+    tor_raw_abort_();                                               \
   STMT_END
 
 void tor_raw_assertion_failed_msg_(const char *file, int line,
@@ -39,9 +39,13 @@ void tor_raw_assertion_failed_msg_(const char *file, int line,
 void tor_log_err_sigsafe(const char *m, ...);
 int tor_log_get_sigsafe_err_fds(const int **out);
 void tor_log_set_sigsafe_err_fds(const int *fds, int n);
+void tor_log_reset_sigsafe_err_fds(void);
+void tor_log_flush_sigsafe_err_fds(void);
 void tor_log_sigsafe_err_set_granularity(int ms);
 
+void tor_raw_abort_(void) ATTR_NORETURN;
+
 int format_hex_number_sigsafe(unsigned long x, char *buf, int max_len);
 int format_dec_number_sigsafe(unsigned long x, char *buf, int max_len);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_TORERR_H) */
diff --git a/src/lib/err/torerr_sys.c b/src/lib/err/torerr_sys.c
new file mode 100644
index 0000000000..8ee1521f3b
--- /dev/null
+++ b/src/lib/err/torerr_sys.c
@@ -0,0 +1,45 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.c
+ * \brief Subsystem object for the error handling subsystem.
+ **/
+
+#include "orconfig.h"
+#include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
+#include "lib/err/torerr_sys.h"
+#include "lib/subsys/subsys.h"
+#include "lib/version/torversion.h"
+
+#include <stddef.h>
+
+static int
+subsys_torerr_initialize(void)
+{
+  if (configure_backtrace_handler(get_version()) < 0)
+    return -1;
+  tor_log_reset_sigsafe_err_fds();
+
+  return 0;
+}
+static void
+subsys_torerr_shutdown(void)
+{
+  /* Stop handling signals with backtraces, then flush the logs. */
+  clean_up_backtrace_handler();
+  tor_log_flush_sigsafe_err_fds();
+}
+
+const subsys_fns_t sys_torerr = {
+  .name = "err",
+  SUBSYS_DECLARE_LOCATION(),
+  /* Low-level error handling is a diagnostic feature, we want it to init
+   * right after windows process security, and shutdown last.
+   * (Security never shuts down.) */
+  .level = -99,
+  .supported = true,
+  .initialize = subsys_torerr_initialize,
+  .shutdown = subsys_torerr_shutdown
+};
diff --git a/src/lib/err/torerr_sys.h b/src/lib/err/torerr_sys.h
new file mode 100644
index 0000000000..b86ccd2790
--- /dev/null
+++ b/src/lib/err/torerr_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file torerr_sys.h
+ * \brief Declare subsystem object for torerr.c
+ **/
+
+#ifndef TOR_TORERR_SYS_H
+#define TOR_TORERR_SYS_H
+
+extern const struct subsys_fns_t sys_torerr;
+
+#endif /* !defined(TOR_TORERR_SYS_H) */
diff --git a/src/lib/evloop/.may_include b/src/lib/evloop/.may_include
index 30af508914..54aa75fbff 100644
--- a/src/lib/evloop/.may_include
+++ b/src/lib/evloop/.may_include
@@ -8,9 +8,10 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/time/*.h
 
-src/ext/timeouts/timeout.c
-tor_queue.h
\ No newline at end of file
+ext/timeouts/timeout.c
+ext/tor_queue.h
\ No newline at end of file
diff --git a/src/lib/evloop/compat_libevent.c b/src/lib/evloop/compat_libevent.c
index 939d77f857..0fd247d331 100644
--- a/src/lib/evloop/compat_libevent.c
+++ b/src/lib/evloop/compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* Copyright (c) 2009-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -130,7 +130,7 @@ rescan_mainloop_cb(evutil_socket_t fd, short events, void *arg)
 
 /** Initialize the Libevent library and set up the event base. */
 void
-tor_libevent_initialize(tor_libevent_cfg *torcfg)
+tor_libevent_initialize(tor_libevent_cfg_t *torcfg)
 {
   tor_assert(the_event_base == NULL);
   /* some paths below don't use torcfg, so avoid unused variable warnings */
@@ -432,7 +432,7 @@ mainloop_event_activate(mainloop_event_t *event)
  *
  * If the event is scheduled for a different time, cancel it and run
  * after this delay instead.  If the event is currently pending to run
- * <em>now</b>, has no effect.
+ * <b>now</b>, has no effect.
  *
  * Do not call this function with <b>tv</b> == NULL -- use
  * mainloop_event_activate() instead.
diff --git a/src/lib/evloop/compat_libevent.h b/src/lib/evloop/compat_libevent.h
index 92724c369c..277ba3add6 100644
--- a/src/lib/evloop/compat_libevent.h
+++ b/src/lib/evloop/compat_libevent.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2009-2019, The Tor Project, Inc. */
+/* Copyright (c) 2009-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -61,15 +61,15 @@ void mainloop_event_free_(mainloop_event_t *event);
 
 /** Defines a configuration for using libevent with Tor: passed as an argument
  * to tor_libevent_initialize() to describe how we want to set up. */
-typedef struct tor_libevent_cfg {
+typedef struct tor_libevent_cfg_t {
   /** How many CPUs should we use (not currently useful). */
   int num_cpus;
   /** How many milliseconds should we allow between updating bandwidth limits?
    * (Not currently useful). */
   int msec_per_tick;
-} tor_libevent_cfg;
+} tor_libevent_cfg_t;
 
-void tor_libevent_initialize(tor_libevent_cfg *cfg);
+void tor_libevent_initialize(tor_libevent_cfg_t *cfg);
 bool tor_libevent_is_initialized(void);
 MOCK_DECL(struct event_base *, tor_libevent_get_base, (void));
 const char *tor_libevent_get_method(void);
diff --git a/src/lib/evloop/evloop_sys.c b/src/lib/evloop/evloop_sys.c
new file mode 100644
index 0000000000..b639810c23
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.c
+ * @brief Subsystem definition for the event loop module
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/evloop/evloop_sys.h"
+#include "lib/log/log.h"
+
+static int
+subsys_evloop_initialize(void)
+{
+  if (tor_init_libevent_rng() < 0) {
+    log_warn(LD_NET, "Problem initializing libevent RNG.");
+    return -1;
+  }
+  return 0;
+}
+
+static void
+subsys_evloop_postfork(void)
+{
+#ifdef TOR_UNIT_TESTS
+  tor_libevent_postfork();
+#endif
+}
+
+static void
+subsys_evloop_shutdown(void)
+{
+  tor_libevent_free_all();
+}
+
+const struct subsys_fns_t sys_evloop = {
+  .name = "evloop",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -20,
+  .initialize = subsys_evloop_initialize,
+  .shutdown = subsys_evloop_shutdown,
+  .postfork = subsys_evloop_postfork,
+};
diff --git a/src/lib/evloop/evloop_sys.h b/src/lib/evloop/evloop_sys.h
new file mode 100644
index 0000000000..a37440e7a6
--- /dev/null
+++ b/src/lib/evloop/evloop_sys.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file evloop_sys.h
+ * @brief Declare subsystem object for the event loop module.
+ **/
+
+#ifndef TOR_LIB_EVLOOP_EVLOOP_SYS_H
+#define TOR_LIB_EVLOOP_EVLOOP_SYS_H
+
+extern const struct subsys_fns_t sys_evloop;
+
+#endif /* !defined(TOR_LIB_EVLOOP_EVLOOP_SYS_H) */
diff --git a/src/lib/evloop/include.am b/src/lib/evloop/include.am
index 6b0076272a..41cd2f45c5 100644
--- a/src/lib/evloop/include.am
+++ b/src/lib/evloop/include.am
@@ -5,21 +5,24 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-evloop-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_evloop_a_SOURCES =			\
 	src/lib/evloop/compat_libevent.c		\
+	src/lib/evloop/evloop_sys.c			\
 	src/lib/evloop/procmon.c			\
 	src/lib/evloop/timers.c				\
 	src/lib/evloop/token_bucket.c			\
 	src/lib/evloop/workqueue.c
 
-
 src_lib_libtor_evloop_testing_a_SOURCES = \
 	$(src_lib_libtor_evloop_a_SOURCES)
 src_lib_libtor_evloop_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_evloop_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/evloop/compat_libevent.h		\
+	src/lib/evloop/evloop_sys.h			\
 	src/lib/evloop/procmon.h			\
 	src/lib/evloop/timers.h				\
 	src/lib/evloop/token_bucket.h			\
diff --git a/src/lib/evloop/lib_evloop.md b/src/lib/evloop/lib_evloop.md
new file mode 100644
index 0000000000..830be88148
--- /dev/null
+++ b/src/lib/evloop/lib_evloop.md
@@ -0,0 +1,7 @@
+@dir /lib/evloop
+@brief lib/evloop: Low-level event loop.
+
+This modules has tools to manage the [libevent](https://libevent.org/) event
+loop and related functionality, in order to implement asynchronous
+networking, timers, periodic events, and other scheduling tasks.
+
diff --git a/src/lib/evloop/procmon.c b/src/lib/evloop/procmon.c
index 52469fa5fc..718c7d4777 100644
--- a/src/lib/evloop/procmon.c
+++ b/src/lib/evloop/procmon.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -165,8 +165,8 @@ tor_validate_process_specifier(const char *process_spec,
   return parse_process_specifier(process_spec, &ppspec, msg);
 }
 
-/* DOCDOC poll_interval_tv */
-static const struct timeval poll_interval_tv = {15, 0};
+/* We check this often for presence of owning controller process. */
+static const struct timeval poll_interval_tv = {15, 0}; // 15 seconds.
 
 /** Create a process-termination monitor for the process specifier
  * given in <b>process_spec</b>.  Return a newly allocated
@@ -303,7 +303,7 @@ tor_process_monitor_poll_cb(periodic_timer_t *event, void *procmon_)
       tor_free(errmsg);
     }
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   /* Unix makes this part easy, if a bit racy. */
   its_dead_jim = kill(procmon->pid, 0);
   its_dead_jim = its_dead_jim && (errno == ESRCH);
diff --git a/src/lib/evloop/procmon.h b/src/lib/evloop/procmon.h
index 6caae5be86..28f443da18 100644
--- a/src/lib/evloop/procmon.h
+++ b/src/lib/evloop/procmon.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/evloop/time_periodic.md b/src/lib/evloop/time_periodic.md
new file mode 100644
index 0000000000..8b3589d9db
--- /dev/null
+++ b/src/lib/evloop/time_periodic.md
@@ -0,0 +1,76 @@
+
+@page time_periodic Time and periodic events in Tor
+
+### What time is it? ###
+
+We have several notions of the current time in Tor.
+
+The *wallclock time* is available from time(NULL) with
+second-granularity and tor_gettimeofday() with microsecond
+granularity.  It corresponds most closely to "the current time and date".
+
+The *monotonic time* is available with the set of monotime_\*
+functions declared in compat_time.h.  Unlike the wallclock time, it
+can only move forward. It does not necessarily correspond to a real
+world time, and it is not portable between systems.
+
+The *coarse monotonic time* is available from the set of
+monotime_coarse_\* functions in compat_time.h. It is the same as
+monotime_\* on some platforms. On others, it gives a monotonic timer
+with less precision, but which it's more efficient to access.
+
+### Cached views of time. ###
+
+On some systems (like Linux), many time functions use a VDSO to avoid
+the overhead of a system call.  But on other systems, gettimeofday()
+and time() can be costly enough that you wouldn't want to call them
+tens of thousands of times.  To get a recent, but not especially
+accurate, view of the current time, see approx_time() and
+tor_gettimeofday_cached().
+
+
+### Parsing and encoding time values ###
+
+Tor has functions to parse and format time in these formats:
+
+ - RFC1123 format. ("Fri, 29 Sep 2006 15:54:20 GMT").  For this,
+   use format_rfc1123_time() and parse_rfc1123_time.
+
+ - ISO8601 format. ("2006-10-29 10:57:20") For this, use
+   format_local_iso_time() and format_iso_time().  We also support the
+   variant format "2006-10-29T10:57:20" with format_iso_time_nospace(), and
+   "2006-10-29T10:57:20.123456" with format_iso_time_nospace_usec().
+
+ - HTTP format collections (preferably "Mon, 25 Jul 2016 04:01:11
+   GMT" or possibly "Wed Jun 30 21:49:08 1993" or even "25-Jul-16
+   04:01:11 GMT"). For this, use parse_http_time().  Don't generate anything
+   but the first format.
+
+Some of these functions use struct tm. You can use the standard
+tor_localtime_r() and tor_gmtime_r() to wrap these in a safe way. We
+also have a tor_timegm() function.
+
+### Scheduling events ###
+
+The main way to schedule a not-too-frequent periodic event with
+respect to the Tor mainloop is via the mechanism in periodic.c.
+There's a big table of periodic_events in mainloop.c, each of which gets
+invoked on its own schedule.  You should not expect more than about
+one second of accuracy with these timers.
+
+You can create an independent timer using libevent directly, or using
+the periodic_timer_new() function.  But you should avoid doing this
+for per-connection or per-circuit timers: Libevent's internal timer
+implementation uses a min-heap, and those tend to start scaling poorly
+once you have a few thousand entries.
+
+If you need to create a large number of fine-grained timers for some
+purpose, you should consider the mechanism in src/common/timers.c,
+which is optimized for the case where you have a large number of
+timers with not-too-long duration, many of which will be deleted
+before they actually expire. These timers should be reasonably
+accurate within a handful of milliseconds -- possibly better on some
+platforms.  (The timers.c module uses William Ahern's timeout.c
+implementation as its backend, which is based on a hierarchical timing
+wheel algorithm. It's cool stuff; check it out.)
+
diff --git a/src/lib/evloop/timers.c b/src/lib/evloop/timers.c
index e46d2635a8..11418e93fd 100644
--- a/src/lib/evloop/timers.c
+++ b/src/lib/evloop/timers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
  * The main advantage of tor_timer_t over using libevent's timers is that
  * they're way more efficient if we need to have thousands or millions of
  * them.  For more information, see
- *   http://www.25thandclement.com/~william/projects/timeout.c.html
+ *   https://www.25thandclement.com/~william/projects/timeout.c.html
  *
  * Periodic timers are available in the backend, but I've turned them off.
  * We can turn them back on if needed.
@@ -48,7 +48,7 @@
 #include <winsock2.h>
 #endif
 
-struct timeout_cb {
+struct timeout_cb_t {
   timer_cb_fn_t cb;
   void *arg;
 };
@@ -56,19 +56,21 @@ struct timeout_cb {
 /*
  * These definitions are for timeouts.c  and timeouts.h.
  */
-#ifdef __GNUC__
+#ifdef COCCI
+#define TIMEOUT_PUBLIC
+#elif defined(__GNUC__)
 /* We're not exposing any of the functions outside this file. */
 #define TIMEOUT_PUBLIC __attribute__((__unused__)) static
 #else
 /* We're not exposing any of the functions outside this file. */
 #define TIMEOUT_PUBLIC static
-#endif /* defined(__GNUC__) */
+#endif /* defined(COCCI) || ... */
 /* We're not using periodic events. */
 #define TIMEOUT_DISABLE_INTERVALS
 /* We always know the global_timeouts object, so we don't need each timeout
  * to keep a pointer to it. */
 #define TIMEOUT_DISABLE_RELATIVE_ACCESS
-/* We're providing our own struct timeout_cb. */
+/* We're providing our own struct timeout_cb_t. */
 #define TIMEOUT_CB_OVERRIDE
 /* We're going to support timers that are pretty far out in advance. Making
  * this big can be inefficient, but having a significant number of timers
@@ -80,7 +82,8 @@ struct timeout_cb {
  * use 32-bit math. */
 #define WHEEL_BIT 5
 #endif
-#include "src/ext/timeouts/timeout.c"
+
+#include "ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
 static struct mainloop_event_t *global_timer_event = NULL;
diff --git a/src/lib/evloop/timers.h b/src/lib/evloop/timers.h
index 7595554204..dd55446121 100644
--- a/src/lib/evloop/timers.h
+++ b/src/lib/evloop/timers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #include "lib/testsupport/testsupport.h"
 
 struct monotime_t;
+struct timeval;
 typedef struct timeout tor_timer_t;
 typedef void (*timer_cb_fn_t)(tor_timer_t *, void *,
                               const struct monotime_t *);
diff --git a/src/lib/evloop/token_bucket.c b/src/lib/evloop/token_bucket.c
index ee6d631e3b..a2b330fddb 100644
--- a/src/lib/evloop/token_bucket.c
+++ b/src/lib/evloop/token_bucket.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -256,3 +256,55 @@ token_bucket_rw_dec(token_bucket_rw_t *bucket,
     flags |= TB_WRITE;
   return flags;
 }
+
+/** Initialize a token bucket in <b>bucket</b>, set up to allow <b>rate</b>
+ * per second, with a maximum burst of <b>burst</b>. The bucket is created
+ * such that <b>now_ts</b> is the current timestamp. The bucket starts out
+ * full. */
+void
+token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                      uint32_t burst, uint32_t now_ts)
+{
+  memset(bucket, 0, sizeof(token_bucket_ctr_t));
+  token_bucket_ctr_adjust(bucket, rate, burst);
+  token_bucket_ctr_reset(bucket, now_ts);
+}
+
+/** Change the configured rate and burst of the given token bucket object in
+ * <b>bucket</b>. */
+void
+token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                        uint32_t burst)
+{
+  token_bucket_cfg_init(&bucket->cfg, rate, burst);
+  token_bucket_raw_adjust(&bucket->counter, &bucket->cfg);
+}
+
+/** Reset <b>bucket</b> to be full, as of timestamp <b>now_ts</b>. */
+void
+token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  token_bucket_raw_reset(&bucket->counter, &bucket->cfg);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
+
+/** Refill <b>bucket</b> as appropriate, given that the current timestamp is
+ * <b>now_ts</b>. */
+void
+token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts)
+{
+  const uint32_t elapsed_ticks =
+    (now_ts - bucket->last_refilled_at_timestamp);
+  if (elapsed_ticks > UINT32_MAX-(300*1000)) {
+    /* Either about 48 days have passed since the last refill, or the
+     * monotonic clock has somehow moved backwards. (We're looking at you,
+     * Windows.).  We accept up to a 5 minute jump backwards as
+     * "unremarkable".
+     */
+    return;
+  }
+
+  token_bucket_raw_refill_steps(&bucket->counter, &bucket->cfg,
+                                elapsed_ticks);
+  bucket->last_refilled_at_timestamp = now_ts;
+}
diff --git a/src/lib/evloop/token_bucket.h b/src/lib/evloop/token_bucket.h
index 9398d2baa3..460dad23e4 100644
--- a/src/lib/evloop/token_bucket.h
+++ b/src/lib/evloop/token_bucket.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -103,6 +103,35 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
   return token_bucket_raw_get(&bucket->write_bucket);
 }
 
+/**
+ * A specialized bucket containing a single counter.
+ */
+
+typedef struct token_bucket_ctr_t {
+  token_bucket_cfg_t cfg;
+  token_bucket_raw_t counter;
+  uint32_t last_refilled_at_timestamp;
+} token_bucket_ctr_t;
+
+void token_bucket_ctr_init(token_bucket_ctr_t *bucket, uint32_t rate,
+                           uint32_t burst, uint32_t now_ts);
+void token_bucket_ctr_adjust(token_bucket_ctr_t *bucket, uint32_t rate,
+                             uint32_t burst);
+void token_bucket_ctr_reset(token_bucket_ctr_t *bucket, uint32_t now_ts);
+void token_bucket_ctr_refill(token_bucket_ctr_t *bucket, uint32_t now_ts);
+
+static inline bool
+token_bucket_ctr_dec(token_bucket_ctr_t *bucket, ssize_t n)
+{
+  return token_bucket_raw_dec(&bucket->counter, n);
+}
+
+static inline size_t
+token_bucket_ctr_get(const token_bucket_ctr_t *bucket)
+{
+  return token_bucket_raw_get(&bucket->counter);
+}
+
 #ifdef TOKEN_BUCKET_PRIVATE
 
 /* To avoid making the rates too small, we consider units of "steps",
@@ -112,6 +141,6 @@ token_bucket_rw_get_write(const token_bucket_rw_t *bucket)
 
 STATIC uint32_t rate_per_sec_to_rate_per_step(uint32_t rate);
 
-#endif
+#endif /* defined(TOKEN_BUCKET_PRIVATE) */
 
-#endif /* TOR_TOKEN_BUCKET_H */
+#endif /* !defined(TOR_TOKEN_BUCKET_H) */
diff --git a/src/lib/evloop/workqueue.c b/src/lib/evloop/workqueue.c
index 931f65e710..603dddd5a3 100644
--- a/src/lib/evloop/workqueue.c
+++ b/src/lib/evloop/workqueue.c
@@ -15,7 +15,7 @@
  *
  * The main thread informs the worker threads of pending work by using a
  * condition variable.  The workers inform the main process of completed work
- * by using an alert_sockets_t object, as implemented in compat_threads.c.
+ * by using an alert_sockets_t object, as implemented in net/alertsock.c.
  *
  * The main thread can also queue an "update" that will be handled by all the
  * workers.  This is useful for updating state that all the workers share.
@@ -36,7 +36,7 @@
 #include "lib/net/socket.h"
 #include "lib/thread/threads.h"
 
-#include "tor_queue.h"
+#include "ext/tor_queue.h"
 #include <event2/event.h>
 #include <string.h>
 
@@ -44,13 +44,13 @@
 #define WORKQUEUE_PRIORITY_LAST WQ_PRI_LOW
 #define WORKQUEUE_N_PRIORITIES (((int) WORKQUEUE_PRIORITY_LAST)+1)
 
-TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_s);
+TOR_TAILQ_HEAD(work_tailq_t, workqueue_entry_t);
 typedef struct work_tailq_t work_tailq_t;
 
-struct threadpool_s {
+struct threadpool_t {
   /** An array of pointers to workerthread_t: one for each running worker
    * thread. */
-  struct workerthread_s **threads;
+  struct workerthread_t **threads;
 
   /** Condition variable that we wait on when we have no work, and which
    * gets signaled when our queue becomes nonempty. */
@@ -59,9 +59,6 @@ struct threadpool_s {
    * <b>p</b> is work[p]. */
   work_tailq_t work[WORKQUEUE_N_PRIORITIES];
 
-  /** Weak RNG, used to decide when to ignore priority. */
-  tor_weak_rng_t weak_rng;
-
   /** The current 'update generation' of the threadpool.  Any thread that is
    * at an earlier generation needs to run the update function. */
   unsigned generation;
@@ -95,14 +92,14 @@ struct threadpool_s {
 /** Number of bits needed to hold all legal values of workqueue_priority_t */
 #define WORKQUEUE_PRIORITY_BITS 2
 
-struct workqueue_entry_s {
+struct workqueue_entry_t {
   /** The next workqueue_entry_t that's pending on the same thread or
    * reply queue. */
-  TOR_TAILQ_ENTRY(workqueue_entry_s) next_work;
+  TOR_TAILQ_ENTRY(workqueue_entry_t) next_work;
   /** The threadpool to which this workqueue_entry_t was assigned. This field
    * is set when the workqueue_entry_t is created, and won't be cleared until
    * after it's handled in the main thread. */
-  struct threadpool_s *on_pool;
+  struct threadpool_t *on_pool;
   /** True iff this entry is waiting for a worker to start processing it. */
   uint8_t pending;
   /** Priority of this entry. */
@@ -115,22 +112,22 @@ struct workqueue_entry_s {
   void *arg;
 };
 
-struct replyqueue_s {
+struct replyqueue_t {
   /** Mutex to protect the answers field */
   tor_mutex_t lock;
   /** Doubly-linked list of answers that the reply queue needs to handle. */
-  TOR_TAILQ_HEAD(, workqueue_entry_s) answers;
+  TOR_TAILQ_HEAD(, workqueue_entry_t) answers;
 
   /** Mechanism to wake up the main thread when it is receiving answers. */
   alert_sockets_t alert;
 };
 
 /** A worker thread represents a single thread in a thread pool. */
-typedef struct workerthread_s {
+typedef struct workerthread_t {
   /** Which thread it this?  In range 0..in_pool->n_threads-1 */
   int index;
   /** The pool this thread is a part of. */
-  struct threadpool_s *in_pool;
+  struct threadpool_t *in_pool;
   /** User-supplied state field that we pass to the worker functions of each
    * work item. */
   void *state;
@@ -238,7 +235,7 @@ worker_thread_extract_next_work(workerthread_t *thread)
     this_queue = &pool->work[i];
     if (!TOR_TAILQ_EMPTY(this_queue)) {
       queue = this_queue;
-      if (! tor_weak_random_one_in_n(&pool->weak_rng,
+      if (! crypto_fast_rng_one_in_n(get_thread_fast_rng(),
                                      thread->lower_priority_chance)) {
         /* Usually we'll just break now, so that we can get out of the loop
          * and use the queue where we found work. But with a small
@@ -555,11 +552,6 @@ threadpool_new(int n_threads,
   for (i = WORKQUEUE_PRIORITY_FIRST; i <= WORKQUEUE_PRIORITY_LAST; ++i) {
     TOR_TAILQ_INIT(&pool->work[i]);
   }
-  {
-    unsigned seed;
-    crypto_rand((void*)&seed, sizeof(seed));
-    tor_init_weak_random(&pool->weak_rng, seed);
-  }
 
   pool->new_thread_state_fn = new_thread_state_fn;
   pool->new_thread_state_arg = arg;
@@ -622,8 +614,8 @@ reply_event_cb(evutil_socket_t sock, short events, void *arg)
     tp->reply_cb(tp);
 }
 
-/** Register the threadpool <b>tp</b>'s reply queue with the libevent
- * mainloop of <b>base</b>. If <b>tp</b> is provided, it is run after
+/** Register the threadpool <b>tp</b>'s reply queue with Tor's global
+ * libevent mainloop. If <b>cb</b> is provided, it is run after
  * each time there is work to process from the reply queue. Return 0 on
  * success, -1 on failure.
  */
diff --git a/src/lib/evloop/workqueue.h b/src/lib/evloop/workqueue.h
index 333a3f6dde..43cfebf788 100644
--- a/src/lib/evloop/workqueue.h
+++ b/src/lib/evloop/workqueue.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,12 +13,12 @@
 
 /** A replyqueue is used to tell the main thread about the outcome of
  * work that we queued for the workers. */
-typedef struct replyqueue_s replyqueue_t;
+typedef struct replyqueue_t replyqueue_t;
 /** A thread-pool manages starting threads and passing work to them. */
-typedef struct threadpool_s threadpool_t;
+typedef struct threadpool_t threadpool_t;
 /** A workqueue entry represents a request that has been passed to a thread
  * pool. */
-typedef struct workqueue_entry_s workqueue_entry_t;
+typedef struct workqueue_entry_t workqueue_entry_t;
 
 /** Possible return value from a work function: */
 typedef enum workqueue_reply_t {
@@ -63,7 +63,6 @@ replyqueue_t *threadpool_get_replyqueue(threadpool_t *tp);
 replyqueue_t *replyqueue_new(uint32_t alertsocks_flags);
 void replyqueue_process(replyqueue_t *queue);
 
-struct event_base;
 int threadpool_register_reply_event(threadpool_t *tp,
                                     void (*cb)(threadpool_t *tp));
 
diff --git a/src/lib/fdio/fdio.c b/src/lib/fdio/fdio.c
index d723d04d2a..56e3818f5c 100644
--- a/src/lib/fdio/fdio.c
+++ b/src/lib/fdio/fdio.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,16 +17,21 @@
 #ifdef _WIN32
 #include <windows.h>
 #endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 #include "lib/fdio/fdio.h"
 #include "lib/cc/torint.h"
 #include "lib/err/torerr.h"
 
 #include <stdlib.h>
+#include <stdio.h>
 
-/** @{ */
-/** Some old versions of Unix didn't define constants for these values,
+/* Some old versions of Unix didn't define constants for these values,
  * and instead expect you to say 0, 1, or 2. */
+
+/** @cond */
 #ifndef SEEK_SET
 #define SEEK_SET 0
 #endif
@@ -36,7 +41,7 @@
 #ifndef SEEK_END
 #define SEEK_END 2
 #endif
-/** @} */
+/** @endcond */
 
 /** Return the position of <b>fd</b> with respect to the start of the file. */
 off_t
diff --git a/src/lib/fdio/fdio.h b/src/lib/fdio/fdio.h
index 8395af353b..99bc33c64b 100644
--- a/src/lib/fdio/fdio.h
+++ b/src/lib/fdio/fdio.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,9 @@
 #define TOR_FDIO_H
 
 #include <stddef.h>
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
 
 off_t tor_fd_getpos(int fd);
 int tor_fd_setpos(int fd, off_t pos);
diff --git a/src/lib/fdio/include.am b/src/lib/fdio/include.am
index 6c18f00a0d..545bbc929e 100644
--- a/src/lib/fdio/include.am
+++ b/src/lib/fdio/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fdio-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fdio_a_SOURCES =			\
 	src/lib/fdio/fdio.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_fdio_testing_a_SOURCES = \
 src_lib_libtor_fdio_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fdio_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fdio/fdio.h
diff --git a/src/lib/fdio/lib_fdio.md b/src/lib/fdio/lib_fdio.md
new file mode 100644
index 0000000000..9fe4b4d2be
--- /dev/null
+++ b/src/lib/fdio/lib_fdio.md
@@ -0,0 +1,5 @@
+@dir /lib/fdio
+@brief lib/fdio: Code to read/write on file descriptors.
+
+(This module also handles sockets, on platforms where a socket is not a kind
+of fd.)
diff --git a/src/lib/fs/.may_include b/src/lib/fs/.may_include
index b1e49fc891..c192e6181c 100644
--- a/src/lib/fs/.may_include
+++ b/src/lib/fs/.may_include
@@ -13,4 +13,4 @@ lib/malloc/*.h
 lib/memarea/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/fs/conffile.c b/src/lib/fs/conffile.c
index 0d5d56b335..9583093c12 100644
--- a/src/lib/fs/conffile.c
+++ b/src/lib/fs/conffile.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -152,6 +152,7 @@ config_process_include(const char *path, int recursion_level, int extended,
 
   int rv = -1;
   SMARTLIST_FOREACH_BEGIN(config_files, const char *, config_file) {
+    log_notice(LD_CONFIG, "Including configuration file \"%s\".", config_file);
     config_line_t *included_config = NULL;
     config_line_t *included_config_last = NULL;
     if (config_get_included_config(config_file, recursion_level, extended,
diff --git a/src/lib/fs/conffile.h b/src/lib/fs/conffile.h
index 7af9119dbb..cbfe4ec7c9 100644
--- a/src/lib/fs/conffile.h
+++ b/src/lib/fs/conffile.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_CONFFILE_H
@@ -20,4 +20,4 @@ int config_get_lines_include(const char *string, struct config_line_t **result,
                              int extended, int *has_include,
                              struct smartlist_t *opened_lst);
 
-#endif /* !defined(TOR_CONFLINE_H) */
+#endif /* !defined(TOR_CONFFILE_H) */
diff --git a/src/lib/fs/dir.c b/src/lib/fs/dir.c
index 3c31e00d99..3432df0299 100644
--- a/src/lib/fs/dir.c
+++ b/src/lib/fs/dir.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -262,7 +262,7 @@ check_private_dir,(const char *dirname, cpd_check_t check,
     }
   }
   close(fd);
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   /* Win32 case: we can't open() a directory. */
   (void)effective_user;
 
@@ -347,7 +347,7 @@ tor_listdir, (const char *dirname))
   }
   FindClose(handle);
   tor_free(pattern);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *prot_dname = sandbox_intern_string(dirname);
   DIR *d;
   struct dirent *de;
diff --git a/src/lib/fs/dir.h b/src/lib/fs/dir.h
index 826bc2dfc5..c4ab430891 100644
--- a/src/lib/fs/dir.h
+++ b/src/lib/fs/dir.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_DIR_H
@@ -30,4 +30,4 @@ MOCK_DECL(int, check_private_dir, (const char *dirname, cpd_check_t check,
 
 MOCK_DECL(struct smartlist_t *, tor_listdir, (const char *dirname));
 
-#endif
+#endif /* !defined(TOR_DIR_H) */
diff --git a/src/lib/fs/files.c b/src/lib/fs/files.c
index ec7dbca0e9..a0b5a40aac 100644
--- a/src/lib/fs/files.c
+++ b/src/lib/fs/files.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/files.h b/src/lib/fs/files.h
index 52c94c914f..a109cd6248 100644
--- a/src/lib/fs/files.h
+++ b/src/lib/fs/files.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,7 +27,7 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
-#endif
+#endif /* defined(_WIN32) */
 
 #ifndef O_BINARY
 #define O_BINARY 0
@@ -108,7 +108,7 @@ char *read_file_to_str_until_eof(int fd, size_t max_bytes_to_read,
  * Tor is built for unit tests, or when Tor is built on an operating system
  * without its own getdelim(). */
 ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
-#endif
+#endif /* !defined(HAVE_GETDELIM) || defined(TOR_UNIT_TESTS) */
 
 #ifdef HAVE_GETDELIM
 /**
@@ -123,10 +123,10 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getdelim(lineptr, n, delim, stream) \
   getdelim((lineptr), (n), (delim), (stream))
-#else
+#else /* !defined(HAVE_GETDELIM) */
 #define tor_getdelim(lineptr, n, delim, stream) \
   compat_getdelim_((lineptr), (n), (delim), (stream))
-#endif
+#endif /* defined(HAVE_GETDELIM) */
 
 #ifdef HAVE_GETLINE
 /**
@@ -137,9 +137,9 @@ ssize_t compat_getdelim_(char **lineptr, size_t *n, int delim, FILE *stream);
  */
 #define tor_getline(lineptr, n, stream) \
   getline((lineptr), (n), (stream))
-#else
+#else /* !defined(HAVE_GETLINE) */
 #define tor_getline(lineptr, n, stream) \
   tor_getdelim((lineptr), (n), '\n', (stream))
-#endif
+#endif /* defined(HAVE_GETLINE) */
 
-#endif
+#endif /* !defined(TOR_FS_H) */
diff --git a/src/lib/fs/freespace.c b/src/lib/fs/freespace.c
index ee0f93073d..511f2a0b98 100644
--- a/src/lib/fs/freespace.c
+++ b/src/lib/fs/freespace.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/include.am b/src/lib/fs/include.am
index f33e4d6430..493db8f044 100644
--- a/src/lib/fs/include.am
+++ b/src/lib/fs/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-fs-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_fs_a_SOURCES =			\
 	src/lib/fs/conffile.c			\
 	src/lib/fs/dir.c			\
@@ -25,6 +26,7 @@ src_lib_libtor_fs_testing_a_SOURCES = \
 src_lib_libtor_fs_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_fs_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/fs/conffile.h				\
 	src/lib/fs/dir.h				\
diff --git a/src/lib/fs/lib_fs.md b/src/lib/fs/lib_fs.md
new file mode 100644
index 0000000000..3b5b0ac7d5
--- /dev/null
+++ b/src/lib/fs/lib_fs.md
@@ -0,0 +1,9 @@
+@dir /lib/fs
+@brief lib/fs: Files, filenames, directories, etc.
+
+This module is mostly a set of compatibility wrappers around
+operating-system-specific filesystem access.
+
+It also contains a set of convenience functions for safely writing to files,
+creating directories, and so on.
+
diff --git a/src/lib/fs/lockfile.c b/src/lib/fs/lockfile.c
index 933ff1e02f..c081f57a5d 100644
--- a/src/lib/fs/lockfile.c
+++ b/src/lib/fs/lockfile.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/lockfile.h b/src/lib/fs/lockfile.h
index 8aeee4cc7f..91effd701d 100644
--- a/src/lib/fs/lockfile.h
+++ b/src/lib/fs/lockfile.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@ tor_lockfile_t *tor_lockfile_lock(const char *filename, int blocking,
                                   int *locked_out);
 void tor_lockfile_unlock(tor_lockfile_t *lockfile);
 
-#endif
+#endif /* !defined(TOR_LOCKFILE_H) */
diff --git a/src/lib/fs/mmap.c b/src/lib/fs/mmap.c
index daaee1f9b1..cc1c40b7ab 100644
--- a/src/lib/fs/mmap.c
+++ b/src/lib/fs/mmap.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,8 +42,8 @@
  * failure, return NULL. Sets errno properly, using ERANGE to mean
  * "empty file". Must only be called on trusted Tor-owned files, as changing
  * the underlying file's size causes unspecified behavior. */
-tor_mmap_t *
-tor_mmap_file(const char *filename)
+MOCK_IMPL(tor_mmap_t *,
+tor_mmap_file,(const char *filename))
 {
   int fd; /* router file */
   char *string;
@@ -111,8 +111,8 @@ tor_mmap_file(const char *filename)
 }
 /** Release storage held for a memory mapping; returns 0 on success,
  * or -1 on failure (and logs a warning). */
-int
-tor_munmap_file(tor_mmap_t *handle)
+MOCK_IMPL(int,
+tor_munmap_file,(tor_mmap_t *handle))
 {
   int res;
 
@@ -132,8 +132,8 @@ tor_munmap_file(tor_mmap_t *handle)
   return res;
 }
 #elif defined(_WIN32)
-tor_mmap_t *
-tor_mmap_file(const char *filename)
+MOCK_IMPL(tor_mmap_t *,
+tor_mmap_file,(const char *filename))
 {
   TCHAR tfilename[MAX_PATH]= {0};
   tor_mmap_t *res = tor_malloc_zero(sizeof(tor_mmap_t));
@@ -213,8 +213,8 @@ tor_mmap_file(const char *filename)
 }
 
 /* Unmap the file, and return 0 for success or -1 for failure */
-int
-tor_munmap_file(tor_mmap_t *handle)
+MOCK_IMPL(int,
+tor_munmap_file,(tor_mmap_t *handle))
 {
   if (handle == NULL)
     return 0;
@@ -237,4 +237,4 @@ tor_munmap_file(tor_mmap_t *handle)
 }
 #else
 #error "cannot implement tor_mmap_file"
-#endif /* defined(HAVE_MMAP) || ... || ... */
+#endif /* defined(HAVE_MMAP) || defined(RUNNING_DOXYGEN) || ... */
diff --git a/src/lib/fs/mmap.h b/src/lib/fs/mmap.h
index 18fb18a13c..e142bd78c3 100644
--- a/src/lib/fs/mmap.h
+++ b/src/lib/fs/mmap.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,6 +13,7 @@
 #define TOR_MMAP_H
 
 #include "lib/cc/compat_compiler.h"
+#include "lib/testsupport/testsupport.h"
 #include <stddef.h>
 
 #ifdef _WIN32
@@ -35,7 +36,7 @@ typedef struct tor_mmap_t {
 
 } tor_mmap_t;
 
-tor_mmap_t *tor_mmap_file(const char *filename);
-int tor_munmap_file(tor_mmap_t *handle);
+MOCK_DECL(tor_mmap_t *, tor_mmap_file, (const char *filename));
+MOCK_DECL(int, tor_munmap_file, (tor_mmap_t *handle));
 
-#endif
+#endif /* !defined(TOR_MMAP_H) */
diff --git a/src/lib/fs/path.c b/src/lib/fs/path.c
index b3ef61979d..0d57be4b06 100644
--- a/src/lib/fs/path.c
+++ b/src/lib/fs/path.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -72,7 +72,7 @@ expand_filename(const char *filename)
    *     Chapter+3.+Input+Validation/3.7+Validating+Filenames+and+Paths/
    */
   return tor_strdup(filename);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   if (*filename == '~') {
     char *home, *result=NULL;
     const char *rest;
@@ -102,7 +102,7 @@ expand_filename(const char *filename)
       }
       tor_free(username);
       rest = slash ? (slash+1) : "";
-#else /* !(defined(HAVE_PWD_H)) */
+#else /* !defined(HAVE_PWD_H) */
       log_warn(LD_CONFIG, "Couldn't expand homedir on system without pwd.h");
       return tor_strdup(filename);
 #endif /* defined(HAVE_PWD_H) */
@@ -153,7 +153,7 @@ clean_fname_for_stat(char *name)
       return;
     name[len-1]='\0';
   }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   (void)name;
 #endif /* defined(_WIN32) */
 }
@@ -233,7 +233,7 @@ alloc_getcwd(void)
     raw_free(cwd); // alias for free to avoid tripping check-spaces.
   }
   return result;
-#else /* !(defined(HAVE_GET_CURRENT_DIR_NAME)) */
+#else /* !defined(HAVE_GET_CURRENT_DIR_NAME) */
   size_t size = 1024;
   char *buf = NULL;
   char *ptr = NULL;
@@ -255,9 +255,10 @@ alloc_getcwd(void)
 #endif /* !defined(_WIN32) */
 
 /** Expand possibly relative path <b>fname</b> to an absolute path.
- * Return a newly allocated string, possibly equal to <b>fname</b>. */
+ * Return a newly allocated string, which may be a duplicate of <b>fname</b>.
+ */
 char *
-make_path_absolute(char *fname)
+make_path_absolute(const char *fname)
 {
 #ifdef _WIN32
   char *absfname_malloced = _fullpath(NULL, fname, 1);
@@ -268,7 +269,7 @@ make_path_absolute(char *fname)
   if (absfname_malloced) raw_free(absfname_malloced);
 
   return absfname;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   char *absfname = NULL, *path = NULL;
 
   tor_assert(fname);
diff --git a/src/lib/fs/path.h b/src/lib/fs/path.h
index 4675ac84e8..f0e253c556 100644
--- a/src/lib/fs/path.h
+++ b/src/lib/fs/path.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,6 +25,6 @@ char *expand_filename(const char *filename);
 int path_is_relative(const char *filename);
 void clean_fname_for_stat(char *name);
 int get_parent_directory(char *fname);
-char *make_path_absolute(char *fname);
+char *make_path_absolute(const char *fname);
 
-#endif
+#endif /* !defined(TOR_PATH_H) */
diff --git a/src/lib/fs/storagedir.c b/src/lib/fs/storagedir.c
index 2caddf1ad9..5e4f9ee257 100644
--- a/src/lib/fs/storagedir.c
+++ b/src/lib/fs/storagedir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/storagedir.h b/src/lib/fs/storagedir.h
index 7e6633a0bb..9997550b8f 100644
--- a/src/lib/fs/storagedir.h
+++ b/src/lib/fs/storagedir.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,7 +15,7 @@
 
 typedef struct storage_dir_t storage_dir_t;
 struct config_line_t;
-struct sandbox_cfg_elem;
+struct sandbox_cfg_elem_t;
 struct tor_mmap_t;
 struct smartlist_t;
 
@@ -25,7 +25,7 @@ void storage_dir_free_(storage_dir_t *d);
   FREE_AND_NULL(storage_dir_t, storage_dir_free_, (d))
 
 int storage_dir_register_with_sandbox(storage_dir_t *d,
-                                      struct sandbox_cfg_elem **cfg);
+                                      struct sandbox_cfg_elem_t **cfg);
 const struct smartlist_t *storage_dir_list(storage_dir_t *d);
 uint64_t storage_dir_get_usage(storage_dir_t *d);
 struct tor_mmap_t *storage_dir_map(storage_dir_t *d, const char *fname);
diff --git a/src/lib/fs/userdb.c b/src/lib/fs/userdb.c
index 95205c670e..40fc4dae97 100644
--- a/src/lib/fs/userdb.c
+++ b/src/lib/fs/userdb.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/userdb.h b/src/lib/fs/userdb.h
index 5c39794873..4341237c5f 100644
--- a/src/lib/fs/userdb.h
+++ b/src/lib/fs/userdb.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,6 +21,6 @@ struct passwd;
 const struct passwd *tor_getpwnam(const char *username);
 const struct passwd *tor_getpwuid(uid_t uid);
 char *get_user_homedir(const char *username);
-#endif
+#endif /* !defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_USERDB_H) */
diff --git a/src/lib/fs/winlib.c b/src/lib/fs/winlib.c
index b7302bd4ca..65ccdae40b 100644
--- a/src/lib/fs/winlib.c
+++ b/src/lib/fs/winlib.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/fs/winlib.h b/src/lib/fs/winlib.h
index 64a22439e5..27837ac46e 100644
--- a/src/lib/fs/winlib.h
+++ b/src/lib/fs/winlib.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,6 +17,6 @@
 #include <tchar.h>
 
 HANDLE load_windows_system_library(const TCHAR *library_name);
-#endif
+#endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_WINLIB_H) */
diff --git a/src/lib/geoip/country.h b/src/lib/geoip/country.h
index 9a8911d494..feab554a16 100644
--- a/src/lib/geoip/country.h
+++ b/src/lib/geoip/country.h
@@ -1,9 +1,14 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file country.h
+ * @brief Country type for geoip.
+ **/
+
 #ifndef TOR_COUNTRY_H
 #define TOR_COUNTRY_H
 
@@ -11,6 +16,7 @@
 /** A signed integer representing a country code. */
 typedef int16_t country_t;
 
+/** Maximum value for country_t. */
 #define COUNTRY_MAX INT16_MAX
 
-#endif
+#endif /* !defined(TOR_COUNTRY_H) */
diff --git a/src/lib/geoip/geoip.c b/src/lib/geoip/geoip.c
index 70b1c2dc8c..ee03d5baa1 100644
--- a/src/lib/geoip/geoip.c
+++ b/src/lib/geoip/geoip.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -70,12 +70,18 @@ static smartlist_t *geoip_countries = NULL;
  * The index is encoded in the pointer, and 1 is added so that NULL can mean
  * not found. */
 static strmap_t *country_idxplus1_by_lc_code = NULL;
-/** Lists of all known geoip_ipv4_entry_t and geoip_ipv6_entry_t, sorted
- * by their respective ip_low. */
-static smartlist_t *geoip_ipv4_entries = NULL, *geoip_ipv6_entries = NULL;
-
-/** SHA1 digest of the GeoIP files to include in extra-info descriptors. */
+/** List of all known geoip_ipv4_entry_t sorted
+ * by their respective ip_low values. */
+static smartlist_t *geoip_ipv4_entries = NULL;
+/** List of all known geoip_ipv6_entry_t, sorted by their respective
+ * ip_low values. */
+static smartlist_t *geoip_ipv6_entries = NULL;
+
+/** SHA1 digest of the IPv4 GeoIP file to include in extra-info
+ * descriptors. */
 static char geoip_digest[DIGEST_LEN];
+/** SHA1 digest of the IPv6 GeoIP file to include in extra-info
+ * descriptors. */
 static char geoip6_digest[DIGEST_LEN];
 
 /** Return a list of geoip_country_t for all known countries. */
diff --git a/src/lib/geoip/geoip.h b/src/lib/geoip/geoip.h
index f872ebd25f..2fc7fae754 100644
--- a/src/lib/geoip/geoip.h
+++ b/src/lib/geoip/geoip.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,6 +31,7 @@ int geoip_get_country_by_ipv6(const struct in6_addr *addr);
 
 /** A per-country GeoIP record. */
 typedef struct geoip_country_t {
+  /** A nul-terminated two-letter country-code. */
   char countrycode[3];
 } geoip_country_t;
 
diff --git a/src/lib/geoip/include.am b/src/lib/geoip/include.am
index 9710d75ac7..ea426d14bc 100644
--- a/src/lib/geoip/include.am
+++ b/src/lib/geoip/include.am
@@ -4,6 +4,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-geoip-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_geoip_a_SOURCES =			\
 	src/lib/geoip/geoip.c
 
@@ -12,6 +13,7 @@ src_lib_libtor_geoip_testing_a_SOURCES = \
 src_lib_libtor_geoip_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_geoip_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/geoip/geoip.h   			\
 	src/lib/geoip/country.h
diff --git a/src/lib/geoip/lib_geoip.md b/src/lib/geoip/lib_geoip.md
new file mode 100644
index 0000000000..a3ee39d574
--- /dev/null
+++ b/src/lib/geoip/lib_geoip.md
@@ -0,0 +1,3 @@
+@dir /lib/geoip
+@brief lib/geoip: IP-to-country mapping
+
diff --git a/src/lib/intmath/addsub.c b/src/lib/intmath/addsub.c
index 12146f4e72..44613417fa 100644
--- a/src/lib/intmath/addsub.c
+++ b/src/lib/intmath/addsub.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/addsub.h b/src/lib/intmath/addsub.h
index 83efa82919..bca911103b 100644
--- a/src/lib/intmath/addsub.h
+++ b/src/lib/intmath/addsub.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,4 +16,4 @@
 
 uint32_t tor_add_u32_nowrap(uint32_t a, uint32_t b);
 
-#endif /* !defined(TOR_INTMATH_MULDIV_H) */
+#endif /* !defined(TOR_INTMATH_ADDSUB_H) */
diff --git a/src/lib/intmath/bits.c b/src/lib/intmath/bits.c
index 2158790e3f..dace9ffe18 100644
--- a/src/lib/intmath/bits.c
+++ b/src/lib/intmath/bits.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/bits.h b/src/lib/intmath/bits.h
index c1483a18b8..687651ba35 100644
--- a/src/lib/intmath/bits.h
+++ b/src/lib/intmath/bits.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/cmp.h b/src/lib/intmath/cmp.h
index d0b0e8b954..4e6c2b649a 100644
--- a/src/lib/intmath/cmp.h
+++ b/src/lib/intmath/cmp.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,4 +36,7 @@
     ((v) > (max)) ? (max) :                     \
     (v) )
 
+/** Give the absolute value of <b>x</b>, independent of its type. */
+#define ABS(x) ( ((x)<0) ? -(x) : (x) )
+
 #endif /* !defined(TOR_INTMATH_CMP_H) */
diff --git a/src/lib/intmath/include.am b/src/lib/intmath/include.am
index 45ee3bd53b..155ffa145a 100644
--- a/src/lib/intmath/include.am
+++ b/src/lib/intmath/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-intmath-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_intmath_a_SOURCES =			\
 	src/lib/intmath/addsub.c			\
 	src/lib/intmath/bits.c				\
@@ -16,6 +17,7 @@ src_lib_libtor_intmath_testing_a_SOURCES = \
 src_lib_libtor_intmath_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_intmath_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/intmath/addsub.h			\
 	src/lib/intmath/cmp.h				\
diff --git a/src/lib/intmath/lib_intmath.md b/src/lib/intmath/lib_intmath.md
new file mode 100644
index 0000000000..4446b715cb
--- /dev/null
+++ b/src/lib/intmath/lib_intmath.md
@@ -0,0 +1,2 @@
+@dir /lib/intmath
+@brief lib/intmath: Integer mathematics.
diff --git a/src/lib/intmath/logic.h b/src/lib/intmath/logic.h
index a4cecd69cc..b5fb79f66e 100644
--- a/src/lib/intmath/logic.h
+++ b/src/lib/intmath/logic.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@
 /** Macro: true if two values have different boolean values. */
 #define bool_neq(a,b) (!(a)!=!(b))
 
-#endif
+#endif /* !defined(HAVE_TOR_LOGIC_H) */
diff --git a/src/lib/intmath/muldiv.c b/src/lib/intmath/muldiv.c
index 6a292db7ba..875cf1bbf2 100644
--- a/src/lib/intmath/muldiv.c
+++ b/src/lib/intmath/muldiv.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -69,6 +69,20 @@ gcd64(uint64_t a, uint64_t b)
   return a;
 }
 
+/** Return the unsigned integer product of <b>a</b> and <b>b</b>. If overflow
+ * is detected, return UINT64_MAX instead. */
+uint64_t
+tor_mul_u64_nowrap(uint64_t a, uint64_t b)
+{
+  if (a == 0 || b == 0) {
+    return 0;
+  } else if (PREDICT_UNLIKELY(UINT64_MAX / a < b)) {
+    return UINT64_MAX;
+  } else {
+    return a*b;
+  }
+}
+
 /* Given a fraction *<b>numer</b> / *<b>denom</b>, simplify it.
  * Requires that the denominator is greater than 0. */
 void
diff --git a/src/lib/intmath/muldiv.h b/src/lib/intmath/muldiv.h
index 64500b6dce..43700cf1dc 100644
--- a/src/lib/intmath/muldiv.h
+++ b/src/lib/intmath/muldiv.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,6 +18,8 @@ unsigned round_to_next_multiple_of(unsigned number, unsigned divisor);
 uint32_t round_uint32_to_next_multiple_of(uint32_t number, uint32_t divisor);
 uint64_t round_uint64_to_next_multiple_of(uint64_t number, uint64_t divisor);
 
+uint64_t tor_mul_u64_nowrap(uint64_t a, uint64_t b);
+
 void simplify_fraction64(uint64_t *numer, uint64_t *denom);
 
 /* Compute the CEIL of <b>a</b> divided by <b>b</b>, for nonnegative <b>a</b>
diff --git a/src/lib/intmath/weakrng.c b/src/lib/intmath/weakrng.c
index 99c9252c2b..a29a6a086b 100644
--- a/src/lib/intmath/weakrng.c
+++ b/src/lib/intmath/weakrng.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/intmath/weakrng.h b/src/lib/intmath/weakrng.h
index e26bf58cbb..0394e05f79 100644
--- a/src/lib/intmath/weakrng.h
+++ b/src/lib/intmath/weakrng.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,11 @@ typedef struct tor_weak_rng_t {
   uint32_t state;
 } tor_weak_rng_t;
 
+#ifndef COCCI
 #define TOR_WEAK_RNG_INIT {383745623}
+#endif
 #define TOR_WEAK_RANDOM_MAX (INT_MAX)
+
 void tor_init_weak_random(tor_weak_rng_t *weak_rng, unsigned seed);
 int32_t tor_weak_random(tor_weak_rng_t *weak_rng);
 int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
@@ -28,4 +31,4 @@ int32_t tor_weak_random_range(tor_weak_rng_t *rng, int32_t top);
  * <b>n</b> */
 #define tor_weak_random_one_in_n(rng, n) (0==tor_weak_random_range((rng),(n)))
 
-#endif
+#endif /* !defined(TOR_WEAKRNG_H) */
diff --git a/src/lib/lib.md b/src/lib/lib.md
new file mode 100644
index 0000000000..4f77a4c1d0
--- /dev/null
+++ b/src/lib/lib.md
@@ -0,0 +1,131 @@
+@dir /lib
+@brief lib: low-level functionality.
+
+The "lib" directory contains low-level functionality.  In general, this
+code is not necessarily Tor-specific, but is instead possibly useful for
+other applications.
+
+The modules in `lib` are currently well-factored: each one depends
+only on lower-level modules.  You can see an up-to-date list of the
+modules, sorted from lowest to highest level, by running
+`./scripts/maint/practracker/includes.py --toposort`.
+
+As of this writing, the library modules are (from lowest to highest
+level):
+
+   - \refdir{lib/cc} -- Macros for managing the C compiler and
+     language.
+
+   - \refdir{lib/version} -- Holds the current version of Tor.
+
+   - \refdir{lib/testsupport} -- Helpers for making
+     test-only code, and test mocking support.
+
+   - \refdir{lib/defs} -- Lowest-level constants.
+
+   - \refdir{lib/subsys} -- Types used for declaring a
+     "subsystem". (_A subsystem is a module with support for initialization,
+     shutdown, configuration, and so on._)
+
+   - \refdir{lib/conf} -- For declaring configuration options.
+
+   - \refdir{lib/arch} -- For handling differences in CPU
+     architecture.
+
+   - \refdir{lib/err} -- Lowest-level error handling code.
+
+   - \refdir{lib/malloc} -- Memory management.
+     management.
+
+   - \refdir{lib/intmath} -- Integer mathematics.
+
+   - \refdir{lib/fdio} -- For
+      reading and writing n file descriptors.
+
+   - \refdir{lib/lock} -- Simple locking support.
+      (_Lower-level than the rest of the threading code._)
+
+   - \refdir{lib/ctime} -- Constant-time code to avoid
+     side-channels.
+
+   - \refdir{lib/string} -- Low-level string manipulation.
+
+   - \refdir{lib/wallclock} --
+     For inspecting and manipulating the current (UTC) time.
+
+   - \refdir{lib/osinfo} -- For inspecting the OS version
+     and capabilities.
+
+   - \refdir{lib/smartlist_core} -- The bare-bones
+     pieces of our dynamic array ("smartlist") implementation.
+
+   - \refdir{lib/log} -- Log messages to files, syslogs, etc.
+
+   - \refdir{lib/container} -- General purpose containers,
+     including dynamic arrays ("smartlists"), hashtables, bit arrays,
+     etc.
+
+   - \refdir{lib/trace} -- A general-purpose API
+     function-tracing functionality Tor.  (_Currently not much used._)
+
+   - \refdir{lib/thread} -- Mid-level Threading.
+
+   - \refdir{lib/term} -- Terminal manipulation
+     (like reading a password from the user).
+
+   - \refdir{lib/memarea} -- A fast
+     "arena" style allocator, where the data is freed all at once.
+
+   - \refdir{lib/encoding} -- Encoding
+     data in various formats, datatypes, and transformations.
+
+   - \refdir{lib/dispatch} -- A general-purpose in-process
+     message delivery system.
+
+   - \refdir{lib/sandbox} -- Our Linux seccomp2 sandbox
+     implementation.
+
+   - \refdir{lib/pubsub} -- A publish/subscribe message passing system.
+
+   - \refdir{lib/fs} -- Files, filenames, directories, etc.
+
+   - \refdir{lib/confmgt} -- Parse, encode, and manipulate onfiguration files.
+
+   - \refdir{lib/crypt_ops} -- Cryptographic operations.
+
+   - \refdir{lib/meminfo} -- Functions for inspecting our
+     memory usage, if the malloc implementation exposes that to us.
+
+   - \refdir{lib/time} -- Higher level time functions, including
+     fine-gained and monotonic timers.
+
+   - \refdir{lib/math} -- Floating-point mathematical utilities.
+
+   - \refdir{lib/buf} -- An efficient byte queue.
+
+   - \refdir{lib/net} -- Networking code, including address
+     manipulation, compatibility wrappers, etc.
+
+   - \refdir{lib/compress} -- Wraps several compression libraries.
+
+   - \refdir{lib/geoip} -- IP-to-country mapping.
+
+   - \refdir{lib/tls} -- TLS library wrappers.
+
+   - \refdir{lib/evloop} -- Low-level event-loop.
+
+   - \refdir{lib/process} -- Launch and manage subprocesses.
+
+### What belongs in lib?
+
+In general, if you can imagine some program wanting the functionality
+you're writing, even if that program had nothing to do with Tor, your
+functionality belongs in lib.
+
+If it falls into one of the existing "lib" categories, your
+functionality belongs in lib.
+
+If you are using platform-specific `ifdef`s to manage compatibility
+issues among platforms, you should probably consider whether you can
+put your code into lib.
+
diff --git a/src/lib/llharden/.may_include b/src/lib/llharden/.may_include
new file mode 100644
index 0000000000..038237dadf
--- /dev/null
+++ b/src/lib/llharden/.may_include
@@ -0,0 +1,3 @@
+lib/llharden/*.h
+lib/subsys/*.h
+orconfig.h
diff --git a/src/lib/llharden/include.am b/src/lib/llharden/include.am
new file mode 100644
index 0000000000..0a4788c7dc
--- /dev/null
+++ b/src/lib/llharden/include.am
@@ -0,0 +1,19 @@
+
+noinst_LIBRARIES += src/lib/libtor-llharden.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-llharden-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_llharden_a_SOURCES =		\
+	src/lib/llharden/winprocess_sys.c
+
+src_lib_libtor_llharden_testing_a_SOURCES = \
+	$(src_lib_libtor_llharden_a_SOURCES)
+src_lib_libtor_llharden_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_llharden_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=				\
+	src/lib/llharden/winprocess_sys.h
diff --git a/src/lib/llharden/lib_llharden.md b/src/lib/llharden/lib_llharden.md
new file mode 100644
index 0000000000..69e9af5327
--- /dev/null
+++ b/src/lib/llharden/lib_llharden.md
@@ -0,0 +1,6 @@
+@dir /lib/llharden
+@brief lib/llharden: low-level unconditional process hardening
+
+This module contains process hardening code that we want to run before any
+other code, including configuration.  It needs to be self-contained, since
+nothing else will be initialized at this point.
diff --git a/src/lib/llharden/winprocess_sys.c b/src/lib/llharden/winprocess_sys.c
new file mode 100644
index 0000000000..f2c88d8c75
--- /dev/null
+++ b/src/lib/llharden/winprocess_sys.c
@@ -0,0 +1,67 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.c
+ * \brief Subsystem object for windows process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/llharden/winprocess_sys.h"
+
+#include <stdbool.h>
+#include <stddef.h>
+
+#ifdef _WIN32
+#include <windows.h>
+
+#define WINPROCESS_SYS_ENABLED true
+
+static int
+subsys_winprocess_initialize(void)
+{
+#ifndef HeapEnableTerminationOnCorruption
+#define HeapEnableTerminationOnCorruption 1
+#endif
+
+  /* On heap corruption, just give up; don't try to play along. */
+  HeapSetInformation(NULL, HeapEnableTerminationOnCorruption, NULL, 0);
+
+  /* SetProcessDEPPolicy is only supported on 32-bit Windows.
+   * (On 64-bit Windows it always fails, and some compilers don't like the
+   * PSETDEP cast.)
+   * 32-bit Windows defines _WIN32.
+   * 64-bit Windows defines _WIN32 and _WIN64. */
+#ifndef _WIN64
+  /* Call SetProcessDEPPolicy to permanently enable DEP.
+     The function will not resolve on earlier versions of Windows,
+     and failure is not dangerous. */
+  HMODULE hMod = GetModuleHandleA("Kernel32.dll");
+  if (hMod) {
+    typedef BOOL (WINAPI *PSETDEP)(DWORD);
+    PSETDEP setdeppolicy = (PSETDEP)GetProcAddress(hMod,
+                           "SetProcessDEPPolicy");
+    if (setdeppolicy) {
+      /* PROCESS_DEP_ENABLE | PROCESS_DEP_DISABLE_ATL_THUNK_EMULATION */
+      setdeppolicy(3);
+    }
+  }
+#endif /* !defined(_WIN64) */
+
+  return 0;
+}
+#else /* !defined(_WIN32) */
+#define WINPROCESS_SYS_ENABLED false
+#define subsys_winprocess_initialize NULL
+#endif /* defined(_WIN32) */
+
+const subsys_fns_t sys_winprocess = {
+  .name = "winprocess",
+  SUBSYS_DECLARE_LOCATION(),
+  /* HeapEnableTerminationOnCorruption and setdeppolicy() are security
+   * features, we want them to run first. */
+  .level = -100,
+  .supported = WINPROCESS_SYS_ENABLED,
+  .initialize = subsys_winprocess_initialize,
+};
diff --git a/src/lib/llharden/winprocess_sys.h b/src/lib/llharden/winprocess_sys.h
new file mode 100644
index 0000000000..bece1b3da9
--- /dev/null
+++ b/src/lib/llharden/winprocess_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file winprocess_sys.h
+ * \brief Declare subsystem object for winprocess.c
+ **/
+
+#ifndef TOR_WINPROCESS_SYS_H
+#define TOR_WINPROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_winprocess;
+
+#endif /* !defined(TOR_WINPROCESS_SYS_H) */
diff --git a/src/lib/lock/compat_mutex.c b/src/lib/lock/compat_mutex.c
index 4ad5929715..b0084a3484 100644
--- a/src/lib/lock/compat_mutex.c
+++ b/src/lib/lock/compat_mutex.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,15 @@ tor_mutex_new_nonrecursive(void)
   tor_mutex_init_nonrecursive(m);
   return m;
 }
-/** Release all storage and system resources held by <b>m</b>. */
+/** Release all storage and system resources held by <b>m</b>.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_free_(tor_mutex_t *m)
 {
diff --git a/src/lib/lock/compat_mutex.h b/src/lib/lock/compat_mutex.h
index b63ce24024..5631993cc4 100644
--- a/src/lib/lock/compat_mutex.h
+++ b/src/lib/lock/compat_mutex.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,7 +48,7 @@ typedef struct tor_mutex_t {
 #else
   /** No-threads only: Dummy variable so that tor_mutex_t takes up space. */
   int _unused;
-#endif /* defined(USE_WIN32_MUTEX) || ... */
+#endif /* defined(USE_WIN32_THREADS) || ... */
 } tor_mutex_t;
 
 tor_mutex_t *tor_mutex_new(void);
@@ -58,6 +58,11 @@ void tor_mutex_init_nonrecursive(tor_mutex_t *m);
 void tor_mutex_acquire(tor_mutex_t *m);
 void tor_mutex_release(tor_mutex_t *m);
 void tor_mutex_free_(tor_mutex_t *m);
+/**
+ * @copydoc tor_mutex_free_
+ *
+ * Additionally, set the pointer <b>m</b> to NULL.
+ **/
 #define tor_mutex_free(m) FREE_AND_NULL(tor_mutex_t, tor_mutex_free_, (m))
 void tor_mutex_uninit(tor_mutex_t *m);
 
diff --git a/src/lib/lock/compat_mutex_pthreads.c b/src/lib/lock/compat_mutex_pthreads.c
index ee5f520cd0..ac83c42a47 100644
--- a/src/lib/lock/compat_mutex_pthreads.c
+++ b/src/lib/lock/compat_mutex_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,8 +17,14 @@
  * "recursive" mutexes (i.e., once we can re-lock if we're already holding
  * them.) */
 static pthread_mutexattr_t attr_recursive;
+/**
+ * True iff <b>attr_recursive</b> has been initialized.
+ **/
 static int attr_initialized = 0;
 
+/**
+ * Initialize the locking module, if it is not already initialized.
+ **/
 void
 tor_locking_init(void)
 {
@@ -88,12 +94,26 @@ tor_mutex_release(tor_mutex_t *m)
 }
 /** Clean up the mutex <b>m</b> so that it no longer uses any system
  * resources.  Does not free <b>m</b>.  This function must only be called on
- * mutexes from tor_mutex_init(). */
+ * mutexes from tor_mutex_init().
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 tor_mutex_uninit(tor_mutex_t *m)
 {
   int err;
   raw_assert(m);
+  /* If the mutex is already locked, wait until after it is unlocked to destroy
+   * it. Locking and releasing the mutex makes undefined behaviour less likely,
+   * but does not prevent it. Another thread can lock the mutex between release
+   * and destroy. */
+  tor_mutex_acquire(m);
+  tor_mutex_release(m);
   err = pthread_mutex_destroy(&m->mutex);
   if (PREDICT_UNLIKELY(err)) {
     // LCOV_EXCL_START
diff --git a/src/lib/lock/compat_mutex_winthreads.c b/src/lib/lock/compat_mutex_winthreads.c
index b0f5999e42..5fe6870a93 100644
--- a/src/lib/lock/compat_mutex_winthreads.c
+++ b/src/lib/lock/compat_mutex_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/lock/include.am b/src/lib/lock/include.am
index 4e6f444347..1475b9911b 100644
--- a/src/lib/lock/include.am
+++ b/src/lib/lock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-lock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_lock_a_SOURCES =			\
 	src/lib/lock/compat_mutex.c
 
@@ -20,5 +21,6 @@ src_lib_libtor_lock_testing_a_SOURCES = \
 src_lib_libtor_lock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_lock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/lock/compat_mutex.h
diff --git a/src/lib/lock/lib_lock.md b/src/lib/lock/lib_lock.md
new file mode 100644
index 0000000000..6f6727bfc2
--- /dev/null
+++ b/src/lib/lock/lib_lock.md
@@ -0,0 +1,6 @@
+@dir /lib/lock
+@brief lib/lock: Simple locking support.
+
+This module is more low-level than the rest of the threading code, since it
+is needed by more intermediate-level modules.
+
diff --git a/src/lib/log/.may_include b/src/lib/log/.may_include
index 852173aab3..54d96324db 100644
--- a/src/lib/log/.may_include
+++ b/src/lib/log/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/cc/*.h
+lib/defs/*.h
 lib/smartlist_core/*.h
 lib/err/*.h
 lib/fdio/*.h
@@ -9,7 +10,7 @@ lib/lock/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
+lib/version/*.h
 lib/wallclock/*.h
-
-micro-revision.i
\ No newline at end of file
diff --git a/src/lib/log/escape.c b/src/lib/log/escape.c
index 6ca01c6963..88ca52a78c 100644
--- a/src/lib/log/escape.c
+++ b/src/lib/log/escape.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/log/escape.h b/src/lib/log/escape.h
index 2f726186c5..0d1eaef4c1 100644
--- a/src/lib/log/escape.h
+++ b/src/lib/log/escape.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ char *esc_for_log(const char *string) ATTR_MALLOC;
 char *esc_for_log_len(const char *chars, size_t n) ATTR_MALLOC;
 const char *escaped(const char *string);
 
-#endif /* !defined(TOR_TORLOG_H) */
+#endif /* !defined(TOR_ESCAPE_H) */
diff --git a/src/lib/log/include.am b/src/lib/log/include.am
index 4a6c9b3686..5b9f7113ba 100644
--- a/src/lib/log/include.am
+++ b/src/lib/log/include.am
@@ -5,11 +5,12 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-log-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_log_a_SOURCES =			\
 	src/lib/log/escape.c			\
-        src/lib/log/git_revision.c              \
 	src/lib/log/ratelim.c			\
 	src/lib/log/log.c			\
+	src/lib/log/log_sys.c			\
 	src/lib/log/util_bug.c
 
 if WIN32
@@ -21,16 +22,11 @@ src_lib_libtor_log_testing_a_SOURCES = \
 src_lib_libtor_log_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_log_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
-# Declare that these object files depend on micro-revision.i.  Without this
-# rule, we could try to build them before micro-revision.i was created.
-src/lib/log/git_revision.$(OBJEXT) \
-  src/lib/log/src_lib_libtor_log_testing_a-git_revision.$(OBJEXT): \
-	micro-revision.i
-
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/log/escape.h				\
-        src/lib/log/git_revision.h                      \
 	src/lib/log/ratelim.h				\
 	src/lib/log/log.h				\
+	src/lib/log/log_sys.h				\
 	src/lib/log/util_bug.h				\
 	src/lib/log/win32err.h
diff --git a/src/lib/log/lib_log.md b/src/lib/log/lib_log.md
new file mode 100644
index 0000000000..8740d6a02f
--- /dev/null
+++ b/src/lib/log/lib_log.md
@@ -0,0 +1,10 @@
+@dir /lib/log
+@brief lib/log: Log messages to files, syslogs, etc.
+
+You can think of this as the logical "midpoint" of the
+\refdir{lib} code": much of the higher-level code is higher-level
+_because_ it uses the logging module, and much of the lower-level code is
+specifically written to avoid having to log, because the logging module
+depends on it.
+
+
diff --git a/src/lib/log/log.c b/src/lib/log/log.c
index a9ad38fb25..9ee87c0668 100644
--- a/src/lib/log/log.c
+++ b/src/lib/log/log.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -32,7 +32,8 @@
 
 #define LOG_PRIVATE
 #include "lib/log/log.h"
-#include "lib/log/git_revision.h"
+#include "lib/log/log_sys.h"
+#include "lib/version/git_revision.h"
 #include "lib/log/ratelim.h"
 #include "lib/lock/compat_mutex.h"
 #include "lib/smartlist_core/smartlist_core.h"
@@ -48,15 +49,12 @@
 #include "lib/wallclock/approx_time.h"
 #include "lib/wallclock/time_to_tm.h"
 #include "lib/fdio/fdio.h"
+#include "lib/cc/ctassert.h"
 
 #ifdef HAVE_ANDROID_LOG_H
 #include <android/log.h>
 #endif // HAVE_ANDROID_LOG_H.
 
-/** Given a severity, yields an index into log_severity_list_t.masks to use
- * for that severity. */
-#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
-
 /** @{ */
 /** The string we stick at the end of a log message when it is too long,
  * and its length. */
@@ -153,7 +151,7 @@ severity_to_android_log_priority(int severity)
       // LCOV_EXCL_STOP
   }
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** A mutex to guard changes to logfiles and logging. */
 static tor_mutex_t log_mutex;
@@ -223,6 +221,7 @@ int log_global_min_severity_ = LOG_NOTICE;
 
 static void delete_log(logfile_t *victim);
 static void close_log(logfile_t *victim);
+static void close_log_sigsafe(logfile_t *victim);
 
 static char *domain_to_string(log_domain_mask_t domain,
                              char *buf, size_t buflen);
@@ -277,8 +276,8 @@ static int log_time_granularity = 1;
 
 /** Define log time granularity for all logs to be <b>granularity_msec</b>
  * milliseconds. */
-void
-set_log_time_granularity(int granularity_msec)
+MOCK_IMPL(void,
+set_log_time_granularity,(int granularity_msec))
 {
   log_time_granularity = granularity_msec;
   tor_log_sigsafe_err_set_granularity(granularity_msec);
@@ -524,7 +523,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
      * pass them, and some very old ones do not detect overflow so well.
      * Regrettably, they call their maximum line length MAXLINE. */
 #if MAXLINE < 64
-#warn "MAXLINE is a very low number; it might not be from syslog.h after all"
+#warning "MAXLINE is very low; it might not be from syslog.h."
 #endif
     char *m = msg_after_prefix;
     if (msg_len >= MAXLINE)
@@ -533,7 +532,7 @@ logfile_deliver(logfile_t *lf, const char *buf, size_t msg_len,
     if (m != msg_after_prefix) {
       tor_free(m);
     }
-#else /* !(defined(MAXLINE)) */
+#else /* !defined(MAXLINE) */
     /* We have syslog but not MAXLINE.  That's promising! */
     syslog(severity, "%s", msg_after_prefix);
 #endif /* defined(MAXLINE) */
@@ -585,8 +584,7 @@ logv,(int severity, log_domain_mask_t domain, const char *funcname,
   /* check that severity is sane.  Overrunning the masks array leads to
    * interesting and hard to diagnose effects */
   raw_assert(severity >= LOG_ERR && severity <= LOG_DEBUG);
-  /* check that we've initialised the log mutex before we try to lock it */
-  raw_assert(log_mutex_initialized);
+
   LOCK_LOGS();
 
   if ((! (domain & LD_NOCB)) && pending_cb_messages
@@ -630,6 +628,10 @@ void
 tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
 {
   va_list ap;
+
+  /* check that domain is composed of known domains and flags */
+  raw_assert((domain & (LD_ALL_DOMAINS|LD_ALL_FLAGS)) == domain);
+
   if (severity > log_global_min_severity_)
     return;
   va_start(ap,format);
@@ -663,18 +665,21 @@ tor_log_update_sigsafe_err_fds(void)
   const logfile_t *lf;
   int found_real_stderr = 0;
 
+  /* The fds are the file descriptors of tor's stdout, stderr, and file
+   * logs. The log and err modules flush these fds during their shutdowns. */
   int fds[TOR_SIGSAFE_LOG_MAX_FDS];
   int n_fds;
 
   LOCK_LOGS();
   /* Reserve the first one for stderr. This is safe because when we daemonize,
-   * we dup2 /dev/null to stderr, */
+   * we dup2 /dev/null to stderr. */
   fds[0] = STDERR_FILENO;
   n_fds = 1;
 
   for (lf = logfiles; lf; lf = lf->next) {
-     /* Don't try callback to the control port, or syslogs: We can't
-      * do them from a signal handler. Don't try stdout: we always do stderr.
+     /* Don't try callback to the control port, syslogs, android logs, or any
+      * other non-file descriptor log: We can't call arbitrary functions from a
+      * signal handler.
       */
     if (lf->is_temporary || logfile_is_external(lf)
         || lf->seems_dead || lf->fd < 0)
@@ -683,10 +688,12 @@ tor_log_update_sigsafe_err_fds(void)
         (LD_BUG|LD_GENERAL)) {
       if (lf->fd == STDERR_FILENO)
         found_real_stderr = 1;
-      /* Avoid duplicates */
+      /* Avoid duplicates by checking the log module fd against fds */
       if (int_array_contains(fds, n_fds, lf->fd))
         continue;
-      fds[n_fds++] = lf->fd;
+      /* Update fds using the log module's fd */
+      fds[n_fds] = lf->fd;
+      n_fds++;
       if (n_fds == TOR_SIGSAFE_LOG_MAX_FDS)
         break;
     }
@@ -694,9 +701,13 @@ tor_log_update_sigsafe_err_fds(void)
 
   if (!found_real_stderr &&
       int_array_contains(fds, n_fds, STDOUT_FILENO)) {
-    /* Don't use a virtual stderr when we're also logging to stdout. */
+    /* Don't use a virtual stderr when we're also logging to stdout.
+     * If we reached max_fds logs, we'll now have (max_fds - 1) logs.
+     * That's ok, max_fds is large enough that most tor instances don't exceed
+     * it. */
     raw_assert(n_fds >= 2); /* Don't tor_assert inside log fns */
-    fds[0] = fds[--n_fds];
+    --n_fds;
+    fds[0] = fds[n_fds];
   }
 
   UNLOCK_LOGS();
@@ -727,7 +738,7 @@ tor_log_get_logfile_names(smartlist_t *out)
 
 /** Implementation of the log_fn backend, used when we have
  * variadic macros. All arguments are as for log_fn, except for
- * <b>fn</b>, which is the name of the calling functions. */
+ * <b>fn</b>, which is the name of the calling function. */
 void
 log_fn_(int severity, log_domain_mask_t domain, const char *fn,
         const char *format, ...)
@@ -804,16 +815,42 @@ logs_free_all(void)
   }
 
   /* We _could_ destroy the log mutex here, but that would screw up any logs
-   * that happened between here and the end of execution. */
+   * that happened between here and the end of execution.
+   * If tor is re-initialized, log_mutex_initialized will still be 1. So we
+   * won't trigger any undefined behaviour by trying to re-initialize the
+   * log mutex. */
+}
+
+/** Flush the signal-safe log files.
+ *
+ * This function is safe to call from a signal handler. It is currenly called
+ * by the BUG() macros, when terminating the process on an abnormal condition.
+ */
+void
+logs_flush_sigsafe(void)
+{
+  /* If we don't have fsync() in unistd.h, we can't flush the logs. */
+#ifdef HAVE_FSYNC
+  logfile_t *victim, *next;
+  /* We can't LOCK_LOGS() in a signal handler, because it may call
+   * signal-unsafe functions. And we can't deallocate memory, either. */
+  next = logfiles;
+  logfiles = NULL;
+  while (next) {
+    victim = next;
+    next = next->next;
+    if (victim->needs_close) {
+      /* We can't do anything useful if the flush fails. */
+      (void)fsync(victim->fd);
+    }
+  }
+#endif /* defined(HAVE_FSYNC) */
 }
 
 /** Remove and free the log entry <b>victim</b> from the linked-list
  * logfiles (it is probably present, but it might not be due to thread
  * racing issues). After this function is called, the caller shouldn't
  * refer to <b>victim</b> anymore.
- *
- * Long-term, we need to do something about races in the log subsystem
- * in general. See bug 222 for more details.
  */
 static void
 delete_log(logfile_t *victim)
@@ -833,13 +870,26 @@ delete_log(logfile_t *victim)
 }
 
 /** Helper: release system resources (but not memory) held by a single
- * logfile_t. */
+ * signal-safe logfile_t. If the log's resources can not be released in
+ * a signal handler, does nothing. */
 static void
-close_log(logfile_t *victim)
+close_log_sigsafe(logfile_t *victim)
 {
   if (victim->needs_close && victim->fd >= 0) {
+    /* We can't do anything useful here if close() fails: we're shutting
+     * down logging, and the err module only does fatal errors. */
     close(victim->fd);
     victim->fd = -1;
+  }
+}
+
+/** Helper: release system resources (but not memory) held by a single
+ * logfile_t. */
+static void
+close_log(logfile_t *victim)
+{
+  if (victim->needs_close) {
+    close_log_sigsafe(victim);
   } else if (victim->is_syslog) {
 #ifdef HAVE_SYSLOG_H
     if (--syslog_count == 0) {
@@ -863,15 +913,15 @@ set_log_severity_config(int loglevelMin, int loglevelMax,
   raw_assert(loglevelMax >= LOG_ERR && loglevelMax <= LOG_DEBUG);
   memset(severity_out, 0, sizeof(log_severity_list_t));
   for (i = loglevelMin; i >= loglevelMax; --i) {
-    severity_out->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+    severity_out->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
 }
 
 /** Add a log handler named <b>name</b> to send all messages in <b>severity</b>
  * to <b>fd</b>. Copies <b>severity</b>. Helper: does no locking. */
-static void
-add_stream_log_impl(const log_severity_list_t *severity,
-                    const char *name, int fd)
+MOCK_IMPL(STATIC void,
+add_stream_log_impl,(const log_severity_list_t *severity,
+                     const char *name, int fd))
 {
   logfile_t *lf;
   lf = tor_malloc_zero(sizeof(logfile_t));
@@ -927,18 +977,16 @@ logs_set_domain_logging(int enabled)
   UNLOCK_LOGS();
 }
 
-/** Add a log handler to receive messages during startup (before the real
- * logs are initialized).
+/** Add a log handler to accept messages when no other log is configured.
  */
 void
-add_temp_log(int min_severity)
+add_default_log(int min_severity)
 {
   log_severity_list_t *s = tor_malloc_zero(sizeof(log_severity_list_t));
   set_log_severity_config(min_severity, LOG_ERR, s);
   LOCK_LOGS();
-  add_stream_log_impl(s, "<temp>", fileno(stdout));
+  add_stream_log_impl(s, "<default>", fileno(stdout));
   tor_free(s);
-  logfiles->is_temporary = 1;
   UNLOCK_LOGS();
 }
 
@@ -1020,7 +1068,7 @@ flush_pending_log_callbacks(void)
   do {
     SMARTLIST_FOREACH_BEGIN(messages, pending_log_message_t *, msg) {
       const int severity = msg->severity;
-      const int domain = msg->domain;
+      const log_domain_mask_t domain = msg->domain;
       for (lf = logfiles; lf; lf = lf->next) {
         if (! lf->callback || lf->seems_dead ||
             ! (lf->severities->masks[SEVERITY_MASK_IDX(severity)] & domain)) {
@@ -1081,8 +1129,7 @@ flush_log_messages_from_startup(void)
   UNLOCK_LOGS();
 }
 
-/** Close any log handlers added by add_temp_log() or marked by
- * mark_logs_temp(). */
+/** Close any log handlers marked by mark_logs_temp(). */
 void
 close_temp_logs(void)
 {
@@ -1134,10 +1181,10 @@ mark_logs_temp(void)
  * opening the logfile failed, -1 is returned and errno is set appropriately
  * (by open(2)).  Takes ownership of fd.
  */
-int
-add_file_log(const log_severity_list_t *severity,
-             const char *filename,
-             int fd)
+MOCK_IMPL(int,
+add_file_log,(const log_severity_list_t *severity,
+              const char *filename,
+              int fd))
 {
   logfile_t *lf;
 
@@ -1231,7 +1278,7 @@ add_android_log(const log_severity_list_t *severity,
   UNLOCK_LOGS();
   return 0;
 }
-#endif // HAVE_ANDROID_LOG_H.
+#endif /* defined(HAVE_ANDROID_LOG_H) */
 
 /** If <b>level</b> is a valid log severity, return the corresponding
  * numeric value.  Otherwise, return -1. */
@@ -1267,9 +1314,16 @@ static const char *domain_list[] = {
   "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM",
   "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV",
   "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL",
-  "SCHED", "GUARD", "CONSDIFF", "DOS", NULL
+  "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", "BTRACK", "MESG",
+  NULL
 };
 
+CTASSERT(ARRAY_LENGTH(domain_list) == N_LOGGING_DOMAINS + 1);
+
+CTASSERT(HIGHEST_RESERVED_LD_DOMAIN_ < LD_ALL_DOMAINS);
+CTASSERT(LD_ALL_DOMAINS < LOWEST_RESERVED_LD_FLAG_);
+CTASSERT(LOWEST_RESERVED_LD_FLAG_ < LD_ALL_FLAGS);
+
 /** Return a bitmask for the log domain for which <b>domain</b> is the name,
  * or 0 if there is no such name. */
 static log_domain_mask_t
@@ -1278,7 +1332,7 @@ parse_log_domain(const char *domain)
   int i;
   for (i=0; domain_list[i]; ++i) {
     if (!strcasecmp(domain, domain_list[i]))
-      return (1u<<i);
+      return (UINT64_C(1)<<i);
   }
   return 0;
 }
@@ -1350,7 +1404,7 @@ parse_log_severity_config(const char **cfg_ptr,
     const char *dash, *space;
     char *sev_lo, *sev_hi;
     int low, high, i;
-    log_domain_mask_t domains = ~0u;
+    log_domain_mask_t domains = LD_ALL_DOMAINS;
 
     if (*cfg == '[') {
       int err = 0;
@@ -1368,9 +1422,9 @@ parse_log_severity_config(const char **cfg_ptr,
       tor_free(domains_str);
       SMARTLIST_FOREACH_BEGIN(domains_list, const char *, domain) {
             if (!strcmp(domain, "*")) {
-              domains = ~0u;
+              domains = LD_ALL_DOMAINS;
             } else {
-              int d;
+              log_domain_mask_t d;
               int negate=0;
               if (*domain == '~') {
                 negate = 1;
@@ -1464,7 +1518,7 @@ switch_logs_debug(void)
   LOCK_LOGS();
   for (lf = logfiles; lf; lf=lf->next) {
     for (i = LOG_DEBUG; i >= LOG_ERR; --i)
-      lf->severities->masks[SEVERITY_MASK_IDX(i)] = ~0u;
+      lf->severities->masks[SEVERITY_MASK_IDX(i)] = LD_ALL_DOMAINS;
   }
   log_global_min_severity_ = get_min_log_level();
   UNLOCK_LOGS();
diff --git a/src/lib/log/log.h b/src/lib/log/log.h
index d7a5070610..aafbf9be2f 100644
--- a/src/lib/log/log.h
+++ b/src/lib/log/log.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001, Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,20 +11,24 @@
  **/
 
 #ifndef TOR_TORLOG_H
+#define TOR_TORLOG_H
 
 #include <stdarg.h>
 #include "lib/cc/torint.h"
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/logging_types.h"
 #include "lib/testsupport/testsupport.h"
 
 #ifdef HAVE_SYSLOG_H
 #include <syslog.h>
 #define LOG_WARN LOG_WARNING
 #if LOG_DEBUG < LOG_ERR
+#ifndef COCCI
 #error "Your syslog.h thinks high numbers are more important.  " \
        "We aren't prepared to deal with that."
 #endif
-#else /* !(defined(HAVE_SYSLOG_H)) */
+#endif /* LOG_DEBUG < LOG_ERR */
+#else /* !defined(HAVE_SYSLOG_H) */
 /* Note: Syslog's logging code refers to priorities, with 0 being the most
  * important.  Thus, all our comparisons needed to be reversed when we added
  * syslog support.
@@ -55,75 +59,92 @@
 /* Logging domains */
 
 /** Catch-all for miscellaneous events and fatal errors. */
-#define LD_GENERAL  (1u<<0)
+#define LD_GENERAL  (UINT64_C(1)<<0)
 /** The cryptography subsystem. */
-#define LD_CRYPTO   (1u<<1)
+#define LD_CRYPTO   (UINT64_C(1)<<1)
 /** Networking. */
-#define LD_NET      (1u<<2)
+#define LD_NET      (UINT64_C(1)<<2)
 /** Parsing and acting on our configuration. */
-#define LD_CONFIG   (1u<<3)
+#define LD_CONFIG   (UINT64_C(1)<<3)
 /** Reading and writing from the filesystem. */
-#define LD_FS       (1u<<4)
+#define LD_FS       (UINT64_C(1)<<4)
 /** Other servers' (non)compliance with the Tor protocol. */
-#define LD_PROTOCOL (1u<<5)
+#define LD_PROTOCOL (UINT64_C(1)<<5)
 /** Memory management. */
-#define LD_MM       (1u<<6)
+#define LD_MM       (UINT64_C(1)<<6)
 /** HTTP implementation. */
-#define LD_HTTP     (1u<<7)
+#define LD_HTTP     (UINT64_C(1)<<7)
 /** Application (socks) requests. */
-#define LD_APP      (1u<<8)
+#define LD_APP      (UINT64_C(1)<<8)
 /** Communication via the controller protocol. */
-#define LD_CONTROL  (1u<<9)
+#define LD_CONTROL  (UINT64_C(1)<<9)
 /** Building, using, and managing circuits. */
-#define LD_CIRC     (1u<<10)
+#define LD_CIRC     (UINT64_C(1)<<10)
 /** Hidden services. */
-#define LD_REND     (1u<<11)
+#define LD_REND     (UINT64_C(1)<<11)
 /** Internal errors in this Tor process. */
-#define LD_BUG      (1u<<12)
+#define LD_BUG      (UINT64_C(1)<<12)
 /** Learning and using information about Tor servers. */
-#define LD_DIR      (1u<<13)
+#define LD_DIR      (UINT64_C(1)<<13)
 /** Learning and using information about Tor servers. */
-#define LD_DIRSERV  (1u<<14)
+#define LD_DIRSERV  (UINT64_C(1)<<14)
 /** Onion routing protocol. */
-#define LD_OR       (1u<<15)
+#define LD_OR       (UINT64_C(1)<<15)
 /** Generic edge-connection functionality. */
-#define LD_EDGE     (1u<<16)
+#define LD_EDGE     (UINT64_C(1)<<16)
 #define LD_EXIT     LD_EDGE
 /** Bandwidth accounting. */
-#define LD_ACCT     (1u<<17)
+#define LD_ACCT     (UINT64_C(1)<<17)
 /** Router history */
-#define LD_HIST     (1u<<18)
+#define LD_HIST     (UINT64_C(1)<<18)
 /** OR handshaking */
-#define LD_HANDSHAKE (1u<<19)
+#define LD_HANDSHAKE (UINT64_C(1)<<19)
 /** Heartbeat messages */
-#define LD_HEARTBEAT (1u<<20)
+#define LD_HEARTBEAT (UINT64_C(1)<<20)
 /** Abstract channel_t code */
-#define LD_CHANNEL   (1u<<21)
+#define LD_CHANNEL   (UINT64_C(1)<<21)
 /** Scheduler */
-#define LD_SCHED     (1u<<22)
+#define LD_SCHED     (UINT64_C(1)<<22)
 /** Guard nodes */
-#define LD_GUARD     (1u<<23)
+#define LD_GUARD     (UINT64_C(1)<<23)
 /** Generation and application of consensus diffs. */
-#define LD_CONSDIFF  (1u<<24)
+#define LD_CONSDIFF  (UINT64_C(1)<<24)
 /** Denial of Service mitigation. */
-#define LD_DOS       (1u<<25)
-/** Number of logging domains in the code. */
-#define N_LOGGING_DOMAINS 26
+#define LD_DOS       (UINT64_C(1)<<25)
+/** Processes */
+#define LD_PROCESS   (UINT64_C(1)<<26)
+/** Pluggable Transports. */
+#define LD_PT        (UINT64_C(1)<<27)
+/** Bootstrap tracker. */
+#define LD_BTRACK    (UINT64_C(1)<<28)
+/** Message-passing backend. */
+#define LD_MESG      (UINT64_C(1)<<29)
 
-/** This log message is not safe to send to a callback-based logger
- * immediately.  Used as a flag, not a log domain. */
-#define LD_NOCB (1u<<31)
-/** This log message should not include a function name, even if it otherwise
- * would. Used as a flag, not a log domain. */
-#define LD_NOFUNCNAME (1u<<30)
+/** The number of log domains. */
+#define N_LOGGING_DOMAINS 30
+/** The highest log domain */
+#define HIGHEST_RESERVED_LD_DOMAIN_ (UINT64_C(1)<<(N_LOGGING_DOMAINS - 1))
+/** All log domains. */
+#define LD_ALL_DOMAINS ((~(UINT64_C(0)))>>(64 - N_LOGGING_DOMAINS))
+
+/** The number of log flags. */
+#define N_LOGGING_FLAGS 3
+/** First bit that is reserved in log_domain_mask_t for non-domain flags. */
+#define LOWEST_RESERVED_LD_FLAG_ (UINT64_C(1)<<(64 - N_LOGGING_FLAGS))
+/** All log flags. */
+#define LD_ALL_FLAGS ((~(UINT64_C(0)))<<(64 - N_LOGGING_FLAGS))
 
 #ifdef TOR_UNIT_TESTS
 /** This log message should not be intercepted by mock_saving_logv */
-#define LD_NO_MOCK (1u<<29)
+#define LD_NO_MOCK (UINT64_C(1)<<61)
 #endif
 
-/** Mask of zero or more log domains, OR'd together. */
-typedef uint32_t log_domain_mask_t;
+/** This log message is not safe to send to a callback-based logger
+ * immediately.  Used as a flag, not a log domain. */
+#define LD_NOCB (UINT64_C(1)<<62)
+/** This log message should not include a function name, even if it otherwise
+ * would. Used as a flag, not a log domain. */
+#define LD_NOFUNCNAME (UINT64_C(1)<<63)
 
 /** Configures which severities are logged for each logging domain for a given
  * log target. */
@@ -134,7 +155,8 @@ typedef struct log_severity_list_t {
 } log_severity_list_t;
 
 /** Callback type used for add_callback_log. */
-typedef void (*log_callback)(int severity, uint32_t domain, const char *msg);
+typedef void (*log_callback)(int severity, log_domain_mask_t domain,
+                             const char *msg);
 
 void init_logging(int disable_startup_queue);
 int parse_log_level(const char *level);
@@ -143,11 +165,11 @@ int parse_log_severity_config(const char **cfg,
                               log_severity_list_t *severity_out);
 void set_log_severity_config(int minSeverity, int maxSeverity,
                              log_severity_list_t *severity_out);
-void add_stream_log(const log_severity_list_t *severity, const char *name,
-                    int fd);
-int add_file_log(const log_severity_list_t *severity,
-                 const char *filename,
-                 int fd);
+void add_stream_log(const log_severity_list_t *severity,
+                    const char *name, int fd);
+MOCK_DECL(int, add_file_log,(const log_severity_list_t *severity,
+                             const char *filename,
+                             int fd));
 
 #ifdef HAVE_SYSLOG_H
 int add_syslog_log(const log_severity_list_t *severity,
@@ -164,7 +186,8 @@ void logs_set_domain_logging(int enabled);
 int get_min_log_level(void);
 void switch_logs_debug(void);
 void logs_free_all(void);
-void add_temp_log(int min_severity);
+void logs_flush_sigsafe(void);
+void add_default_log(int min_severity);
 void close_temp_logs(void);
 void rollback_log_changes(void);
 void mark_logs_temp(void);
@@ -173,7 +196,7 @@ void change_callback_log_severity(int loglevelMin, int loglevelMax,
 void flush_pending_log_callbacks(void);
 void flush_log_messages_from_startup(void);
 void log_set_application_name(const char *name);
-void set_log_time_granularity(int granularity_msec);
+MOCK_DECL(void, set_log_time_granularity,(int granularity_msec));
 void truncate_logs(void);
 
 void tor_log(int severity, log_domain_mask_t domain, const char *format, ...)
@@ -186,6 +209,21 @@ void tor_log_get_logfile_names(struct smartlist_t *out);
 
 extern int log_global_min_severity_;
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we try to avoid our log_debug optimization, since it
+ * can have weird effects on internal macro coverage. */
+#define debug_logging_enabled() (1)
+#else
+static inline bool debug_logging_enabled(void);
+/**
+ * Return true iff debug logging is enabled for at least one domain.
+ */
+static inline bool debug_logging_enabled(void)
+{
+  return PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG);
+}
+#endif /* defined(TOR_COVERAGE) */
+
 void log_fn_(int severity, log_domain_mask_t domain,
              const char *funcname, const char *format, ...)
   CHECK_PRINTF(4,5);
@@ -215,8 +253,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
   log_fn_ratelim_(ratelim, severity, domain, __FUNCTION__, args)
 #define log_debug(domain, args...)                                      \
   STMT_BEGIN                                                            \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))        \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);            \
+    if (debug_logging_enabled())                                        \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args);                   \
   STMT_END
 #define log_info(domain, args...)                           \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args)
@@ -233,8 +271,8 @@ void tor_log_string(int severity, log_domain_mask_t domain,
 
 #define log_debug(domain, args, ...)                                        \
   STMT_BEGIN                                                                \
-    if (PREDICT_UNLIKELY(log_global_min_severity_ == LOG_DEBUG))            \
-      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__); \
+    if (debug_logging_enabled())                                            \
+      log_fn_(LOG_DEBUG, domain, __FUNCTION__, args, ##__VA_ARGS__);        \
   STMT_END
 #define log_info(domain, args,...)                                      \
   log_fn_(LOG_INFO, domain, __FUNCTION__, args, ##__VA_ARGS__)
@@ -270,7 +308,14 @@ extern const log_domain_mask_t LD_GENERAL_;
 MOCK_DECL(STATIC void, logv, (int severity, log_domain_mask_t domain,
     const char *funcname, const char *suffix, const char *format,
     va_list ap) CHECK_PRINTF(5,0));
+MOCK_DECL(STATIC void, add_stream_log_impl,(
+         const log_severity_list_t *severity, const char *name, int fd));
+#endif /* defined(LOG_PRIVATE) */
+
+#if defined(LOG_PRIVATE) || defined(TOR_UNIT_TESTS)
+/** Given a severity, yields an index into log_severity_list_t.masks to use
+ * for that severity. */
+#define SEVERITY_MASK_IDX(sev) ((sev) - LOG_ERR)
 #endif
 
-# define TOR_TORLOG_H
 #endif /* !defined(TOR_TORLOG_H) */
diff --git a/src/lib/log/log_sys.c b/src/lib/log/log_sys.c
new file mode 100644
index 0000000000..021c05d3e6
--- /dev/null
+++ b/src/lib/log/log_sys.c
@@ -0,0 +1,38 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.c
+ * \brief Setup and tear down the logging module.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/log/escape.h"
+#include "lib/log/log.h"
+#include "lib/log/log_sys.h"
+
+static int
+subsys_logging_initialize(void)
+{
+  init_logging(0);
+  return 0;
+}
+
+static void
+subsys_logging_shutdown(void)
+{
+  logs_free_all();
+  escaped(NULL);
+}
+
+const subsys_fns_t sys_logging = {
+  .name = "log",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  /* Logging depends on threads, approx time, raw logging, and security.
+   * Most other lib modules depend on logging. */
+  .level = -90,
+  .initialize = subsys_logging_initialize,
+  .shutdown = subsys_logging_shutdown,
+};
diff --git a/src/lib/log/log_sys.h b/src/lib/log/log_sys.h
new file mode 100644
index 0000000000..523c2e5008
--- /dev/null
+++ b/src/lib/log/log_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file log_sys.h
+ * \brief Declare subsystem object for the logging module.
+ **/
+
+#ifndef TOR_LOG_SYS_H
+#define TOR_LOG_SYS_H
+
+extern const struct subsys_fns_t sys_logging;
+
+#endif /* !defined(TOR_LOG_SYS_H) */
diff --git a/src/lib/log/ratelim.c b/src/lib/log/ratelim.c
index 5eec742aa7..ac401fb398 100644
--- a/src/lib/log/ratelim.c
+++ b/src/lib/log/ratelim.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/log/ratelim.h b/src/lib/log/ratelim.h
index 48edd7c849..e9b55d40dc 100644
--- a/src/lib/log/ratelim.h
+++ b/src/lib/log/ratelim.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -45,9 +45,11 @@ typedef struct ratelim_t {
   int n_calls_since_last_time;
 } ratelim_t;
 
+#ifndef COCCI
 #define RATELIM_INIT(r) { (r), 0, 0 }
+#endif
 #define RATELIM_TOOMANY (16*1000*1000)
 
 char *rate_limit_log(ratelim_t *lim, time_t now);
 
-#endif
+#endif /* !defined(TOR_RATELIM_H) */
diff --git a/src/lib/log/util_bug.c b/src/lib/log/util_bug.c
index c65a91ae9e..d698ddd8a6 100644
--- a/src/lib/log/util_bug.c
+++ b/src/lib/log/util_bug.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,7 @@
 #include "lib/log/util_bug.h"
 #include "lib/log/log.h"
 #include "lib/err/backtrace.h"
+#include "lib/err/torerr.h"
 #ifdef TOR_UNIT_TESTS
 #include "lib/smartlist_core/smartlist_core.h"
 #include "lib/smartlist_core/smartlist_foreach.h"
@@ -63,7 +64,7 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 {
   failed_assertion_cb = fn;
 }
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
 #define capturing_bugs() (0)
 #define add_captured_bug(s) do { } while (0)
 #endif /* defined(TOR_UNIT_TESTS) */
@@ -71,24 +72,42 @@ tor_set_failed_assertion_callback(void (*fn)(void))
 /** Helper for tor_assert: report the assertion failure. */
 void
 tor_assertion_failed_(const char *fname, unsigned int line,
-                      const char *func, const char *expr)
+                      const char *func, const char *expr,
+                      const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
+  char *extra = NULL;
+  va_list ap;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+  if (fmt) {
+    va_start(ap,fmt);
+    tor_vasprintf(&extra, fmt, ap);
+    va_end(ap);
+  }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
   log_err(LD_BUG, "%s:%u: %s: Assertion %s failed; aborting.",
           fname, line, func, expr);
-  tor_snprintf(buf, sizeof(buf),
-               "Assertion %s failed in %s at %s:%u",
-               expr, func, fname, line);
+  tor_asprintf(&buf, "Assertion %s failed in %s at %s:%u: %s",
+               expr, func, fname, line, extra ? extra : "");
+  tor_free(extra);
   log_backtrace(LOG_ERR, LD_BUG, buf);
+  tor_free(buf);
 }
 
 /** Helper for tor_assert_nonfatal: report the assertion failure. */
 void
 tor_bug_occurred_(const char *fname, unsigned int line,
                   const char *func, const char *expr,
-                  int once)
+                  int once, const char *fmt, ...)
 {
-  char buf[256];
+  char *buf = NULL;
   const char *once_str = once ?
     " (Future instances of this warning will be silenced.)": "";
   if (! expr) {
@@ -98,7 +117,7 @@ tor_bug_occurred_(const char *fname, unsigned int line,
     }
     log_warn(LD_BUG, "%s:%u: %s: This line should not have been reached.%s",
              fname, line, func, once_str);
-    tor_snprintf(buf, sizeof(buf),
+    tor_asprintf(&buf,
                  "Line unexpectedly reached at %s at %s:%u",
                  func, fname, line);
   } else {
@@ -106,13 +125,32 @@ tor_bug_occurred_(const char *fname, unsigned int line,
       add_captured_bug(expr);
       return;
     }
+
+    va_list ap;
+    char *extra = NULL;
+
+#ifdef __clang__
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wformat-nonliteral"
+#endif
+    if (fmt) {
+      va_start(ap,fmt);
+      tor_vasprintf(&extra, fmt, ap);
+      va_end(ap);
+    }
+#ifdef __clang__
+#pragma clang diagnostic pop
+#endif
+
     log_warn(LD_BUG, "%s:%u: %s: Non-fatal assertion %s failed.%s",
              fname, line, func, expr, once_str);
-    tor_snprintf(buf, sizeof(buf),
-                 "Non-fatal assertion %s failed in %s at %s:%u",
-                 expr, func, fname, line);
+    tor_asprintf(&buf, "Non-fatal assertion %s failed in %s at %s:%u%s%s",
+                 expr, func, fname, line, fmt ? " : " : "",
+                 extra ? extra : "");
+    tor_free(extra);
   }
   log_backtrace(LOG_WARN, LD_BUG, buf);
+  tor_free(buf);
 
 #ifdef TOR_UNIT_TESTS
   if (failed_assertion_cb) {
@@ -122,16 +160,18 @@ tor_bug_occurred_(const char *fname, unsigned int line,
 }
 
 /**
- * Call the abort() function to kill the current process with a fatal
- * error.
+ * Call the tor_raw_abort_() function to close raw logs, then kill the current
+ * process with a fatal error. But first, close the file-based log file
+ * descriptors, so error messages are written before process termination.
  *
  * (This is a separate function so that we declare it in util_bug.h without
- * including stdlib in all the users of util_bug.h)
+ * including torerr.h in all the users of util_bug.h)
  **/
 void
 tor_abort_(void)
 {
-  abort();
+  logs_flush_sigsafe();
+  tor_raw_abort_();
 }
 
 #ifdef _WIN32
diff --git a/src/lib/log/util_bug.h b/src/lib/log/util_bug.h
index 17e8d0c5a7..684dc7c6dd 100644
--- a/src/lib/log/util_bug.h
+++ b/src/lib/log/util_bug.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -80,10 +80,10 @@
     tor__assert_tmp_value__;                    \
   } )
 #define ASSERT_PREDICT_LIKELY_(e) ASSERT_PREDICT_UNLIKELY_(e)
-#else
+#else /* !(defined(TOR_UNIT_TESTS) && defined(__GNUC__)) */
 #define ASSERT_PREDICT_UNLIKELY_(e) PREDICT_UNLIKELY(e)
 #define ASSERT_PREDICT_LIKELY_(e) PREDICT_LIKELY(e)
-#endif
+#endif /* defined(TOR_UNIT_TESTS) && defined(__GNUC__) */
 
 /* Sometimes we don't want to use assertions during branch coverage tests; it
  * leads to tons of unreached branches which in reality are only assertions we
@@ -92,21 +92,28 @@
 #define tor_assert(a) STMT_BEGIN                                        \
   (void)(a);                                                            \
   STMT_END
-#else
+#define tor_assertf(a, fmt, ...) STMT_BEGIN                             \
+  (void)(a);                                                            \
+  (void)(fmt);                                                          \
+  STMT_END
+#else /* !(defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_T...)) */
 /** Like assert(3), but send assertion failures to the log as well as to
  * stderr. */
-#define tor_assert(expr) STMT_BEGIN                                     \
+#define tor_assert(expr) tor_assertf(expr, NULL)
+
+#define tor_assertf(expr, fmt, ...) STMT_BEGIN                          \
   if (ASSERT_PREDICT_LIKELY_(expr)) {                                   \
   } else {                                                              \
-    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr);     \
-    tor_abort_();                                                       \
+    tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__, #expr,      \
+                          fmt, ##__VA_ARGS__);                          \
+    tor_abort_();                                                        \
   } STMT_END
 #endif /* defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS) */
 
 #define tor_assert_unreached()                                  \
   STMT_BEGIN {                                                  \
     tor_assertion_failed_(SHORT_FILE__, __LINE__, __func__,     \
-                          "line should be unreached");          \
+                          "line should be unreached", NULL);    \
     tor_abort_();                                               \
   } STMT_END
 
@@ -124,7 +131,9 @@
 #undef BUG
 // Coverity defines this in global headers; let's override it.  This is a
 // magic coverity-only preprocessor thing.
+#ifndef COCCI
 #nodef BUG(x) (x)
+#endif
 #endif /* defined(__COVERITY__) */
 
 #if defined(__COVERITY__) || defined(__clang_analyzer__)
@@ -133,37 +142,58 @@
 #define ALL_BUGS_ARE_FATAL
 #endif
 
+/** Define ALL_BUGS_ARE_FATAL if you want Tor to crash when any problem comes
+ * up, so you can get a coredump and track things down. */
 #ifdef ALL_BUGS_ARE_FATAL
 #define tor_assert_nonfatal_unreached() tor_assert(0)
 #define tor_assert_nonfatal(cond) tor_assert((cond))
+#define tor_assertf_nonfatal(cond, fmt, ...)    \
+  tor_assertf(cond, fmt, ##__VA_ARGS__)
 #define tor_assert_nonfatal_unreached_once() tor_assert(0)
 #define tor_assert_nonfatal_once(cond) tor_assert((cond))
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")"), \
+   (tor_assertion_failed_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",NULL), \
     tor_abort_(), 1)                                                    \
    : 0)
+#ifndef COCCI
+#define IF_BUG_ONCE(cond) if (BUG(cond))
+#endif
 #elif defined(TOR_UNIT_TESTS) && defined(DISABLE_ASSERTS_IN_UNIT_TESTS)
 #define tor_assert_nonfatal_unreached() STMT_NIL
 #define tor_assert_nonfatal(cond) ((void)(cond))
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  (void)cond;                                                           \
+  (void)fmt;                                                            \
+  STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_NIL
 #define tor_assert_nonfatal_once(cond) ((void)(cond))
 #define BUG(cond) (ASSERT_PREDICT_UNLIKELY_(cond) ? 1 : 0)
+#ifndef COCCI
+#define IF_BUG_ONCE(cond) if (BUG(cond))
+#endif
 #else /* Normal case, !ALL_BUGS_ARE_FATAL, !DISABLE_ASSERTS_IN_UNIT_TESTS */
 #define tor_assert_nonfatal_unreached() STMT_BEGIN                      \
-  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0);         \
+  tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 0, NULL);   \
   STMT_END
 #define tor_assert_nonfatal(cond) STMT_BEGIN                            \
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else {                                                              \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0, NULL);\
+  }                                                                     \
+  STMT_END
+#define tor_assertf_nonfatal(cond, fmt, ...) STMT_BEGIN                 \
+  if (ASSERT_PREDICT_UNLIKELY_(cond)) {                                 \
+  } else {                                                              \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 0,        \
+                      fmt, ##__VA_ARGS__);                               \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_unreached_once() STMT_BEGIN                 \
   static int warning_logged__ = 0;                                      \
   if (!warning_logged__) {                                              \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1);       \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, NULL, 1, NULL); \
   }                                                                     \
   STMT_END
 #define tor_assert_nonfatal_once(cond) STMT_BEGIN                       \
@@ -171,15 +201,15 @@
   if (ASSERT_PREDICT_LIKELY_(cond)) {                                   \
   } else if (!warning_logged__) {                                       \
     warning_logged__ = 1;                                               \
-    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1);      \
+    tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__, #cond, 1, NULL);\
   }                                                                     \
   STMT_END
 #define BUG(cond)                                                       \
   (ASSERT_PREDICT_UNLIKELY_(cond) ?                                     \
-   (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0), 1) \
+  (tor_bug_occurred_(SHORT_FILE__,__LINE__,__func__,"!("#cond")",0,NULL),1) \
    : 0)
-#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
 
+#ifndef COCCI
 #ifdef __GNUC__
 #define IF_BUG_ONCE__(cond,var)                                         \
   if (( {                                                               \
@@ -188,26 +218,28 @@
       if (bool_result && !var) {                                        \
         var = 1;                                                        \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                          "!("#cond")", 1);                             \
+                          ("!("#cond")"), 1, NULL);                     \
       }                                                                 \
       bool_result; } ))
-#else /* !(defined(__GNUC__)) */
+#else /* !defined(__GNUC__) */
 #define IF_BUG_ONCE__(cond,var)                                         \
   static int var = 0;                                                   \
   if ((cond) ?                                                          \
       (var ? 1 :                                                        \
        (var=1,                                                          \
         tor_bug_occurred_(SHORT_FILE__, __LINE__, __func__,             \
-                           "!("#cond")", 1),                            \
+                          ("!("#cond")"), 1, NULL),                     \
         1))                                                             \
       : 0)
 #endif /* defined(__GNUC__) */
+#endif /* !defined(COCCI) */
+
 #define IF_BUG_ONCE_VARNAME_(a)               \
   warning_logged_on_ ## a ## __
 #define IF_BUG_ONCE_VARNAME__(a)              \
   IF_BUG_ONCE_VARNAME_(a)
 
-/** This macro behaves as 'if (bug(x))', except that it only logs its
+/** This macro behaves as 'if (BUG(x))', except that it only logs its
  * warning once, no matter how many times it triggers.
  */
 
@@ -215,6 +247,8 @@
   IF_BUG_ONCE__(ASSERT_PREDICT_UNLIKELY_(cond),                 \
                 IF_BUG_ONCE_VARNAME__(__LINE__))
 
+#endif /* defined(ALL_BUGS_ARE_FATAL) || ... */
+
 /**
  * Use this macro after a nonfatal assertion, and before a case statement
  * where you would want to fall through.
@@ -226,16 +260,23 @@
 #define FALLTHROUGH_UNLESS_ALL_BUGS_ARE_FATAL FALLTHROUGH
 #endif
 
-/** Define this if you want Tor to crash when any problem comes up,
- * so you can get a coredump and track things down. */
-// #define tor_fragile_assert() tor_assert_unreached(0)
+/** In older code, we used tor_fragile_assert() to mark optional failure
+ * points. At these points, we could make some debug builds fail.
+ * (But release builds would continue.)
+ *
+ * To get the same behaviour in recent tor versions, define
+ * ALL_BUGS_ARE_FATAL, and use any non-fatal assertion or *BUG() macro.
+ */
 #define tor_fragile_assert() tor_assert_nonfatal_unreached_once()
 
 void tor_assertion_failed_(const char *fname, unsigned int line,
-                           const char *func, const char *expr);
+                           const char *func, const char *expr,
+                           const char *fmt, ...)
+    CHECK_PRINTF(5,6);
 void tor_bug_occurred_(const char *fname, unsigned int line,
                        const char *func, const char *expr,
-                       int once);
+                       int once, const char *fmt, ...)
+  CHECK_PRINTF(6,7);
 
 void tor_abort_(void) ATTR_NORETURN;
 
diff --git a/src/lib/log/win32err.c b/src/lib/log/win32err.c
index dc45cb4c3d..8136813aab 100644
--- a/src/lib/log/win32err.c
+++ b/src/lib/log/win32err.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -47,7 +47,7 @@ format_win32_error(DWORD err)
     result = tor_malloc(len);
     wcstombs(result,str,len);
     result[len-1] = '\0';
-#else /* !(defined(UNICODE)) */
+#else /* !defined(UNICODE) */
     result = tor_strdup(str);
 #endif /* defined(UNICODE) */
   } else {
diff --git a/src/lib/log/win32err.h b/src/lib/log/win32err.h
index 33413dfd15..5c1386a64d 100644
--- a/src/lib/log/win32err.h
+++ b/src/lib/log/win32err.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@
 char *format_win32_error(DWORD err);
 #endif
 
-#endif
+#endif /* !defined(TOR_WIN32ERR_H) */
diff --git a/src/lib/malloc/.may_include b/src/lib/malloc/.may_include
index cc62bb1013..7686bf862a 100644
--- a/src/lib/malloc/.may_include
+++ b/src/lib/malloc/.may_include
@@ -3,4 +3,4 @@ orconfig.h
 lib/cc/*.h
 lib/err/*.h
 lib/malloc/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/malloc/include.am b/src/lib/malloc/include.am
index 502cc1c6b7..b74292bc6e 100644
--- a/src/lib/malloc/include.am
+++ b/src/lib/malloc/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-malloc-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_malloc_a_SOURCES =			\
-	src/lib/malloc/malloc.c
+	src/lib/malloc/malloc.c				\
+	src/lib/malloc/map_anon.c
 
 if USE_OPENBSD_MALLOC
 src_lib_libtor_malloc_a_SOURCES += src/ext/OpenBSD_malloc_Linux.c
@@ -17,5 +19,7 @@ src_lib_libtor_malloc_testing_a_SOURCES = \
 src_lib_libtor_malloc_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_malloc_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/malloc/malloc.h
+	src/lib/malloc/malloc.h				\
+	src/lib/malloc/map_anon.h
diff --git a/src/lib/malloc/lib_malloc.md b/src/lib/malloc/lib_malloc.md
new file mode 100644
index 0000000000..ff61722f02
--- /dev/null
+++ b/src/lib/malloc/lib_malloc.md
@@ -0,0 +1,76 @@
+@dir /lib/malloc
+@brief lib/malloc: Wrappers and utilities for memory management.
+
+
+Tor imposes a few light wrappers over C's native malloc and free
+functions, to improve convenience, and to allow wholescale replacement
+of malloc and free as needed.
+
+You should never use 'malloc', 'calloc', 'realloc, or 'free' on their
+own; always use the variants prefixed with 'tor_'.
+They are the same as the standard C functions, with the following
+exceptions:
+
+   * `tor_free(NULL)` is a no-op.
+   * `tor_free()` is a macro that takes an lvalue as an argument and sets it to
+     NULL after freeing it.  To avoid this behavior, you can use `tor_free_()`
+     instead.
+   * tor_malloc() and friends fail with an assertion if they are asked to
+     allocate a value so large that it is probably an underflow.
+   * It is always safe to `tor_malloc(0)`, regardless of whether your libc
+     allows it.
+   * `tor_malloc()`, `tor_realloc()`, and friends are never allowed to fail.
+     Instead, Tor will die with an assertion.  This means that you never
+     need to check their return values.  See the next subsection for
+     information on why we think this is a good idea.
+
+We define additional general-purpose memory allocation functions as well:
+
+   * `tor_malloc_zero(x)` behaves as `calloc(1, x)`, except the it makes clear
+     the intent to allocate a single zeroed-out value.
+   * `tor_reallocarray(x,y)` behaves as the OpenBSD reallocarray function.
+     Use it for cases when you need to realloc() in a multiplication-safe
+     way.
+
+And specific-purpose functions as well:
+
+   * `tor_strdup()` and `tor_strndup()` behaves as the underlying libc
+     functions, but use `tor_malloc()` instead of the underlying function.
+   * `tor_memdup()` copies a chunk of memory of a given size.
+   * `tor_memdup_nulterm()` copies a chunk of memory of a given size, then
+     NUL-terminates it just to be safe.
+
+#### Why assert on allocation failure?
+
+Why don't we allow `tor_malloc()` and its allies to return NULL?
+
+First, it's error-prone.  Many programmers forget to check for NULL return
+values, and testing for `malloc()` failures is a major pain.
+
+Second, it's not necessarily a great way to handle OOM conditions. It's
+probably better (we think) to have a memory target where we dynamically free
+things ahead of time in order to stay under the target.  Trying to respond to
+an OOM at the point of `tor_malloc()` failure, on the other hand, would involve
+a rare operation invoked from deep in the call stack.  (Again, that's
+error-prone and hard to debug.)
+
+Third, thanks to the rise of Linux and other operating systems that allow
+memory to be overcommitted, you can't actually ever rely on getting a NULL
+from `malloc()` when you're out of memory; instead you have to use an approach
+closer to tracking the total memory usage.
+
+#### Conventions for your own allocation functions.
+
+Whenever you create a new type, the convention is to give it a pair of
+`x_new()` and `x_free_()` functions, named after the type.
+
+Calling `x_free(NULL)` should always be a no-op.
+
+There should additionally be an `x_free()` macro, defined in terms of
+`x_free_()`.  This macro should set its lvalue to NULL.  You can define it
+using the FREE_AND_NULL macro, as follows:
+
+```
+#define x_free(ptr) FREE_AND_NULL(x_t, x_free_, (ptr))
+```
+
diff --git a/src/lib/malloc/malloc.c b/src/lib/malloc/malloc.c
index 8628acfc97..9c9d600260 100644
--- a/src/lib/malloc/malloc.c
+++ b/src/lib/malloc/malloc.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/malloc/malloc.h b/src/lib/malloc/malloc.h
index ef6b509ca4..80e8091adc 100644
--- a/src/lib/malloc/malloc.h
+++ b/src/lib/malloc/malloc.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -48,12 +48,12 @@ void tor_free_(void *mem);
     raw_free(*tor_free__tmpvar);                               \
     *tor_free__tmpvar=NULL;                                    \
   STMT_END
-#else
+#else /* !defined(__GNUC__) */
 #define tor_free(p) STMT_BEGIN                                 \
   raw_free(p);                                                 \
   (p)=NULL;                                                    \
   STMT_END
-#endif
+#endif /* defined(__GNUC__) */
 
 #define tor_malloc(size)       tor_malloc_(size)
 #define tor_malloc_zero(size)  tor_malloc_zero_(size)
diff --git a/src/lib/malloc/map_anon.c b/src/lib/malloc/map_anon.c
new file mode 100644
index 0000000000..628966012a
--- /dev/null
+++ b/src/lib/malloc/map_anon.c
@@ -0,0 +1,271 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.c
+ * \brief Manage anonymous mappings.
+ **/
+
+#include "orconfig.h"
+#include "lib/malloc/map_anon.h"
+#include "lib/malloc/malloc.h"
+#include "lib/err/torerr.h"
+
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_MACH_VM_INHERIT_H
+#include <mach/vm_inherit.h>
+#endif
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
+#include <string.h>
+#include <errno.h>
+
+/**
+ * Macro to get the high bytes of a size_t, if there are high bytes.
+ * Windows needs this; other operating systems define a size_t that does
+ * what it should.
+ */
+#if SIZEOF_SIZE_T > 4
+#define HIGH_SIZE_T_BYTES(sz) ((sz) >> 32)
+#else
+#define HIGH_SIZE_T_BYTES(sz) (0)
+#endif
+
+/* Here we define a MINHERIT macro that is minherit() or madvise(), depending
+ * on what we actually want.
+ *
+ * If there's a flag that sets pages to zero after fork, we define FLAG_ZERO
+ * to be that flag.  If there's a flag unmaps pages after fork, we define
+ * FLAG_NOINHERIT to be that flag.
+ */
+#if defined(HAVE_MINHERIT)
+#define MINHERIT minherit
+
+#ifdef INHERIT_ZERO
+#define FLAG_ZERO INHERIT_ZERO
+#elif defined(MAP_INHERIT_ZERO)
+#define FLAG_ZERO MAP_INHERIT_ZERO
+#endif
+#ifdef INHERIT_NONE
+#define FLAG_NOINHERIT INHERIT_NONE
+#elif defined(VM_INHERIT_NONE)
+#define FLAG_NOINHERIT VM_INHERIT_NONE
+#elif defined(MAP_INHERIT_NONE)
+#define FLAG_NOINHERIT MAP_INHERIT_NONE
+#endif /* defined(INHERIT_NONE) || ... */
+
+#elif defined(HAVE_MADVISE)
+
+#define MINHERIT madvise
+
+#ifdef MADV_WIPEONFORK
+#define FLAG_ZERO MADV_WIPEONFORK
+#endif
+#ifdef MADV_DONTFORK
+#define FLAG_NOINHERIT MADV_DONTFORK
+#endif
+
+#endif /* defined(HAVE_MINHERIT) || ... */
+
+#if defined(HAVE_MINHERIT) && !defined(FLAG_ZERO) && !defined(FLAG_NOINHERIT)
+#warning "minherit() is defined, but FLAG_ZERO/NOINHERIT are not."
+#warning "This is probably a bug in Tor's support for this platform."
+#endif
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being swapped
+ * to disk.  Return 0 on success or if the facility is not available on this
+ * OS; return -1 on failure.
+ */
+static int
+lock_mem(void *mem, size_t sz)
+{
+#ifdef _WIN32
+  return VirtualLock(mem, sz) ? 0 : -1;
+#elif defined(HAVE_MLOCK)
+  return mlock(mem, sz);
+#else
+  (void) mem;
+  (void) sz;
+
+  return 0;
+#endif /* defined(_WIN32) || ... */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from appearing in
+ * a core dump.  Return 0 on success or if the facility is not available on
+ * this OS; return -1 on failure.
+ */
+static int
+nodump_mem(void *mem, size_t sz)
+{
+#if defined(MADV_DONTDUMP)
+  int rv = madvise(mem, sz, MADV_DONTDUMP);
+  if (rv == 0) {
+    return 0;
+  } else if (errno == ENOSYS || errno == EINVAL) {
+    return 0; // syscall not supported, or flag not supported.
+  } else {
+    tor_log_err_sigsafe("Unexpected error from madvise: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !defined(MADV_DONTDUMP) */
+  (void) mem;
+  (void) sz;
+  return 0;
+#endif /* defined(MADV_DONTDUMP) */
+}
+
+/**
+ * Helper: try to prevent the <b>sz</b> bytes at <b>mem</b> from being
+ * accessible in child processes -- ideally by having them set to 0 after a
+ * fork, and if that doesn't work, by having them unmapped after a fork.
+ * Return 0 on success or if the facility is not available on this OS; return
+ * -1 on failure.
+ *
+ * If we successfully make the memory uninheritable, adjust the value of
+ * *<b>inherit_result_out</b>.
+ */
+static int
+noinherit_mem(void *mem, size_t sz, inherit_res_t *inherit_result_out)
+{
+#ifdef FLAG_ZERO
+  int r = MINHERIT(mem, sz, FLAG_ZERO);
+  if (r == 0) {
+    *inherit_result_out = INHERIT_RES_ZERO;
+    return 0;
+  }
+#endif /* defined(FLAG_ZERO) */
+
+#ifdef FLAG_NOINHERIT
+  int r2 = MINHERIT(mem, sz, FLAG_NOINHERIT);
+  if (r2 == 0) {
+    *inherit_result_out = INHERIT_RES_DROP;
+    return 0;
+  }
+#endif /* defined(FLAG_NOINHERIT) */
+
+#if defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)
+  /* At least one operation was tried, and neither succeeded. */
+
+  if (errno == ENOSYS || errno == EINVAL) {
+    /* Syscall not supported, or flag not supported. */
+    return 0;
+  } else {
+    tor_log_err_sigsafe("Unexpected error from minherit: ",
+                        strerror(errno),
+                        NULL);
+    return -1;
+  }
+#else /* !(defined(FLAG_ZERO) || defined(FLAG_NOINHERIT)) */
+  (void)inherit_result_out;
+  (void)mem;
+  (void)sz;
+  return 0;
+#endif /* defined(FLAG_ZERO) || defined(FLAG_NOINHERIT) */
+}
+
+/**
+ * Return a new anonymous memory mapping that holds <b>sz</b> bytes.
+ *
+ * Memory mappings are unlike the results from malloc() in that they are
+ * handled separately by the operating system, and as such can have different
+ * kernel-level flags set on them.
+ *
+ * The "flags" argument may be zero or more of ANONMAP_PRIVATE and
+ * ANONMAP_NOINHERIT.
+ *
+ * Memory returned from this function must be released with
+ * tor_munmap_anonymous().
+ *
+ * If <b>inherit_result_out</b> is non-NULL, set it to one of
+ * INHERIT_RES_KEEP, INHERIT_RES_DROP, or INHERIT_RES_ZERO, depending on the
+ * properties of the returned memory.
+ *
+ * [Note: OS people use the word "anonymous" here to mean that the memory
+ * isn't associated with any file. This has *nothing* to do with the kind of
+ * anonymity that Tor is trying to provide.]
+ */
+void *
+tor_mmap_anonymous(size_t sz, unsigned flags,
+                   inherit_res_t *inherit_result_out)
+{
+  void *ptr;
+  inherit_res_t itmp=0;
+  if (inherit_result_out == NULL) {
+    inherit_result_out = &itmp;
+  }
+  *inherit_result_out = INHERIT_RES_KEEP;
+
+#if defined(_WIN32)
+  HANDLE mapping = CreateFileMapping(INVALID_HANDLE_VALUE,
+                                     NULL, /*attributes*/
+                                     PAGE_READWRITE,
+                                     HIGH_SIZE_T_BYTES(sz),
+                                     sz & 0xffffffff,
+                                     NULL /* name */);
+  raw_assert(mapping != NULL);
+  ptr = MapViewOfFile(mapping, FILE_MAP_WRITE,
+                      0, 0, /* Offset */
+                      0 /* Extend to end of mapping */);
+  raw_assert(ptr);
+  CloseHandle(mapping); /* mapped view holds a reference */
+#elif defined(HAVE_SYS_MMAN_H)
+  ptr = mmap(NULL, sz,
+             PROT_READ|PROT_WRITE,
+             MAP_ANON|MAP_PRIVATE,
+             -1, 0);
+  raw_assert(ptr != MAP_FAILED);
+  raw_assert(ptr != NULL);
+#else
+  ptr = tor_malloc_zero(sz);
+#endif /* defined(_WIN32) || ... */
+
+  if (flags & ANONMAP_PRIVATE) {
+    int lock_result = lock_mem(ptr, sz);
+    raw_assert(lock_result == 0);
+    int nodump_result = nodump_mem(ptr, sz);
+    raw_assert(nodump_result == 0);
+  }
+
+  if (flags & ANONMAP_NOINHERIT) {
+    int noinherit_result = noinherit_mem(ptr, sz, inherit_result_out);
+    raw_assert(noinherit_result == 0);
+  }
+
+  return ptr;
+}
+
+/**
+ * Release <b>sz</b> bytes of memory that were previously mapped at
+ * <b>mapping</b> by tor_mmap_anonymous().
+ **/
+void
+tor_munmap_anonymous(void *mapping, size_t sz)
+{
+  if (!mapping)
+    return;
+
+#if defined(_WIN32)
+  (void)sz;
+  UnmapViewOfFile(mapping);
+#elif defined(HAVE_SYS_MMAN_H)
+  munmap(mapping, sz);
+#else
+  (void)sz;
+  tor_free(mapping);
+#endif /* defined(_WIN32) || ... */
+}
diff --git a/src/lib/malloc/map_anon.h b/src/lib/malloc/map_anon.h
new file mode 100644
index 0000000000..0354668d65
--- /dev/null
+++ b/src/lib/malloc/map_anon.h
@@ -0,0 +1,71 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file map_anon.h
+ * \brief Headers for map_anon.c
+ **/
+
+#ifndef TOR_MAP_ANON_H
+#define TOR_MAP_ANON_H
+
+#include "lib/malloc/malloc.h"
+#include <stddef.h>
+
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * swapped or dumped.
+ *
+ * In some operating systems, this flag is not implemented.
+ */
+#define ANONMAP_PRIVATE   (1u<<0)
+/**
+ * When this flag is specified, try to prevent the mapping from being
+ * inherited after a fork().  In some operating systems, trying to access it
+ * afterwards will cause its contents to be zero.  In others, trying to access
+ * it afterwards will cause a crash.
+ *
+ * In some operating systems, this flag is not implemented at all.
+ */
+#define ANONMAP_NOINHERIT (1u<<1)
+
+typedef enum {
+  /** Possible value for inherit_result_out: the memory will be kept
+   * by any child process. */
+  INHERIT_RES_KEEP=0,
+  /** Possible value for inherit_result_out: the memory will be dropped in the
+   * child process. Attempting to access it will likely cause a segfault. */
+  INHERIT_RES_DROP,
+  /** Possible value for inherit_result_out: the memory will be cleared in
+   * the child process. */
+  INHERIT_RES_ZERO
+} inherit_res_t;
+
+/* Here we define the NOINHERIT_CAN_FAIL macro if and only if
+ * it's possible that ANONMAP_NOINHERIT might yield inheritable memory.
+ */
+#ifdef _WIN32
+/* Windows can't fork, so NOINHERIT is never needed. */
+#elif defined(HAVE_MINHERIT)
+/* minherit() will always have a working MAP_INHERIT_NONE or MAP_INHERIT_ZERO.
+ * NOINHERIT should always work.
+ */
+#elif defined(HAVE_MADVISE)
+/* madvise() sometimes has neither MADV_DONTFORK and MADV_WIPEONFORK.
+ * We need to be ready for the possibility it failed.
+ *
+ * (Linux added DONTFORK in 2.6.16 and WIPEONFORK in 4.14. If we someday
+ * require 2.6.16 or later, we can assume that DONTFORK will work.)
+ */
+#define NOINHERIT_CAN_FAIL
+#else
+#define NOINHERIT_CAN_FAIL
+#endif /* defined(_WIN32) || ... */
+
+void *tor_mmap_anonymous(size_t sz, unsigned flags,
+                         inherit_res_t *inherit_result_out);
+void tor_munmap_anonymous(void *mapping, size_t sz);
+
+#endif /* !defined(TOR_MAP_ANON_H) */
diff --git a/src/lib/math/.may_include b/src/lib/math/.may_include
index 1fd26864dc..f8bc264a5f 100644
--- a/src/lib/math/.may_include
+++ b/src/lib/math/.may_include
@@ -3,3 +3,5 @@ orconfig.h
 lib/cc/*.h
 lib/log/*.h
 lib/math/*.h
+lib/testsupport/*.h
+lib/crypt_ops/*.h
diff --git a/src/lib/math/fp.c b/src/lib/math/fp.c
index eafad358c3..c09555209e 100644
--- a/src/lib/math/fp.c
+++ b/src/lib/math/fp.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -74,8 +74,8 @@ clamp_double_to_int64(double number)
   branches that are not taken.
 */
 #define PROBLEMATIC_FLOAT_CONVERSION_WARNING
-DISABLE_GCC_WARNING(float-conversion)
-#endif /* defined(MINGW_ANY) && GCC_VERSION >= 409 */
+DISABLE_GCC_WARNING("-Wfloat-conversion")
+#endif /* (defined(MINGW_ANY)||defined(__FreeBSD__)) && GCC_VERSION >= 409 */
 
 /*
   With clang 4.0 we apparently run into "double promotion" warnings here,
@@ -84,7 +84,7 @@ DISABLE_GCC_WARNING(float-conversion)
 #if defined(__clang__)
 #if __has_warning("-Wdouble-promotion")
 #define PROBLEMATIC_DOUBLE_PROMOTION_WARNING
-DISABLE_GCC_WARNING(double-promotion)
+DISABLE_GCC_WARNING("-Wdouble-promotion")
 #endif
 #endif /* defined(__clang__) */
 
@@ -115,9 +115,29 @@ DISABLE_GCC_WARNING(double-promotion)
   return signbit(number) ? INT64_MIN : INT64_MAX;
 
 #ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
-ENABLE_GCC_WARNING(double-promotion)
+ENABLE_GCC_WARNING("-Wdouble-promotion")
 #endif
 #ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
-ENABLE_GCC_WARNING(float-conversion)
+ENABLE_GCC_WARNING("-Wfloat-conversion")
+#endif
+}
+
+/* isinf() wrapper for tor */
+int
+tor_isinf(double x)
+{
+  /* Same as above, work around the "double promotion" warnings */
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+DISABLE_GCC_WARNING("-Wfloat-conversion")
+#endif
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+DISABLE_GCC_WARNING("-Wdouble-promotion")
+#endif
+  return isinf(x);
+#ifdef PROBLEMATIC_DOUBLE_PROMOTION_WARNING
+ENABLE_GCC_WARNING("-Wdouble-promotion")
+#endif
+#ifdef PROBLEMATIC_FLOAT_CONVERSION_WARNING
+ENABLE_GCC_WARNING("-Wfloat-conversion")
 #endif
 }
diff --git a/src/lib/math/fp.h b/src/lib/math/fp.h
index 6f07152e92..0a7a685485 100644
--- a/src/lib/math/fp.h
+++ b/src/lib/math/fp.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,5 +19,6 @@ double tor_mathlog(double d) ATTR_CONST;
 long tor_lround(double d) ATTR_CONST;
 int64_t tor_llround(double d) ATTR_CONST;
 int64_t clamp_double_to_int64(double number);
+int tor_isinf(double x);
 
-#endif
+#endif /* !defined(TOR_FP_H) */
diff --git a/src/lib/math/include.am b/src/lib/math/include.am
index b088b3f3cc..b2ca280f47 100644
--- a/src/lib/math/include.am
+++ b/src/lib/math/include.am
@@ -5,16 +5,19 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-math-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_math_a_SOURCES =	\
 		src/lib/math/fp.c		\
-		src/lib/math/laplace.c
-
+		src/lib/math/laplace.c 	\
+		src/lib/math/prob_distr.c
 
 src_lib_libtor_math_testing_a_SOURCES = \
 	$(src_lib_libtor_math_a_SOURCES)
 src_lib_libtor_math_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_math_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/math/fp.h		\
-		src/lib/math/laplace.h
+		src/lib/math/laplace.h  \
+		src/lib/math/prob_distr.h
diff --git a/src/lib/math/laplace.c b/src/lib/math/laplace.c
index 302edb20b8..a0e67384e6 100644
--- a/src/lib/math/laplace.c
+++ b/src/lib/math/laplace.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,7 +29,7 @@ sample_laplace_distribution(double mu, double b, double p)
   tor_assert(p >= 0.0 && p < 1.0);
 
   /* This is the "inverse cumulative distribution function" from:
-   * http://en.wikipedia.org/wiki/Laplace_distribution */
+   * https://en.wikipedia.org/wiki/Laplace_distribution */
   if (p <= 0.0) {
     /* Avoid taking log(0.0) == -INFINITY, as some processors or compiler
      * options can cause the program to trap. */
diff --git a/src/lib/math/laplace.h b/src/lib/math/laplace.h
index e8651e5197..e0dd166bbd 100644
--- a/src/lib/math/laplace.h
+++ b/src/lib/math/laplace.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ int64_t sample_laplace_distribution(double mu, double b, double p);
 int64_t add_laplace_noise(int64_t signal, double random, double delta_f,
                           double epsilon);
 
-#endif
+#endif /* !defined(TOR_LAPLACE_H) */
diff --git a/src/lib/math/lib_math.md b/src/lib/math/lib_math.md
new file mode 100644
index 0000000000..9cc256d24b
--- /dev/null
+++ b/src/lib/math/lib_math.md
@@ -0,0 +1,6 @@
+@dir /lib/math
+@brief lib/math: Floating-point math utilities.
+
+This module includes a bunch of floating-point compatibility code, and
+implementations for several probability distributions.
+
diff --git a/src/lib/math/prob_distr.c b/src/lib/math/prob_distr.c
new file mode 100644
index 0000000000..31d485120e
--- /dev/null
+++ b/src/lib/math/prob_distr.c
@@ -0,0 +1,1692 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file prob_distr.c
+ *
+ * \brief
+ *  Implements various probability distributions.
+ *  Almost all code is courtesy of Riastradh.
+ *
+ * \details
+ * Here are some details that might help you understand this file:
+ *
+ * - Throughout this file, `eps' means the largest relative error of a
+ *   correctly rounded floating-point operation, which in binary64
+ *   floating-point arithmetic is 2^-53.  Here the relative error of a
+ *   true value x from a computed value y is |x - y|/|x|.  This
+ *   definition of epsilon is conventional for numerical analysts when
+ *   writing error analyses.  (If your libm doesn't provide correctly
+ *   rounded exp and log, their relative error is usually below 2*2^-53
+ *   and probably closer to 1.1*2^-53 instead.)
+ *
+ *   The C constant DBL_EPSILON is actually twice this, and should
+ *   perhaps rather be named ulp(1) -- that is, it is the distance from
+ *   1 to the next greater floating-point number, which is usually of
+ *   more interest to programmers and hardware engineers.
+ *
+ *   Since this file is concerned mainly with error bounds rather than
+ *   with low-level bit-hacking of floating-point numbers, we adopt the
+ *   numerical analysts' definition in the comments, though we do use
+ *   DBL_EPSILON in a handful of places where it is convenient to use
+ *   some function of eps = DBL_EPSILON/2 in a case analysis.
+ *
+ * - In various functions (e.g. sample_log_logistic()) we jump through hoops so
+ *   that we can use reals closer to 0 than closer to 1, since we achieve much
+ *   greater accuracy for floating point numbers near 0. In particular, we can
+ *   represent differences as small as 10^-300 for numbers near 0, but of no
+ *   less than 10^-16 for numbers near 1.
+ **/
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "lib/math/prob_distr.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "lib/cc/ctassert.h"
+#include "lib/log/util_bug.h"
+
+#include <float.h>
+#include <math.h>
+#include <stddef.h>
+
+#ifndef COCCI
+/** Declare a function that downcasts from a generic dist struct to the actual
+ *  subtype probablity distribution it represents. */
+#define DECLARE_PROB_DISTR_DOWNCAST_FN(name) \
+  static inline                           \
+  const struct name##_t *                             \
+  dist_to_const_##name(const struct dist_t *obj) {    \
+    tor_assert(obj->ops == &name##_ops);            \
+    return SUBTYPE_P(obj, struct name ## _t, base);   \
+  }
+DECLARE_PROB_DISTR_DOWNCAST_FN(uniform)
+DECLARE_PROB_DISTR_DOWNCAST_FN(geometric)
+DECLARE_PROB_DISTR_DOWNCAST_FN(logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(log_logistic)
+DECLARE_PROB_DISTR_DOWNCAST_FN(genpareto)
+DECLARE_PROB_DISTR_DOWNCAST_FN(weibull)
+#endif /* !defined(COCCI) */
+
+/**
+ * Count number of one bits in 32-bit word.
+ */
+static unsigned
+bitcount32(uint32_t x)
+{
+
+  /* Count two-bit groups.  */
+  x -= (x >> 1) & UINT32_C(0x55555555);
+
+  /* Count four-bit groups.  */
+  x = ((x >> 2) & UINT32_C(0x33333333)) + (x & UINT32_C(0x33333333));
+
+  /* Count eight-bit groups.  */
+  x = (x + (x >> 4)) & UINT32_C(0x0f0f0f0f);
+
+  /* Sum all eight-bit groups, and extract the sum.  */
+  return (x * UINT32_C(0x01010101)) >> 24;
+}
+
+/**
+ * Count leading zeros in 32-bit word.
+ */
+static unsigned
+clz32(uint32_t x)
+{
+
+  /* Round up to a power of two.  */
+  x |= x >> 1;
+  x |= x >> 2;
+  x |= x >> 4;
+  x |= x >> 8;
+  x |= x >> 16;
+
+  /* Subtract count of one bits from 32.  */
+  return (32 - bitcount32(x));
+}
+
+/*
+ * Some lemmas that will be used throughout this file to prove various error
+ * bounds:
+ *
+ * Lemma 1.  If |d| <= 1/2, then 1/(1 + d) <= 2.
+ *
+ * Proof.  If 0 <= d <= 1/2, then 1 + d >= 1, so that 1/(1 + d) <= 1.
+ * If -1/2 <= d <= 0, then 1 + d >= 1/2, so that 1/(1 + d) <= 2.  QED.
+ *
+ * Lemma 2. If b = a*(1 + d)/(1 + d') for |d'| < 1/2 and nonzero a, b,
+ * then b = a*(1 + e) for |e| <= 2|d' - d|.
+ *
+ * Proof.  |a - b|/|a|
+ *             = |a - a*(1 + d)/(1 + d')|/|a|
+ *             = |1 - (1 + d)/(1 + d')|
+ *             = |(1 + d' - 1 - d)/(1 + d')|
+ *             = |(d' - d)/(1 + d')|
+ *            <= 2|d' - d|, by Lemma 1,
+ *
+ * QED.
+ *
+ * Lemma 3.  For |d|, |d'| < 1/4,
+ *
+ *     |log((1 + d)/(1 + d'))| <= 4|d - d'|.
+ *
+ * Proof.  Write
+ *
+ *     log((1 + d)/(1 + d'))
+ *      = log(1 + (1 + d)/(1 + d') - 1)
+ *      = log(1 + (1 + d - 1 - d')/(1 + d')
+ *      = log(1 + (d - d')/(1 + d')).
+ *
+ * By Lemma 1, |(d - d')/(1 + d')| < 2|d' - d| < 1, so the Taylor
+ * series of log(1 + x) converges absolutely for (d - d')/(1 + d'),
+ * and thus we have
+ *
+ *     |log(1 + (d - d')/(1 + d'))|
+ *      = |\sum_{n=1}^\infty ((d - d')/(1 + d'))^n/n|
+ *     <= \sum_{n=1}^\infty |(d - d')/(1 + d')|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n/n
+ *     <= \sum_{n=1}^\infty |2(d' - d)|^n
+ *      = 1/(1 - |2(d' - d)|)
+ *     <= 4|d' - d|,
+ *
+ * QED.
+ *
+ * Lemma 4.  If 1/e <= 1 + x <= e, then
+ *
+ *     log(1 + (1 + d) x) = (1 + d') log(1 + x)
+ *
+ * for |d'| < 8|d|.
+ *
+ * Proof.  Write
+ *
+ *     log(1 + (1 + d) x)
+ *     = log(1 + x + x*d)
+ *     = log((1 + x) (1 + x + x*d)/(1 + x))
+ *     = log(1 + x) + log((1 + x + x*d)/(1 + x))
+ *     = log(1 + x) (1 + log((1 + x + x*d)/(1 + x))/log(1 + x)).
+ *
+ * The relative error is bounded by
+ *
+ *     |log((1 + x + x*d)/(1 + x))/log(1 + x)|
+ *     <= 4|x + x*d - x|/|log(1 + x)|, by Lemma 3,
+ *      = 4|x*d|/|log(1 + x)|
+ *      < 8|d|,
+ *
+ * since in this range 0 < 1 - 1/e < x/log(1 + x) <= e - 1 < 2.  QED.
+ */
+
+/**
+ * Compute the logistic function: f(x) = 1/(1 + e^{-x}) = e^x/(1 + e^x).
+ * Maps a log-odds-space probability in [-infinity, +infinity] into a
+ * direct-space probability in [0,1].  Inverse of logit.
+ *
+ * Ill-conditioned for large x; the identity logistic(-x) = 1 -
+ * logistic(x) and the function logistichalf(x) = logistic(x) - 1/2 may
+ * help to rearrange a computation.
+ *
+ * This implementation gives relative error bounded by 7 eps.
+ */
+STATIC double
+logistic(double x)
+{
+  if (x <= log(DBL_EPSILON/2)) {
+    /*
+     * If x <= log(DBL_EPSILON/2) = log(eps), then e^x <= eps. In this case
+     * we will approximate the logistic() function with e^x because the
+     * relative error is less than eps. Here is a calculation of the
+     * relative error between the logistic() function and e^x and a proof
+     * that it's less than eps:
+     *
+     *     |e^x - e^x/(1 + e^x)|/|e^x/(1 + e^x)|
+     *     <= |1 - 1/(1 + e^x)|*|1 + e^x|
+     *      = |e^x/(1 + e^x)|*|1 + e^x|
+     *      = |e^x|
+     *     <= eps.
+     */
+    return exp(x); /* return e^x */
+  } else if (x <= -log(DBL_EPSILON/2)) {
+    /*
+     * e^{-x} > 0, so 1 + e^{-x} > 1, and 0 < 1/(1 +
+     * e^{-x}) < 1; further, since e^{-x} < 1 + e^{-x}, we
+     * also have 0 < 1/(1 + e^{-x}) < 1.  Thus, if exp has
+     * relative error d0, + has relative error d1, and /
+     * has relative error d2, then we get
+     *
+     *     (1 + d2)/[(1 + (1 + d0) e^{-x})(1 + d1)]
+     *     = (1 + d0)/[1 + e^{-x} + d0 e^{-x}
+     *                     + d1 + d1 e^{-x} + d0 d1 e^{-x}]
+     *     = (1 + d0)/[(1 + e^{-x})
+     *                 * (1 + d0 e^{-x}/(1 + e^{-x})
+     *                      + d1/(1 + e^{-x})
+     *                      + d0 d1 e^{-x}/(1 + e^{-x}))].
+     *     = (1 + d0)/[(1 + e^{-x})(1 + d')]
+     *     = [1/(1 + e^{-x})] (1 + d0)/(1 + d')
+     *
+     * where
+     *
+     *     d' = d0 e^{-x}/(1 + e^{-x})
+     *          + d1/(1 + e^{-x})
+     *          + d0 d1 e^{-x}/(1 + e^{-x}).
+     *
+     * By Lemma 2 this relative error is bounded by
+     *
+     *     2|d0 - d'|
+     *      = 2|d0 - d0 e^{-x}/(1 + e^{-x})
+     *             - d1/(1 + e^{-x})
+     *             - d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0 e^{-x}/(1 + e^{-x})|
+     *             + 2|d1/(1 + e^{-x})|
+     *             + 2|d0 d1 e^{-x}/(1 + e^{-x})|
+     *     <= 2|d0| + 2|d0| + 2|d1| + 2|d0 d1|
+     *     <= 4|d0| + 2|d1| + 2|d0 d1|
+     *     <= 6 eps + 2 eps^2.
+     */
+    return 1/(1 + exp(-x));
+  } else {
+    /*
+     * e^{-x} <= eps, so the relative error of 1 from 1/(1
+     * + e^{-x}) is
+     *
+     *     |1/(1 + e^{-x}) - 1|/|1/(1 + e^{-x})|
+     *      = |e^{-x}/(1 + e^{-x})|/|1/(1 + e^{-x})|
+     *      = |e^{-x}|
+     *     <= eps.
+     *
+     * This computation avoids an intermediate overflow
+     * exception, although the effect on the result is
+     * harmless.
+     *
+     * XXX Should maybe raise inexact here.
+     */
+    return 1;
+  }
+}
+
+/**
+ * Compute the logit function: log p/(1 - p).  Defined on [0,1].  Maps
+ * a direct-space probability in [0,1] to a log-odds-space probability
+ * in [-infinity, +infinity].  Inverse of logistic.
+ *
+ * Ill-conditioned near 1/2 and 1; the identity logit(1 - p) =
+ * -logit(p) and the function logithalf(p0) = logit(1/2 + p0) may help
+ * to rearrange a computation for p in [1/(1 + e), 1 - 1/(1 + e)].
+ *
+ * This implementation gives relative error bounded by 10 eps.
+ */
+STATIC double
+logit(double p)
+{
+
+  /* logistic(-1) <= p <= logistic(+1) */
+  if (1/(1 + exp(1)) <= p && p <= 1/(1 + exp(-1))) {
+    /*
+     * For inputs near 1/2, we want to compute log1p(near
+     * 0) rather than log(near 1), so write this as:
+     *
+     * log(p/(1 - p)) = -log((1 - p)/p)
+     * = -log(1 + (1 - p)/p - 1)
+     * = -log(1 + (1 - p - p)/p)
+     * = -log(1 + (1 - 2p)/p).
+     *
+     * Since p = 2p/2 <= 1 <= 2*2p = 4p, the floating-point
+     * evaluation of 1 - 2p is exact; the only error arises
+     * from division and log1p.  First, note that if
+     * logistic(-1) <= p <= logistic(+1), (1 - 2p)/p lies
+     * in the bounds of Lemma 4.
+     *
+     * If division has relative error d0 and log1p has
+     * relative error d1, the outcome is
+     *
+     *     -(1 + d1) log(1 + (1 - 2p) (1 + d0)/p)
+     *     = -(1 + d1) (1 + d') log(1 + (1 - 2p)/p)
+     *     = -(1 + d1 + d' + d1 d') log(1 + (1 - 2p)/p).
+     *
+     * where |d'| < 8|d0| by Lemma 4.  The relative error
+     * is then bounded by
+     *
+     *     |d1 + d' + d1 d'|
+     *     <= |d1| + 8|d0| + 8|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return -log1p((1 - 2*p)/p);
+  } else {
+    /*
+     * For inputs near 0, although 1 - p may be rounded to
+     * 1, it doesn't matter much because the magnitude of
+     * the result is so much larger.  For inputs near 1, we
+     * can compute 1 - p exactly, although the precision on
+     * the input is limited so we won't ever get more than
+     * about 700 for the output.
+     *
+     * If - has relative error d0, / has relative error d1,
+     * and log has relative error d2, then
+     *
+     *     (1 + d2) log((1 + d0) p/[(1 - p)(1 + d1)])
+     *     = (1 + d2) [log(p/(1 - p)) + log((1 + d0)/(1 + d1))]
+     *     = log(p/(1 - p)) + d2 log(p/(1 - p))
+     *       + (1 + d2) log((1 + d0)/(1 + d1))
+     *     = log(p/(1 - p))*[1 + d2 +
+     *         + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))]
+     *
+     * Since 0 <= p < logistic(-1) or logistic(+1) < p <=
+     * 1, we have |log(p/(1 - p))| > 1.  Hence this error
+     * is bounded by
+     *
+     *     |d2 + (1 + d2) log((1 + d0)/(1 + d1))/log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))
+     *                      / log(p/(1 - p))|
+     *     <= |d2| + |(1 + d2) log((1 + d0)/(1 + d1))|
+     *     <= |d2| + 4|(1 + d2) (d0 - d1)|, by Lemma 3,
+     *     <= |d2| + 4|d0 - d1 + d2 d0 - d1 d0|
+     *     <= |d2| + 4|d0| + 4|d1| + 4|d2 d0| + 4|d1 d0|
+     *     <= 9 eps + 8 eps^2.
+     */
+    return log(p/(1 - p));
+  }
+}
+
+/**
+ * Compute the logit function, translated in input by 1/2: logithalf(p)
+ * = logit(1/2 + p).  Defined on [-1/2, 1/2].  Inverse of logistichalf.
+ *
+ * Ill-conditioned near +/-1/2.  If |p0| > 1/2 - 1/(1 + e), it may be
+ * better to compute 1/2 + p0 or -1/2 - p0 and to use logit instead.
+ * This implementation gives relative error bounded by 34 eps.
+ */
+STATIC double
+logithalf(double p0)
+{
+
+  if (fabs(p0) <= 0.5 - 1/(1 + exp(1))) {
+    /*
+     * logit(1/2 + p0)
+     * = log((1/2 + p0)/(1 - (1/2 + p0)))
+     * = log((1/2 + p0)/(1/2 - p0))
+     * = log(1 + (1/2 + p0)/(1/2 - p0) - 1)
+     * = log(1 + (1/2 + p0 - (1/2 - p0))/(1/2 - p0))
+     * = log(1 + (1/2 + p0 - 1/2 + p0)/(1/2 - p0))
+     * = log(1 + 2 p0/(1/2 - p0))
+     *
+     * If the error of subtraction is d0, the error of
+     * division is d1, and the error of log1p is d2, then
+     * what we compute is
+     *
+     *  (1 + d2) log(1 + (1 + d1) 2 p0/[(1 + d0) (1/2 - p0)])
+     *  = (1 + d2) log(1 + (1 + d') 2 p0/(1/2 - p0))
+     *  = (1 + d2) (1 + d'') log(1 + 2 p0/(1/2 - p0))
+     *  = (1 + d2 + d'' + d2 d'') log(1 + 2 p0/(1/2 - p0)),
+     *
+     * where |d'| < 2|d0 - d1| <= 4 eps by Lemma 2, and
+     * |d''| < 8|d'| < 32 eps by Lemma 4 since
+     *
+     *  1/e <= 1 + 2*p0/(1/2 - p0) <= e
+     *
+     * when |p0| <= 1/2 - 1/(1 + e).  Hence the relative
+     * error is bounded by
+     *
+     *  |d2 + d'' + d2 d''|
+     *  <= |d2| + |d''| + |d2 d''|
+     *  <= |d1| + 32 |d0| + 32 |d1 d0|
+     *  <= 33 eps + 32 eps^2.
+     */
+    return log1p(2*p0/(0.5 - p0));
+  } else {
+    /*
+     * We have a choice of computing logit(1/2 + p0) or
+     * -logit(1 - (1/2 + p0)) = -logit(1/2 - p0).  It
+     * doesn't matter which way we do this: either way,
+     * since 1/2 p0 <= 1/2 <= 2 p0, the sum and difference
+     * are computed exactly.  So let's do the one that
+     * skips the final negation.
+     *
+     * The result is
+     *
+     *  (1 + d1) log((1 + d0) (1/2 + p0)/[(1 + d2) (1/2 - p0)])
+     *  = (1 + d1) (1 + log((1 + d0)/(1 + d2))
+     *                  / log((1/2 + p0)/(1/2 - p0)))
+     *    * log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') log((1/2 + p0)/(1/2 - p0))
+     *  = (1 + d') logit(1/2 + p0)
+     *
+     * where
+     *
+     *  d' = d1 + log((1 + d0)/(1 + d2))/logit(1/2 + p0)
+     *       + d1 log((1 + d0)/(1 + d2))/logit(1/2 + p0).
+     *
+     * For |p| > 1/2 - 1/(1 + e), logit(1/2 + p0) > 1.
+     * Provided |d0|, |d2| < 1/4, by Lemma 3 we have
+     *
+     *  |log((1 + d0)/(1 + d2))| <= 4|d0 - d2|.
+     *
+     * Hence the relative error is bounded by
+     *
+     *  |d'| <= |d1| + 4|d0 - d2| + 4|d1| |d0 - d2|
+     *       <= |d1| + 4|d0| + 4|d2| + 4|d1 d0| + 4|d1 d2|
+     *       <= 9 eps + 8 eps^2.
+     */
+    return log((0.5 + p0)/(0.5 - p0));
+  }
+}
+
+/*
+ * The following random_uniform_01 is tailored for IEEE 754 binary64
+ * floating-point or smaller.  It can be adapted to larger
+ * floating-point formats like i387 80-bit or IEEE 754 binary128, but
+ * it may require sampling more bits.
+ */
+CTASSERT(FLT_RADIX == 2);
+CTASSERT(-DBL_MIN_EXP <= 1021);
+CTASSERT(DBL_MANT_DIG <= 53);
+
+/**
+ * Draw a floating-point number in [0, 1] with uniform distribution.
+ *
+ * Note that the probability of returning 0 is less than 2^-1074, so
+ * callers need not check for it.  However, callers that cannot handle
+ * rounding to 1 must deal with that, because it occurs with
+ * probability 2^-54, which is small but nonnegligible.
+ */
+STATIC double
+random_uniform_01(void)
+{
+  uint32_t z, x, hi, lo;
+  double s;
+
+  /*
+   * Draw an exponent, geometrically distributed, but give up if
+   * we get a run of more than 1088 zeros, which really means the
+   * system is broken.
+   */
+  z = 0;
+  while ((x = crypto_fast_rng_get_u32(get_thread_fast_rng())) == 0) {
+    if (z >= 1088)
+      /* Your bit sampler is broken.  Go home.  */
+      return 0;
+    z += 32;
+  }
+  z += clz32(x);
+
+  /*
+   * Pick 32-bit halves of an odd normalized significand.
+   * Picking it odd breaks ties in the subsequent rounding, which
+   * occur only with measure zero in the uniform distribution on
+   * [0, 1].
+   */
+  hi = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x80000000);
+  lo = crypto_fast_rng_get_u32(get_thread_fast_rng()) | UINT32_C(0x00000001);
+
+  /* Round to nearest scaled significand in [2^63, 2^64].  */
+  s = hi*(double)4294967296 + lo;
+
+  /* Rescale into [1/2, 1] and apply exponent in one swell foop.  */
+  return s * ldexp(1, -(64 + z));
+}
+
+/*******************************************************************/
+
+/* Functions for specific probability distributions start here: */
+
+/*
+ * Logistic(mu, sigma) distribution, supported on (-infinity,+infinity)
+ *
+ * This is the uniform distribution on [0,1] mapped into log-odds
+ * space, scaled by sigma and translated by mu.
+ *
+ * pdf(x) = e^{-(x - mu)/sigma} sigma (1 + e^{-(x - mu)/sigma})^2
+ * cdf(x) = 1/(1 + e^{-(x - mu)/sigma}) = logistic((x - mu)/sigma)
+ * sf(x) = 1 - cdf(x) = 1 - logistic((x - mu)/sigma = logistic(-(x - mu)/sigma)
+ * icdf(p) = mu + sigma log p/(1 - p) = mu + sigma logit(p)
+ * isf(p) = mu + sigma log (1 - p)/p = mu - sigma logit(p)
+ */
+
+/**
+ * Compute the CDF of the Logistic(mu, sigma) distribution: the
+ * logistic function.  Well-conditioned for negative inputs and small
+ * positive inputs; ill-conditioned for large positive inputs.
+ */
+STATIC double
+cdf_logistic(double x, double mu, double sigma)
+{
+  return logistic((x - mu)/sigma);
+}
+
+/**
+ * Compute the SF of the Logistic(mu, sigma) distribution: the logistic
+ * function reflected over the y axis.  Well-conditioned for positive
+ * inputs and small negative inputs; ill-conditioned for large negative
+ * inputs.
+ */
+STATIC double
+sf_logistic(double x, double mu, double sigma)
+{
+  return logistic(-(x - mu)/sigma);
+}
+
+/**
+ * Compute the inverse of the CDF of the Logistic(mu, sigma)
+ * distribution: the logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+icdf_logistic(double p, double mu, double sigma)
+{
+  return mu + sigma*logit(p);
+}
+
+/**
+ * Compute the inverse of the SF of the Logistic(mu, sigma)
+ * distribution: the -logit function.  Well-conditioned near 0;
+ * ill-conditioned near 1/2 and 1.
+ */
+STATIC double
+isf_logistic(double p, double mu, double sigma)
+{
+  return mu - sigma*logit(p);
+}
+
+/*
+ * LogLogistic(alpha, beta) distribution, supported on (0, +infinity).
+ *
+ * This is the uniform distribution on [0,1] mapped into odds space,
+ * scaled by positive alpha and shaped by positive beta.
+ *
+ * Equivalent to computing exp of a Logistic(log alpha, 1/beta) sample.
+ * (Name arises because the pdf has LogLogistic(x; alpha, beta) =
+ * Logistic(log x; log alpha, 1/beta) and mathematicians got their
+ * covariance contravariant.)
+ *
+ * pdf(x) = (beta/alpha) (x/alpha)^{beta - 1}/(1 + (x/alpha)^beta)^2
+ *        = (1/e^mu sigma) (x/e^mu)^{1/sigma - 1} /
+ *              (1 + (x/e^mu)^{1/sigma})^2
+ * cdf(x) = 1/(1 + (x/alpha)^-beta) = 1/(1 + (x/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x}/e^mu)^{-1/sigma})
+ *        = 1/(1 + (e^{log x - mu})^{-1/sigma})
+ *        = 1/(1 + e^{-(log x - mu)/sigma})
+ *        = logistic((log x - mu)/sigma)
+ *        = logistic((log x - log alpha)/(1/beta))
+ * sf(x) = 1 - 1/(1 + (x/alpha)^-beta)
+ *       = (x/alpha)^-beta/(1 + (x/alpha)^-beta)
+ *       = 1/((x/alpha)^beta + 1)
+ *       = 1/(1 + (x/alpha)^beta)
+ * icdf(p) = alpha (p/(1 - p))^{1/beta}
+ *         = alpha e^{logit(p)/beta}
+ *         = e^{mu + sigma logit(p)}
+ * isf(p) = alpha ((1 - p)/p)^{1/beta}
+ *        = alpha e^{-logit(p)/beta}
+ *        = e^{mu - sigma logit(p)}
+ */
+
+/**
+ * Compute the CDF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      -beta/[1 + (x/alpha)^{-beta}]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps),
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, around which point it levels off at 1.
+ */
+STATIC double
+cdf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * Let d0 be the error of x/alpha; d1, of pow; d2, of +; and
+   * d3, of the final quotient.  The exponentiation gives
+   *
+   *    ((1 + d0) x/alpha)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + d0)^{-beta}
+   *    = (x/alpha)^{-beta} (1 + (1 + d0)^{-beta} - 1)
+   *    = (x/alpha)^{-beta} (1 + d')
+   *
+   * where d' = (1 + d0)^{-beta} - 1.  If y = (x/alpha)^{-beta},
+   * the denominator is
+   *
+   *    (1 + d2) (1 + (1 + d1) (1 + d') y)
+   *    = (1 + d2) (1 + y + (d1 + d' + d1 d') y)
+   *    = 1 + y + (1 + d2) (d1 + d' + d1 d') y
+   *    = (1 + y) (1 + (1 + d2) (d1 + d' + d1 d') y/(1 + y))
+   *    = (1 + y) (1 + d''),
+   *
+   * where d'' = (1 + d2) (d1 + d' + d1 d') y/(1 + y).  The
+   * final result is
+   *
+   *    (1 + d3) / [(1 + d2) (1 + d'') (1 + y)]
+   *    = (1 + d''') / (1 + y)
+   *
+   * for |d'''| <= 2|d3 - d''| by Lemma 2 as long as |d''| < 1/2
+   * (which may not be the case for very large beta).  This
+   * relative error is therefore bounded by
+   *
+   *    |d'''|
+   *    <= 2|d3 - d''|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d') y/(1 + y)|
+   *    <= 2|d3| + 2|(1 + d2) (d1 + d' + d1 d')|
+   *     = 2|d3| + 2|d1 + d' + d1 d' + d2 d1 + d2 d' + d2 d1 d'|
+   *      <= 2|d3| + 2|d1| + 2|d'| + 2|d1 d'| + 2|d2 d1| + 2|d2 d'|
+   *         + 2|d2 d1 d'|
+   *      <= 4 eps + 2 eps^2 + (2 + 2 eps + 2 eps^2) |d'|.
+   *
+   * Roughly, |d'| = |(1 + d0)^{-beta} - 1| grows like beta eps,
+   * until it levels off at 1.
+   */
+  return 1/(1 + pow(x/alpha, -beta));
+}
+
+/**
+ * Compute the SF of the LogLogistic(alpha, beta) distribution.
+ * Well-conditioned for all x and alpha, and the condition number
+ *
+ *      beta/[1 + (x/alpha)^beta]
+ *
+ * grows linearly with beta.
+ *
+ * Loosely, the relative error of this implementation is bounded by
+ *
+ *      4 eps + 2 eps^2 + O(beta eps)
+ *
+ * so don't bother trying this for beta anywhere near as large as
+ * 1/eps, beyond which point it grows unbounded.
+ */
+STATIC double
+sf_log_logistic(double x, double alpha, double beta)
+{
+  /*
+   * The error analysis here is essentially the same as in
+   * cdf_log_logistic, except that rather than levelling off at
+   * 1, |(1 + d0)^beta - 1| grows unbounded.
+   */
+  return 1/(1 + pow(x/alpha, beta));
+}
+
+/**
+ * Compute the inverse of the CDF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and beta near 0 with
+ * condition number 1/[beta (1 - p)].
+ */
+STATIC double
+icdf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow(p/(1 - p), 1/beta);
+}
+
+/**
+ * Compute the inverse of the SF of the LogLogistic(alpha, beta)
+ * distribution.  Ill-conditioned for p near 1 and for large beta, with
+ * condition number -1/[beta (1 - p)].
+ */
+STATIC double
+isf_log_logistic(double p, double alpha, double beta)
+{
+  return alpha*pow((1 - p)/p, 1/beta);
+}
+
+/*
+ * Weibull(lambda, k) distribution, supported on (0, +infinity).
+ *
+ * pdf(x) = (k/lambda) (x/lambda)^{k - 1} e^{-(x/lambda)^k}
+ * cdf(x) = 1 - e^{-(x/lambda)^k}
+ * icdf(p) = lambda * (-log (1 - p))^{1/k}
+ * sf(x) = e^{-(x/lambda)^k}
+ * isf(p) = lambda * (-log p)^{1/k}
+ */
+
+/**
+ * Compute the CDF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k exp(-(x/lambda)^k)/[exp(-(x/lambda)^k) - 1]
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+cdf_weibull(double x, double lambda, double k)
+{
+  return -expm1(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the SF of the Weibull(lambda, k) distribution.
+ * Well-conditioned for small x and k, and for large lambda --
+ * condition number
+ *
+ *      -k (x/lambda)^k
+ *
+ * grows linearly with k, x^k, and lambda^{-k}.
+ */
+STATIC double
+sf_weibull(double x, double lambda, double k)
+{
+  return exp(-pow(x/lambda, k));
+}
+
+/**
+ * Compute the inverse of the CDF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 1, and for k near 0;
+ * condition number is
+ *
+ *      (p/(1 - p))/(k log(1 - p)).
+ */
+STATIC double
+icdf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log1p(-p), 1/k);
+}
+
+/**
+ * Compute the inverse of the SF of the Weibull(lambda, k)
+ * distribution.  Ill-conditioned for p near 0, and for k near 0;
+ * condition number is
+ *
+ *      1/(k log(p)).
+ */
+STATIC double
+isf_weibull(double p, double lambda, double k)
+{
+  return lambda*pow(-log(p), 1/k);
+}
+
+/*
+ * GeneralizedPareto(mu, sigma, xi), supported on (mu, +infinity) for
+ * nonnegative xi, or (mu, mu - sigma/xi) for negative xi.
+ *
+ * Samples:
+ * = mu - sigma log U, if xi = 0;
+ * = mu + sigma (U^{-xi} - 1)/xi = mu + sigma*expm1(-xi log U)/xi, if xi =/= 0,
+ * where U is uniform on (0,1].
+ * = mu + sigma (e^{xi X} - 1)/xi,
+ * where X has standard exponential distribution.
+ *
+ * pdf(x) = sigma^{-1} (1 + xi (x - mu)/sigma)^{-(1 + 1/xi)}
+ * cdf(x) = 1 - (1 + xi (x - mu)/sigma)^{-1/xi}
+ *        = 1 - e^{-log(1 + xi (x - mu)/sigma)/xi}
+ *        --> 1 - e^{-(x - mu)/sigma}  as  xi --> 0
+ * sf(x) = (1 + xi (x - mu)/sigma)^{-1/xi}
+ *       --> e^{-(x - mu)/sigma}  as  xi --> 0
+ * icdf(p) = mu + sigma*(p^{-xi} - 1)/xi
+ *         = mu + sigma*expm1(-xi log p)/xi
+ *         --> mu + sigma*log p  as  xi --> 0
+ * isf(p) = mu + sigma*((1 - p)^{xi} - 1)/xi
+ *        = mu + sigma*expm1(-xi log1p(-p))/xi
+ *        --> mu + sigma*log1p(-p)  as  xi --> 0
+ */
+
+/**
+ * Compute the CDF of the GeneralizedPareto(mu, sigma, xi)
+ * distribution.  Well-conditioned everywhere.  For standard
+ * distribution (mu=0, sigma=1), condition number
+ *
+ *      (x/(1 + x xi)) / ((1 + x xi)^{1/xi} - 1)
+ *
+ * is bounded by 1, attained only at x = 0.
+ */
+STATIC double
+cdf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  /*
+   * log(1 + xi x_0)/xi
+   * = (-1/xi) \sum_{n=1}^infinity (-xi x_0)^n/n
+   * = (-1/xi) (-xi x_0 + \sum_{n=2}^infinity (-xi x_0)^n/n)
+   * = x_0 - (1/xi) \sum_{n=2}^infinity (-xi x_0)^n/n
+   * = x_0 - x_0 \sum_{n=2}^infinity (-xi x_0)^{n-1}/n
+   * = x_0 (1 - d),
+   *
+   * where d = \sum_{n=2}^infinity (-xi x_0)^{n-1}/n.  If |xi| <
+   * eps/4|x_0|, then
+   *
+   * |d| <= \sum_{n=2}^infinity (eps/4)^{n-1}/n
+   *     <= \sum_{n=2}^infinity (eps/4)^{n-1}
+   *      = \sum_{n=1}^infinity (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^infinity (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2
+   *
+   * for any 0 < eps < 2.  Thus, the relative error of x_0 from
+   * log(1 + xi x_0)/xi is bounded by eps.
+   */
+  if (fabs(xi) < 1e-17/x_0)
+    return -expm1(-x_0);
+  else
+    return -expm1(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the SF of the GeneralizedPareto(mu, sigma, xi) distribution.
+ * For standard distribution (mu=0, sigma=1), ill-conditioned for xi
+ * near 0; condition number
+ *
+ *      -x (1 + x xi)^{(-1 - xi)/xi}/(1 + x xi)^{-1/xi}
+ *      = -x (1 + x xi)^{-1/xi - 1}/(1 + x xi)^{-1/xi}
+ *      = -(x/(1 + x xi)) (1 + x xi)^{-1/xi}/(1 + x xi)^{-1/xi}
+ *      = -x/(1 + x xi)
+ *
+ * is bounded by 1/xi.
+ */
+STATIC double
+sf_genpareto(double x, double mu, double sigma, double xi)
+{
+  double x_0 = (x - mu)/sigma;
+
+  if (fabs(xi) < 1e-17/x_0)
+    return exp(-x_0);
+  else
+    return exp(-log1p(xi*x_0)/xi);
+}
+
+/**
+ * Compute the inverse of the CDF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; condition number is
+ *
+ *      xi (p/(1 - p))/(1 - (1 - p)^xi)
+ */
+STATIC double
+icdf_genpareto(double p, double mu, double sigma, double xi)
+{
+  /*
+   * To compute f(xi) = (U^{-xi} - 1)/xi = (e^{-xi log U} - 1)/xi
+   * for xi near zero (note f(xi) --> -log U as xi --> 0), write
+   * the absolutely convergent Taylor expansion
+   *
+   * f(xi) = (1/xi)*(-xi log U + \sum_{n=2}^infinity (-xi log U)^n/n!
+   *       = -log U + (1/xi)*\sum_{n=2}^infinity (-xi log U)^n/n!
+   *       = -log U + \sum_{n=2}^infinity xi^{n-1} (-log U)^n/n!
+   *       = -log U - log U \sum_{n=2}^infinity (-xi log U)^{n-1}/n!
+   *       = -log U (1 + \sum_{n=2}^infinity (-xi log U)^{n-1}/n!).
+   *
+   * Let d = \sum_{n=2}^infinity (-xi log U)^{n-1}/n!.  What do we
+   * lose if we discard it and use -log U as an approximation to
+   * f(xi)?  If |xi| < eps/-4log U, then
+   *
+   * |d| <= \sum_{n=2}^infinity |xi log U|^{n-1}/n!
+   *     <= \sum_{n=2}^infinity (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^infinity (eps/4)^n
+   *      = (eps/4) \sum_{n=0}^infinity (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/-2log U,
+   * f(xi) = -log U (1 + d) for |d| <= eps/2.  |d| is the
+   * relative error of f(xi) from -log U; from this bound, the
+   * relative error of -log U from f(xi) is at most (eps/2)/(1 -
+   * eps/2) = eps/2 + (eps/2)^2 + (eps/2)^3 + ... < eps for 0 <
+   * eps < 1.  Since -log U < 1000 for all U in (0, 1] in
+   * binary64 floating-point, we can safely cut xi off at 1e-20 <
+   * eps/4000 and attain <1ulp error from series truncation.
+   */
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log1p(-p);
+  else
+    return mu + sigma*expm1(-xi*log1p(-p))/xi;
+}
+
+/**
+ * Compute the inverse of the SF of the GeneralizedPareto(mu, sigma,
+ * xi) distribution.  Ill-conditioned for p near 1; conditon number is
+ *
+ *      -xi/(1 - p^{-xi})
+ */
+STATIC double
+isf_genpareto(double p, double mu, double sigma, double xi)
+{
+  if (fabs(xi) <= 1e-20)
+    return mu - sigma*log(p);
+  else
+    return mu + sigma*expm1(-xi*log(p))/xi;
+}
+
+/*******************************************************************/
+
+/**
+ * Deterministic samplers, parametrized by uniform integer and (0,1]
+ * samples.  No guarantees are made about _which_ mapping from the
+ * integer and (0,1] samples these use; all that is guaranteed is the
+ * distribution of the outputs conditioned on a uniform distribution on
+ * the inputs.  The automatic tests in test_prob_distr.c double-check
+ * the particular mappings we use.
+ *
+ * Beware: Unlike random_uniform_01(), these are not guaranteed to be
+ * supported on all possible outputs.  See Ilya Mironov, `On the
+ * Significance of the Least Significant Bits for Differential
+ * Privacy', for an example of what can go wrong if you try to use
+ * these to conceal information from an adversary but you expose the
+ * specific full-precision floating-point values.
+ *
+ * Note: None of these samplers use rejection sampling; they are all
+ * essentially inverse-CDF transforms with tweaks.  If you were to add,
+ * say, a Gamma sampler with the Marsaglia-Tsang method, you would have
+ * to parametrize it by a potentially infinite stream of uniform (and
+ * perhaps normal) samples rather than a fixed number, which doesn't
+ * make for quite as nice automatic testing as for these.
+ */
+
+/**
+ * Deterministically sample from the interval [a, b], indexed by a
+ * uniform random floating-point number p0 in (0, 1].
+ *
+ * Note that even if p0 is nonzero, the result may be equal to a, if
+ * ulp(a)/2 is nonnegligible, e.g. if a = 1.  For maximum resolution,
+ * arrange |a| <= |b|.
+ */
+STATIC double
+sample_uniform_interval(double p0, double a, double b)
+{
+  /*
+   * XXX Prove that the distribution is, in fact, uniform on
+   * [a,b], particularly around p0 = 1, or at least has very
+   * small deviation from uniform, quantified appropriately
+   * (e.g., like in Monahan 1984, or by KL divergence).  It
+   * almost certainly does but it would be nice to quantify the
+   * error.
+   */
+  if ((a <= 0 && 0 <= b) || (b <= 0 && 0 <= a)) {
+    /*
+     * When ab < 0, (1 - t) a + t b is monotonic, since for
+     * a <= b it is a sum of nondecreasing functions of t,
+     * and for b <= a, of nonincreasing functions of t.
+     * Further, clearly at 0 and 1 it attains a and b,
+     * respectively.  Hence it is bounded within [a, b].
+     */
+    return (1 - p0)*a + p0*b;
+  } else {
+    /*
+     * a + (b - a) t is monotonic -- it is obviously a
+     * nondecreasing function of t for a <= b.  Further, it
+     * attains a at 0, and while it may overshoot b at 1,
+     * we have a
+     *
+     * Theorem.  If 0 <= t < 1, then the floating-point
+     * evaluation of a + (b - a) t is bounded in [a, b].
+     *
+     * Lemma 1.  If 0 <= t < 1 is a floating-point number,
+     * then for any normal floating-point number x except
+     * the smallest in magnitude, |round(x*t)| < |x|.
+     *
+     * Proof.  WLOG, assume x >= 0.  Since the rounding
+     * function and t |---> x*t are nondecreasing, their
+     * composition t |---> round(x*t) is also
+     * nondecreasing, so it suffices to consider the
+     * largest floating-point number below 1, in particular
+     * t = 1 - ulp(1)/2.
+     *
+     * Case I: If x is a power of two, then the next
+     * floating-point number below x is x - ulp(x)/2 = x -
+     * x*ulp(1)/2 = x*(1 - ulp(1)/2) = x*t, so, since x*t
+     * is a floating-point number, multiplication is exact,
+     * and thus round(x*t) = x*t < x.
+     *
+     * Case II: If x is not a power of two, then the
+     * greatest lower bound of real numbers rounded to x is
+     * x - ulp(x)/2 = x - ulp(T(x))/2 = x - T(x)*ulp(1)/2,
+     * where T(X) is the largest power of two below x.
+     * Anything below this bound is rounded to a
+     * floating-point number smaller than x, and x*t = x*(1
+     * - ulp(1)/2) = x - x*ulp(1)/2 < x - T(x)*ulp(1)/2
+     * since T(x) < x, so round(x*t) < x*t < x.  QED.
+     *
+     * Lemma 2.  If x and y are subnormal, then round(x +
+     * y) = x + y.
+     *
+     * Proof.  It is a matter of adding the significands,
+     * since if we treat subnormals as having an implicit
+     * zero bit before the `binary' point, their exponents
+     * are all the same.  There is at most one carry/borrow
+     * bit, which can always be acommodated either in a
+     * subnormal, or, at largest, in the implicit one bit
+     * of a normal.
+     *
+     * Lemma 3.  Let x and y be floating-point numbers.  If
+     * round(x - y) is subnormal or zero, then it is equal
+     * to x - y.
+     *
+     * Proof.  Case I (equal): round(x - y) = 0 iff x = y;
+     * hence if round(x - y) = 0, then round(x - y) = 0 = x
+     * - y.
+     *
+     * Case II (subnormal/subnormal): If x and y are both
+     * subnormal, this follows directly from Lemma 2.
+     *
+     * Case IIIa (normal/subnormal): If x is normal and y
+     * is subnormal, then x and y must share sign, or else
+     * x - y would be larger than x and thus rounded to
+     * normal.  If s is the smallest normal positive
+     * floating-point number, |x| < 2s since by
+     * construction 2s - |y| is normal for all subnormal y.
+     * This means that x and y must have the same exponent,
+     * so the difference is the difference of significands,
+     * which is exact.
+     *
+     * Case IIIb (subnormal/normal): Same as case IIIa for
+     * -(y - x).
+     *
+     * Case IV (normal/normal): If x and y are both normal,
+     * then they must share sign, or else x - y would be
+     * larger than x and thus rounded to normal.  Note that
+     * |y| < 2|x|, for if |y| >= 2|x|, then |x| - |y| <=
+     * -|x| but -|x| is normal like x.  Also, |x|/2 < |y|:
+     * if |x|/2 is subnormal, it must hold because y is
+     * normal; if |x|/2 is normal, then |x|/2 >= s, so
+     * since |x| - |y| < s,
+     *
+     *  |x|/2 = |x| - |x|/2 <= |x| - s <= |y|;
+     *
+     * that is, |x|/2 < |y| < 2|x|, so by the Sterbenz
+     * lemma, round(x - y) = x - y.  QED.
+     *
+     * Proof of theorem.  WLOG, assume 0 <= a <= b.  Since
+     * round(a + round(round(b - a)*t) is nondecreasing in
+     * t and attains a at 0, the lower end of the bound is
+     * trivial; we must show the upper end of the bound
+     * strictly.  It suffices to show this for the largest
+     * floating-point number below 1, namely 1 - ulp(1)/2.
+     *
+     * Case I: round(b - a) is normal.  Then it is at most
+     * the smallest floating-point number above b - a.  By
+     * Lemma 1, round(round(b - a)*t) < round(b - a).
+     * Since the inequality is strict, and since
+     * round(round(b - a)*t) is a floating-point number
+     * below round(b - a), and since there are no
+     * floating-point numbers between b - a and round(b -
+     * a), we must have round(round(b - a)*t) < b - a.
+     * Then since y |---> round(a + y) is nondecreasing, we
+     * must have
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b) = b.
+     *
+     * Case II: round(b - a) is subnormal.  In this case,
+     * Lemma 1 falls apart -- we are not guaranteed the
+     * strict inequality.  However, by Lemma 3, the
+     * difference is exact: round(b - a) = b - a.  Thus,
+     *
+     *  round(a + round(round(b - a)*t))
+     *  <= round(a + round((b - a)*t))
+     *  <= round(a + (b - a))
+     *   = round(b)
+     *   = b,
+     *
+     * QED.
+     */
+
+    /* p0 is restricted to [0,1], but we use >= to silence -Wfloat-equal.  */
+    if (p0 >= 1)
+      return b;
+    return a + (b - a)*p0;
+  }
+}
+
+/**
+ * Deterministically sample from the standard logistic distribution,
+ * indexed by a uniform random 32-bit integer s and uniform random
+ * floating-point numbers t and p0 in (0, 1].
+ */
+STATIC double
+sample_logistic(uint32_t s, double t, double p0)
+{
+  double sign = (s & 1) ? -1 : +1;
+  double r;
+
+  /*
+   * We carve up the interval (0, 1) into subregions to compute
+   * the inverse CDF precisely:
+   *
+   * A = (0, 1/(1 + e)] ---> (-infinity, -1]
+   * B = [1/(1 + e), 1/2] ---> [-1, 0]
+   * C = [1/2, 1 - 1/(1 + e)] ---> [0, 1]
+   * D = [1 - 1/(1 + e), 1) ---> [1, +infinity)
+   *
+   * Cases D and C are mirror images of cases A and B,
+   * respectively, so we choose between them by the sign chosen
+   * by a fair coin toss.  We choose between cases A and B by a
+   * coin toss weighted by
+   *
+   *    2/(1 + e) = 1 - [1/2 - 1/(1 + e)]/(1/2):
+   *
+   * if it comes up heads, scale p0 into a uniform (0, 1/(1 + e)]
+   * sample p; if it comes up tails, scale p0 into a uniform (0,
+   * 1/2 - 1/(1 + e)] sample and compute the inverse CDF of p =
+   * 1/2 - p0.
+   */
+  if (t <= 2/(1 + exp(1))) {
+    /* p uniform in (0, 1/(1 + e)], represented by p.  */
+    p0 /= 1 + exp(1);
+    r = logit(p0);
+  } else {
+    /*
+     * p uniform in [1/(1 + e), 1/2), actually represented
+     * by p0 = 1/2 - p uniform in (0, 1/2 - 1/(1 + e)], so
+     * that p = 1/2 - p.
+     */
+    p0 *= 0.5 - 1/(1 + exp(1));
+    r = logithalf(p0);
+  }
+
+  /*
+   * We have chosen from the negative half of the standard
+   * logistic distribution, which is symmetric with the positive
+   * half.  Now use the sign to choose uniformly between them.
+   */
+  return sign*r;
+}
+
+/**
+ * Deterministically sample from the logistic distribution scaled by
+ * sigma and translated by mu.
+ */
+static double
+sample_logistic_locscale(uint32_t s, double t, double p0, double mu,
+    double sigma)
+{
+
+  return mu + sigma*sample_logistic(s, t, p0);
+}
+
+/**
+ * Deterministically sample from the standard log-logistic
+ * distribution, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_log_logistic(uint32_t s, double p0)
+{
+
+  /*
+   * Carve up the interval (0, 1) into (0, 1/2] and [1/2, 1); the
+   * condition numbers of the icdf and the isf coincide at 1/2.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0) {
+    /* p = p0 in (0, 1/2] */
+    return p0/(1 - p0);
+  } else {
+    /* p = 1 - p0 in [1/2, 1) */
+    return (1 - p0)/p0;
+  }
+}
+
+/**
+ * Deterministically sample from the log-logistic distribution with
+ * scale alpha and shape beta.
+ */
+static double
+sample_log_logistic_scaleshape(uint32_t s, double p0, double alpha,
+    double beta)
+{
+  double x = sample_log_logistic(s, p0);
+
+  return alpha*pow(x, 1/beta);
+}
+
+/**
+ * Deterministically sample from the standard exponential distribution,
+ * indexed by a uniform random 32-bit integer s and a uniform random
+ * floating-point number p0 in (0, 1].
+ */
+static double
+sample_exponential(uint32_t s, double p0)
+{
+  /*
+   * We would like to evaluate log(p) for p near 0, and log1p(-p)
+   * for p near 1.  Simply carve the interval into (0, 1/2] and
+   * [1/2, 1) by a fair coin toss.
+   */
+  p0 *= 0.5;
+  if ((s & 1) == 0)
+    /* p = p0 in (0, 1/2] */
+    return -log(p0);
+  else
+    /* p = 1 - p0 in [1/2, 1) */
+    return -log1p(-p0);
+}
+
+/**
+ * Deterministically sample from a Weibull distribution with scale
+ * lambda and shape k -- just an exponential with a shape parameter in
+ * addition to a scale parameter.  (Yes, lambda really is the scale,
+ * _not_ the rate.)
+ */
+STATIC double
+sample_weibull(uint32_t s, double p0, double lambda, double k)
+{
+
+  return lambda*pow(sample_exponential(s, p0), 1/k);
+}
+
+/**
+ * Deterministically sample from the generalized Pareto distribution
+ * with shape xi, indexed by a uniform random 32-bit integer s and a
+ * uniform random floating-point number p0 in (0, 1].
+ */
+STATIC double
+sample_genpareto(uint32_t s, double p0, double xi)
+{
+  double x = sample_exponential(s, p0);
+
+  /*
+   * Write f(xi) = (e^{xi x} - 1)/xi for xi near zero as the
+   * absolutely convergent Taylor series
+   *
+   * f(x) = (1/xi) (xi x + \sum_{n=2}^infinity (xi x)^n/n!)
+   *      = x + (1/xi) \sum_{n=2}^infinity (xi x)^n/n!
+   *      = x + \sum_{n=2}^infinity xi^{n-1} x^n/n!
+   *      = x + x \sum_{n=2}^infinity (xi x)^{n-1}/n!
+   *      = x (1 + \sum_{n=2}^infinity (xi x)^{n-1}/n!).
+   *
+   * d = \sum_{n=2}^infinity (xi x)^{n-1}/n! is the relative error
+   * of f(x) from x.  If |xi| < eps/4x, then
+   *
+   * |d| <= \sum_{n=2}^infinity |xi x|^{n-1}/n!
+   *     <= \sum_{n=2}^infinity (eps/4)^{n-1}/n!
+   *     <= \sum_{n=1}^infinity (eps/4)
+   *      = (eps/4) \sum_{n=0}^infinity (eps/4)^n
+   *      = (eps/4)/(1 - eps/4)
+   *      < eps/2,
+   *
+   * for any 0 < eps < 2.  Hence, as long as |xi| < eps/2x, f(xi)
+   * = x (1 + d) for |d| <= eps/2, so x = f(xi) (1 + d') for |d'|
+   * <= eps.  What bound should we use for x?
+   *
+   * - If x is exponentially distributed, x > 200 with
+   *   probability below e^{-200} << 2^{-256}, i.e. never.
+   *
+   * - If x is computed by -log(U) for U in (0, 1], x is
+   *   guaranteed to be below 1000 in IEEE 754 binary64
+   *   floating-point.
+   *
+   * We can safely cut xi off at 1e-20 < eps/4000 and attain an
+   * error bounded by 0.5 ulp for this expression.
+   */
+  return (fabs(xi) < 1e-20 ? x : expm1(xi*x)/xi);
+}
+
+/**
+ * Deterministically sample from a generalized Pareto distribution with
+ * shape xi, scaled by sigma and translated by mu.
+ */
+static double
+sample_genpareto_locscale(uint32_t s, double p0, double mu, double sigma,
+    double xi)
+{
+
+  return mu + sigma*sample_genpareto(s, p0, xi);
+}
+
+/**
+ * Deterministically sample from the geometric distribution with
+ * per-trial success probability p.
+ **/
+// clang-format off
+/*
+ * XXX Quantify the error (KL divergence?) of this
+ * ceiling-of-exponential sampler from a true geometric distribution,
+ * which we could get by rejection sampling.  Relevant papers:
+ *
+ *      John F. Monahan, `Accuracy in Random Number Generation',
+ *      Mathematics of Computation 45(172), October 1984, pp. 559--568.
+https://pdfs.semanticscholar.org/aca6/74b96da1df77b2224e8cfc5dd6d61a471632.pdf
+ *      Karl Bringmann and Tobias Friedrich, `Exact and Efficient
+ *      Generation of Geometric Random Variates and Random Graphs', in
+ *      Proceedings of the 40th International Colloaquium on Automata,
+ *      Languages, and Programming -- ICALP 2013, Springer LNCS 7965,
+ *      pp.267--278.
+ *      https://doi.org/10.1007/978-3-642-39206-1_23
+ *      https://people.mpi-inf.mpg.de/~kbringma/paper/2013ICALP-1.pdf
+ */
+// clang-format on
+static double
+sample_geometric(uint32_t s, double p0, double p)
+{
+  double x = sample_exponential(s, p0);
+
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1)
+    return 1;
+
+  return ceil(-x/log1p(-p));
+}
+
+/*******************************************************************/
+
+/** Public API for probability distributions:
+ *
+ *  These are wrapper functions on top of the various probability distribution
+ *  operations using the generic <b>dist</b> structure.
+
+ *  These are the functions that should be used by consumers of this API.
+ */
+
+/** Returns the name of the distribution in <b>dist</b>. */
+const char *
+dist_name(const struct dist_t *dist)
+{
+  return dist->ops->name;
+}
+
+/* Sample a value from <b>dist</b> and return it. */
+double
+dist_sample(const struct dist_t *dist)
+{
+  return dist->ops->sample(dist);
+}
+
+/** Compute the CDF of <b>dist</b> at <b>x</b>. */
+double
+dist_cdf(const struct dist_t *dist, double x)
+{
+  return dist->ops->cdf(dist, x);
+}
+
+/** Compute the SF (Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_sf(const struct dist_t *dist, double x)
+{
+  return dist->ops->sf(dist, x);
+}
+
+/** Compute the iCDF (Inverse CDF) of <b>dist</b> at <b>x</b>. */
+double
+dist_icdf(const struct dist_t *dist, double p)
+{
+  return dist->ops->icdf(dist, p);
+}
+
+/** Compute the iSF (Inverse Survival function) of <b>dist</b> at <b>x</b>. */
+double
+dist_isf(const struct dist_t *dist, double p)
+{
+  return dist->ops->isf(dist, p);
+}
+
+/** Functions for uniform distribution */
+
+static double
+uniform_sample(const struct dist_t *dist)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  double p0 = random_uniform_01();
+
+  return sample_uniform_interval(p0, U->a, U->b);
+}
+
+static double
+uniform_cdf(const struct dist_t *dist, double x)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  if (x < U->a)
+    return 0;
+  else if (x < U->b)
+    return (x - U->a)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_sf(const struct dist_t *dist, double x)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+
+  if (x > U->b)
+    return 0;
+  else if (x > U->a)
+    return (U->b - x)/(U->b - U->a);
+  else
+    return 1;
+}
+
+static double
+uniform_icdf(const struct dist_t *dist, double p)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->a + w*p) : (U->b - w*(1 - p)));
+}
+
+static double
+uniform_isf(const struct dist_t *dist, double p)
+{
+  const struct uniform_t *U = dist_to_const_uniform(dist);
+  double w = U->b - U->a;
+
+  return (p < 0.5 ? (U->b - w*p) : (U->a + w*(1 - p)));
+}
+
+const struct dist_ops_t uniform_ops = {
+  .name = "uniform",
+  .sample = uniform_sample,
+  .cdf = uniform_cdf,
+  .sf = uniform_sf,
+  .icdf = uniform_icdf,
+  .isf = uniform_isf,
+};
+
+/*******************************************************************/
+
+/** Private functions for each probability distribution. */
+
+/** Functions for logistic distribution: */
+
+static double
+logistic_sample(const struct dist_t *dist)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double t = random_uniform_01();
+  double p0 = random_uniform_01();
+
+  return sample_logistic_locscale(s, t, p0, L->mu, L->sigma);
+}
+
+static double
+logistic_cdf(const struct dist_t *dist, double x)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return cdf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_sf(const struct dist_t *dist, double x)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return sf_logistic(x, L->mu, L->sigma);
+}
+
+static double
+logistic_icdf(const struct dist_t *dist, double p)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return icdf_logistic(p, L->mu, L->sigma);
+}
+
+static double
+logistic_isf(const struct dist_t *dist, double p)
+{
+  const struct logistic_t *L = dist_to_const_logistic(dist);
+  return isf_logistic(p, L->mu, L->sigma);
+}
+
+const struct dist_ops_t logistic_ops = {
+  .name = "logistic",
+  .sample = logistic_sample,
+  .cdf = logistic_cdf,
+  .sf = logistic_sf,
+  .icdf = logistic_icdf,
+  .isf = logistic_isf,
+};
+
+/** Functions for log-logistic distribution: */
+
+static double
+log_logistic_sample(const struct dist_t *dist)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_log_logistic_scaleshape(s, p0, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_cdf(const struct dist_t *dist, double x)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return cdf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_sf(const struct dist_t *dist, double x)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return sf_log_logistic(x, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_icdf(const struct dist_t *dist, double p)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return icdf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+static double
+log_logistic_isf(const struct dist_t *dist, double p)
+{
+  const struct log_logistic_t *LL = dist_to_const_log_logistic(dist);
+  return isf_log_logistic(p, LL->alpha, LL->beta);
+}
+
+const struct dist_ops_t log_logistic_ops = {
+  .name = "log logistic",
+  .sample = log_logistic_sample,
+  .cdf = log_logistic_cdf,
+  .sf = log_logistic_sf,
+  .icdf = log_logistic_icdf,
+  .isf = log_logistic_isf,
+};
+
+/** Functions for Weibull distribution */
+
+static double
+weibull_sample(const struct dist_t *dist)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_weibull(s, p0, W->lambda, W->k);
+}
+
+static double
+weibull_cdf(const struct dist_t *dist, double x)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return cdf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_sf(const struct dist_t *dist, double x)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return sf_weibull(x, W->lambda, W->k);
+}
+
+static double
+weibull_icdf(const struct dist_t *dist, double p)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return icdf_weibull(p, W->lambda, W->k);
+}
+
+static double
+weibull_isf(const struct dist_t *dist, double p)
+{
+  const struct weibull_t *W = dist_to_const_weibull(dist);
+  return isf_weibull(p, W->lambda, W->k);
+}
+
+const struct dist_ops_t weibull_ops = {
+  .name = "Weibull",
+  .sample = weibull_sample,
+  .cdf = weibull_cdf,
+  .sf = weibull_sf,
+  .icdf = weibull_icdf,
+  .isf = weibull_isf,
+};
+
+/** Functions for generalized Pareto distributions */
+
+static double
+genpareto_sample(const struct dist_t *dist)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_genpareto_locscale(s, p0, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_cdf(const struct dist_t *dist, double x)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return cdf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_sf(const struct dist_t *dist, double x)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return sf_genpareto(x, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_icdf(const struct dist_t *dist, double p)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return icdf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+static double
+genpareto_isf(const struct dist_t *dist, double p)
+{
+  const struct genpareto_t *GP = dist_to_const_genpareto(dist);
+  return isf_genpareto(p, GP->mu, GP->sigma, GP->xi);
+}
+
+const struct dist_ops_t genpareto_ops = {
+  .name = "generalized Pareto",
+  .sample = genpareto_sample,
+  .cdf = genpareto_cdf,
+  .sf = genpareto_sf,
+  .icdf = genpareto_icdf,
+  .isf = genpareto_isf,
+};
+
+/** Functions for geometric distribution on number of trials before success */
+
+static double
+geometric_sample(const struct dist_t *dist)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+  uint32_t s = crypto_fast_rng_get_u32(get_thread_fast_rng());
+  double p0 = random_uniform_01();
+
+  return sample_geometric(s, p0, G->p);
+}
+
+static double
+geometric_cdf(const struct dist_t *dist, double x)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* 1 - (1 - p)^floor(x) = 1 - e^{floor(x) log(1 - p)} */
+  return -expm1(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_sf(const struct dist_t *dist, double x)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  if (x < 1)
+    return 0;
+  /* (1 - p)^floor(x) = e^{ceil(x) log(1 - p)} */
+  return exp(floor(x)*log1p(-G->p));
+}
+
+static double
+geometric_icdf(const struct dist_t *dist, double p)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  return log1p(-p)/log1p(-G->p);
+}
+
+static double
+geometric_isf(const struct dist_t *dist, double p)
+{
+  const struct geometric_t *G = dist_to_const_geometric(dist);
+
+  return log(p)/log1p(-G->p);
+}
+
+const struct dist_ops_t geometric_ops = {
+  .name = "geometric (1-based)",
+  .sample = geometric_sample,
+  .cdf = geometric_cdf,
+  .sf = geometric_sf,
+  .icdf = geometric_icdf,
+  .isf = geometric_isf,
+};
diff --git a/src/lib/math/prob_distr.h b/src/lib/math/prob_distr.h
new file mode 100644
index 0000000000..a036073b93
--- /dev/null
+++ b/src/lib/math/prob_distr.h
@@ -0,0 +1,253 @@
+
+/**
+ * \file prob_distr.h
+ *
+ * \brief Header for prob_distr.c
+ **/
+
+#ifndef TOR_PROB_DISTR_H
+#define TOR_PROB_DISTR_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/cc/torint.h"
+#include "lib/testsupport/testsupport.h"
+
+/**
+ * Container for distribution parameters for sampling, CDF, &c.
+ */
+struct dist_t {
+  const struct dist_ops_t *ops;
+};
+
+/**
+ * Untyped initializer element for struct dist_t using the specified
+ * struct dist_ops_t pointer.  Don't actually use this directly -- use
+ * the type-specific macro built out of DIST_BASE_TYPED below -- but if
+ * you did use this directly, it would be something like:
+ *
+ *   struct weibull mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     .lambda = ...,
+ *     .k = ...,
+ *   };
+ *
+ * Note there is NO COMPILER FEEDBACK if you accidentally do something
+ * like
+ *
+ *   struct geometric mydist = {
+ *     DIST_BASE(&weibull_ops),
+ *     ...
+ *   };
+ */
+#define DIST_BASE(OPS)  { .ops = (OPS) }
+
+/** A compile-time type-checking macro for use with DIST_BASE_TYPED.
+ *
+ *  This macro works by checking that &OBJ is a pointer type that is the same
+ *  type (except for qualifiers) as (const TYPE *)&OBJ. It's a C constraint
+ *  violation (which requires a diagnostic) if two pointers are different types
+ *  and are subtracted. The sizeof() forces compile-time evaluation, and the
+ *  multiplication by zero is to discard the result of the sizeof() from the
+ *  expression.
+ *
+ *  We define this conditionally to suppress false positives from
+ *  Coverity, which gets confused by the sizeof business.
+ */
+#ifdef __COVERITY__
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) 0
+#else
+#define TYPE_CHECK_OBJ(OPS, OBJ, TYPE) \
+  (0*sizeof(&(OBJ) - (const TYPE *)&(OBJ)))
+#endif /* defined(__COVERITY__) */
+
+/**
+* Typed initializer element for struct dist_t using the specified struct
+* dist_ops_t pointer.  Don't actually use this directly -- use a
+* type-specific macro built out of it -- but if you did use this
+* directly, it would be something like:
+*
+*     struct weibull mydist = {
+*       DIST_BASE_TYPED(&weibull_ops, mydist, struct weibull_t),
+*       .lambda = ...,
+*       .k = ...,
+*     };
+*
+* If you want to define a distribution type, define a canonical set of
+* operations and define a type-specific initializer element like so:
+*
+*     struct foo_t {
+*       struct dist_t base;
+*       int omega;
+*       double tau;
+*       double phi;
+*     };
+*
+*     struct dist_ops_t foo_ops = ...;
+*
+*     #define FOO(OBJ) DIST_BASE_TYPED(&foo_ops, OBJ, struct foo_t)
+*
+* Then users can do:
+*
+*     struct foo_t mydist = {
+*       FOO(mydist),
+*       .omega = ...,
+*       .tau = ...,
+*       .phi = ...,
+*     };
+*
+* If you accidentally write
+*
+*     struct bar_t mydist = {
+*       FOO(mydist),
+*       ...
+*     };
+*
+* then the compiler will report a type mismatch in the sizeof
+* expression, which otherwise evaporates at runtime.
+*/
+#define DIST_BASE_TYPED(OPS, OBJ, TYPE)                         \
+  DIST_BASE((OPS) + TYPE_CHECK_OBJ(OPS,OBJ,TYPE))
+
+/**
+ * Generic operations on distributions.  These simply defer to the
+ * corresponding dist_ops_t function.  In the parlance of C++, these call
+ * virtual member functions.
+ */
+const char *dist_name(const struct dist_t *);
+double dist_sample(const struct dist_t *);
+double dist_cdf(const struct dist_t *, double x);
+double dist_sf(const struct dist_t *, double x);
+double dist_icdf(const struct dist_t *, double p);
+double dist_isf(const struct dist_t *, double p);
+
+/**
+ * Set of operations on a potentially parametric family of
+ * distributions.  In the parlance of C++, this would be called a
+ * `vtable' and the members are virtual member functions.
+ */
+struct dist_ops_t {
+  const char *name;
+  double (*sample)(const struct dist_t *);
+  double (*cdf)(const struct dist_t *, double x);
+  double (*sf)(const struct dist_t *, double x);
+  double (*icdf)(const struct dist_t *, double p);
+  double (*isf)(const struct dist_t *, double p);
+};
+
+/* Geometric distribution on positive number of trials before first success */
+
+struct geometric_t {
+  struct dist_t base;
+  double p; /* success probability */
+};
+
+extern const struct dist_ops_t geometric_ops;
+
+#define GEOMETRIC(OBJ)                                      \
+  DIST_BASE_TYPED(&geometric_ops, OBJ, struct geometric_t)
+
+/* Pareto distribution */
+
+struct genpareto_t {
+  struct dist_t base;
+  double mu;
+  double sigma;
+  double xi;
+};
+
+extern const struct dist_ops_t genpareto_ops;
+
+#define GENPARETO(OBJ)                                      \
+  DIST_BASE_TYPED(&genpareto_ops, OBJ, struct genpareto_t)
+
+/* Weibull distribution */
+
+struct weibull_t {
+  struct dist_t base;
+  double lambda;
+  double k;
+};
+
+extern const struct dist_ops_t weibull_ops;
+
+#define WEIBULL(OBJ)                                    \
+  DIST_BASE_TYPED(&weibull_ops, OBJ, struct weibull_t)
+
+/* Log-logistic distribution */
+
+struct log_logistic_t {
+  struct dist_t base;
+  double alpha;
+  double beta;
+};
+
+extern const struct dist_ops_t log_logistic_ops;
+
+#define LOG_LOGISTIC(OBJ)                                       \
+  DIST_BASE_TYPED(&log_logistic_ops, OBJ, struct log_logistic_t)
+
+/* Logistic distribution */
+
+struct logistic_t {
+  struct dist_t base;
+  double mu;
+  double sigma;
+};
+
+extern const struct dist_ops_t logistic_ops;
+
+#define LOGISTIC(OBJ)                                   \
+  DIST_BASE_TYPED(&logistic_ops, OBJ, struct logistic_t)
+
+/* Uniform distribution */
+
+struct uniform_t {
+  struct dist_t base;
+  double a;
+  double b;
+};
+
+extern const struct dist_ops_t uniform_ops;
+
+#define UNIFORM(OBJ)                                    \
+  DIST_BASE_TYPED(&uniform_ops, OBJ, struct uniform_t)
+
+/** Only by unittests */
+
+#ifdef PROB_DISTR_PRIVATE
+
+STATIC double logithalf(double p0);
+STATIC double logit(double p);
+
+STATIC double random_uniform_01(void);
+
+STATIC double logistic(double x);
+STATIC double cdf_logistic(double x, double mu, double sigma);
+STATIC double sf_logistic(double x, double mu, double sigma);
+STATIC double icdf_logistic(double p, double mu, double sigma);
+STATIC double isf_logistic(double p, double mu, double sigma);
+STATIC double sample_logistic(uint32_t s, double t, double p0);
+
+STATIC double cdf_log_logistic(double x, double alpha, double beta);
+STATIC double sf_log_logistic(double x, double alpha, double beta);
+STATIC double icdf_log_logistic(double p, double alpha, double beta);
+STATIC double isf_log_logistic(double p, double alpha, double beta);
+STATIC double sample_log_logistic(uint32_t s, double p0);
+
+STATIC double cdf_weibull(double x, double lambda, double k);
+STATIC double sf_weibull(double x, double lambda, double k);
+STATIC double icdf_weibull(double p, double lambda, double k);
+STATIC double isf_weibull(double p, double lambda, double k);
+STATIC double sample_weibull(uint32_t s, double p0, double lambda, double k);
+
+STATIC double sample_uniform_interval(double p0, double a, double b);
+
+STATIC double cdf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double sf_genpareto(double x, double mu, double sigma, double xi);
+STATIC double icdf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double isf_genpareto(double p, double mu, double sigma, double xi);
+STATIC double sample_genpareto(uint32_t s, double p0, double xi);
+
+#endif /* defined(PROB_DISTR_PRIVATE) */
+
+#endif /* !defined(TOR_PROB_DISTR_H) */
diff --git a/src/lib/memarea/.may_include b/src/lib/memarea/.may_include
index 814652a93c..a1edaf2231 100644
--- a/src/lib/memarea/.may_include
+++ b/src/lib/memarea/.may_include
@@ -1,7 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
-lib/container/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/memarea/*.h
+lib/smartlist_core/*.h
\ No newline at end of file
diff --git a/src/lib/memarea/include.am b/src/lib/memarea/include.am
index 94343dcead..83fb99ec73 100644
--- a/src/lib/memarea/include.am
+++ b/src/lib/memarea/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-memarea-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_memarea_a_SOURCES =			\
 	src/lib/memarea/memarea.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_memarea_testing_a_SOURCES = \
 src_lib_libtor_memarea_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_memarea_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/memarea/memarea.h
diff --git a/src/lib/memarea/lib_memarea.md b/src/lib/memarea/lib_memarea.md
new file mode 100644
index 0000000000..fe5cb8293f
--- /dev/null
+++ b/src/lib/memarea/lib_memarea.md
@@ -0,0 +1,28 @@
+@dir /lib/memarea
+@brief lib/memarea: A fast arena-style allocator.
+
+This module has a fast "arena" style allocator, where memory is freed all at
+once.  This kind of allocation is very fast and avoids fragmentation, at the
+expense of requiring all the data to be freed at the same time.  We use this
+for parsing and diff calculations.
+
+It's often handy to allocate a large number of tiny objects, all of which
+need to disappear at the same time.  You can do this in tor using the
+memarea.c abstraction, which uses a set of grow-only buffers for allocation,
+and only supports a single "free" operation at the end.
+
+Using memareas also helps you avoid memory fragmentation.  You see, some libc
+malloc implementations perform badly on the case where a large number of
+small temporary objects are allocated at the same time as a few long-lived
+objects of similar size.  But if you use tor_malloc() for the long-lived ones
+and a memarea for the temporary object, the malloc implementation is likelier
+to do better.
+
+To create a new memarea, use `memarea_new()`.  To drop all the storage from a
+memarea, and invalidate its pointers, use `memarea_drop_all()`.
+
+The allocation functions `memarea_alloc()`, `memarea_alloc_zero()`,
+`memarea_memdup()`, `memarea_strdup()`, and `memarea_strndup()` are analogous
+to the similarly-named malloc() functions.  There is intentionally no
+`memarea_free()` or `memarea_realloc()`.
+
diff --git a/src/lib/memarea/memarea.c b/src/lib/memarea/memarea.c
index 486673116c..4d26c20eeb 100644
--- a/src/lib/memarea/memarea.c
+++ b/src/lib/memarea/memarea.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2019, The Tor Project, Inc. */
+/* Copyright (c) 2008-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/torint.h"
-#include "lib/container/smartlist.h"
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
@@ -38,7 +39,7 @@
 #elif MEMAREA_ALIGN == 8
 #define MEMAREA_ALIGN_MASK ((uintptr_t)7)
 #else
-#error "void* is neither 4 nor 8 bytes long. I don't know how to align stuff."
+#error "void* is neither 4 nor 8 bytes long."
 #endif /* MEMAREA_ALIGN == 4 || ... */
 
 #if defined(__GNUC__) && defined(FLEXIBLE_ARRAY_MEMBER)
@@ -67,7 +68,7 @@
   uint32_t sent_val = get_uint32(&(chunk)->U_MEM[chunk->mem_size]);     \
   tor_assert(sent_val == SENTINEL_VAL);                                 \
   STMT_END
-#else /* !(defined(USE_SENTINELS)) */
+#else /* !defined(USE_SENTINELS) */
 #define SENTINEL_LEN 0
 #define SET_SENTINEL(chunk) STMT_NIL
 #define CHECK_SENTINEL(chunk) STMT_NIL
@@ -314,7 +315,7 @@ memarea_assert_ok(memarea_t *area)
   }
 }
 
-#else /* !(!defined(DISABLE_MEMORY_SENTINELS)) */
+#else /* defined(DISABLE_MEMORY_SENTINELS) */
 
 struct memarea_t {
   smartlist_t *pieces;
diff --git a/src/lib/memarea/memarea.h b/src/lib/memarea/memarea.h
index 9c23cf62e9..8b5e63e6b3 100644
--- a/src/lib/memarea/memarea.h
+++ b/src/lib/memarea/memarea.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2008-2019, The Tor Project, Inc. */
+/* Copyright (c) 2008-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,9 @@ typedef struct memarea_t memarea_t;
 
 memarea_t *memarea_new(void);
 void memarea_drop_all_(memarea_t *area);
+/** @copydoc memarea_drop_all_
+ *
+ * Additionally, set <b>area</b> to NULL. */
 #define memarea_drop_all(area) \
   do {                                          \
     memarea_drop_all_(area);                    \
diff --git a/src/lib/meminfo/include.am b/src/lib/meminfo/include.am
index d1fdde6313..12c1bff72d 100644
--- a/src/lib/meminfo/include.am
+++ b/src/lib/meminfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-meminfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_meminfo_a_SOURCES =			\
 	src/lib/meminfo/meminfo.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_meminfo_testing_a_SOURCES = \
 src_lib_libtor_meminfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_meminfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/meminfo/meminfo.h
diff --git a/src/lib/meminfo/lib_meminfo.md b/src/lib/meminfo/lib_meminfo.md
new file mode 100644
index 0000000000..87f509d648
--- /dev/null
+++ b/src/lib/meminfo/lib_meminfo.md
@@ -0,0 +1,5 @@
+@dir /lib/meminfo
+@brief lib/meminfo: Inspecting malloc() usage.
+
+Only available when malloc() provides mallinfo() or something similar.
+
diff --git a/src/lib/meminfo/meminfo.c b/src/lib/meminfo/meminfo.c
index bc3eef8419..b7d991e410 100644
--- a/src/lib/meminfo/meminfo.c
+++ b/src/lib/meminfo/meminfo.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/meminfo/meminfo.h b/src/lib/meminfo/meminfo.h
index 6f29fe0ea8..36a85c133d 100644
--- a/src/lib/meminfo/meminfo.h
+++ b/src/lib/meminfo/meminfo.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@
 
 MOCK_DECL(int, get_total_system_memory, (size_t *mem_out));
 
-#endif
+#endif /* !defined(TOR_MEMINFO_H) */
diff --git a/src/lib/net/.may_include b/src/lib/net/.may_include
index 13b209bbed..6e9af9737a 100644
--- a/src/lib/net/.may_include
+++ b/src/lib/net/.may_include
@@ -1,8 +1,9 @@
 orconfig.h
-siphash.h
-ht.h
+ext/siphash.h
+ext/ht.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
@@ -11,5 +12,7 @@ lib/lock/*.h
 lib/log/*.h
 lib/net/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
-lib/malloc/*.h
\ No newline at end of file
+lib/malloc/*.h
+lib/smartlist_core/*.h
diff --git a/src/lib/net/address.c b/src/lib/net/address.c
index 69004ddb0e..6d46f9b955 100644
--- a/src/lib/net/address.c
+++ b/src/lib/net/address.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -40,6 +40,7 @@
 
 #include "lib/net/address.h"
 #include "lib/net/socket.h"
+#include "lib/cc/ctassert.h"
 #include "lib/container/smartlist.h"
 #include "lib/ctime/di_ops.h"
 #include "lib/log/log.h"
@@ -52,7 +53,7 @@
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>
@@ -96,8 +97,9 @@
  * work correctly. Bail out here if we've found a platform where AF_UNSPEC
  * isn't 0. */
 #if AF_UNSPEC != 0
-#error We rely on AF_UNSPEC being 0. Let us know about your platform, please!
+#error "We rely on AF_UNSPEC being 0. Yours isn't. Please tell us more!"
 #endif
+CTASSERT(AF_UNSPEC == 0);
 
 /** Convert the tor_addr_t in <b>a</b>, with port in <b>port</b>, into a
  * sockaddr object in *<b>sa_out</b> of object size <b>len</b>.  If not enough
@@ -337,7 +339,7 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
       break;
     case AF_INET6:
       /* Shortest addr [ :: ] + \0 */
-      if (len < (3 + (decorate ? 2u : 0u)))
+      if (len < (3u + (decorate ? 2 : 0)))
         return NULL;
 
       if (decorate)
@@ -371,7 +373,8 @@ tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len, int decorate)
  *
  * If <b>accept_regular</b> is set and the address is in neither recognized
  * reverse lookup hostname format, try parsing the address as a regular
- * IPv4 or IPv6 address too.
+ * IPv4 or IPv6 address too. This mode will accept IPv6 addresses with or
+ * without square brackets.
  */
 int
 tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
@@ -605,7 +608,8 @@ tor_addr_parse_mask_ports(const char *s,
         family = AF_INET;
         tor_addr_from_ipv4h(addr_out, 0);
       } else if (flags & TAPMP_STAR_IPV6_ONLY) {
-        static char nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
+        static uint8_t nil_bytes[16] =
+          { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
         family = AF_INET6;
         tor_addr_from_ipv6_bytes(addr_out, nil_bytes);
       } else {
@@ -626,7 +630,7 @@ tor_addr_parse_mask_ports(const char *s,
     tor_addr_from_ipv4h(addr_out, 0);
     any_flag = 1;
   } else if (!strcmp(address, "*6") && (flags & TAPMP_EXTENDED_STAR)) {
-    static char nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
+    static uint8_t nil_bytes[16] = { [0]=0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 };
     family = AF_INET6;
     tor_addr_from_ipv6_bytes(addr_out, nil_bytes);
     any_flag = 1;
@@ -814,8 +818,12 @@ tor_addr_is_loopback(const tor_addr_t *addr)
 
 /* Is addr valid?
  * Checks that addr is non-NULL and not tor_addr_is_null().
- * If for_listening is true, IPv4 addr 0.0.0.0 is allowed.
- * It means "bind to all addresses on the local machine". */
+ * If for_listening is true, all IPv4 and IPv6 addresses are valid, including
+ * 0.0.0.0 (for IPv4) and :: (for IPv6). When listening, these addresses mean
+ * "bind to all addresses on the local machine".
+ * Otherwise, 0.0.0.0 and ::  are invalid, because they are null addresses.
+ * All unspecified and unix addresses are invalid, regardless of for_listening.
+ */
 int
 tor_addr_is_valid(const tor_addr_t *addr, int for_listening)
 {
@@ -824,10 +832,11 @@ tor_addr_is_valid(const tor_addr_t *addr, int for_listening)
     return 0;
   }
 
-  /* Only allow IPv4 0.0.0.0 for_listening. */
-  if (for_listening && addr->family == AF_INET
-      && tor_addr_to_ipv4h(addr) == 0) {
-    return 1;
+  /* Allow all IPv4 and IPv6 addresses, when for_listening is true */
+  if (for_listening) {
+    if (addr->family == AF_INET || addr->family == AF_INET6) {
+      return 1;
+    }
   }
 
   /* Otherwise, the address is valid if it's not tor_addr_is_null() */
@@ -879,7 +888,7 @@ tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr)
 /** Set <b>dest</b> to equal the IPv6 address in the 16 bytes at
  * <b>ipv6_bytes</b>. */
 void
-tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *ipv6_bytes)
+tor_addr_from_ipv6_bytes(tor_addr_t *dest, const uint8_t *ipv6_bytes)
 {
   tor_assert(dest);
   tor_assert(ipv6_bytes);
@@ -892,7 +901,21 @@ tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *ipv6_bytes)
 void
 tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6)
 {
-  tor_addr_from_ipv6_bytes(dest, (const char*)in6->s6_addr);
+  tor_addr_from_ipv6_bytes(dest, in6->s6_addr);
+}
+
+/** Set the 16 bytes at <b>dest</b> to equal the IPv6 address <b>src</b>.
+ * <b>src</b> must be an IPv6 address, if it is not, log a warning, and clear
+ * <b>dest</b>. */
+void
+tor_addr_copy_ipv6_bytes(uint8_t *dest, const tor_addr_t *src)
+{
+  tor_assert(dest);
+  tor_assert(src);
+  memset(dest, 0, 16);
+  IF_BUG_ONCE(src->family != AF_INET6)
+    return;
+  memcpy(dest, src->addr.in6_addr.s6_addr, 16);
 }
 
 /** Copy a tor_addr_t from <b>src</b> to <b>dest</b>.
@@ -1166,57 +1189,137 @@ fmt_addr_impl(const tor_addr_t *addr, int decorate)
 const char *
 fmt_addrport(const tor_addr_t *addr, uint16_t port)
 {
-  /* Add space for a colon and up to 5 digits. */
-  static char buf[TOR_ADDR_BUF_LEN + 6];
+  static char buf[TOR_ADDRPORT_BUF_LEN];
   tor_snprintf(buf, sizeof(buf), "%s:%u", fmt_and_decorate_addr(addr), port);
   return buf;
 }
 
 /** Like fmt_addr(), but takes <b>addr</b> as a host-order IPv4
  * addresses. Also not thread-safe, also clobbers its return buffer on
- * repeated calls. */
+ * repeated calls. Clean internal buffer and return empty string on failure. */
 const char *
 fmt_addr32(uint32_t addr)
 {
   static char buf[INET_NTOA_BUF_LEN];
   struct in_addr in;
+  int success;
+
   in.s_addr = htonl(addr);
-  tor_inet_ntoa(&in, buf, sizeof(buf));
+
+  success = tor_inet_ntoa(&in, buf, sizeof(buf));
+  tor_assertf_nonfatal(success >= 0,
+      "Failed to convert IP 0x%08X (HBO) to string", addr);
+
+  IF_BUG_ONCE(success < 0) {
+    memset(buf, 0, INET_NTOA_BUF_LEN);
+  }
+
   return buf;
 }
 
+/** Return a string representing the family of <b>addr</b>.
+ *
+ * This string is a string constant, and must not be freed.
+ * This function is thread-safe.
+ */
+const char *
+fmt_addr_family(const tor_addr_t *addr)
+{
+  static int default_bug_once = 0;
+
+  IF_BUG_ONCE(!addr)
+    return "NULL pointer";
+
+  switch (tor_addr_family(addr)) {
+    case AF_INET6:
+      return "IPv6";
+    case AF_INET:
+      return "IPv4";
+    case AF_UNIX:
+      return "UNIX socket";
+    case AF_UNSPEC:
+      return "unspecified";
+    default:
+      if (!default_bug_once) {
+        log_warn(LD_BUG, "Called with unknown address family %d",
+                 (int)tor_addr_family(addr));
+        default_bug_once = 1;
+      }
+      return "unknown";
+  }
+  //return "(unreachable code)";
+}
+
 /** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
- * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
- * square brackets.
+ * may be an IPv4 address, or an IPv6 address surrounded by square brackets.
  *
- *  Return an address family on success, or -1 if an invalid address string is
- *  provided. */
-int
-tor_addr_parse(tor_addr_t *addr, const char *src)
+ * If <b>allow_ipv6_without_brackets</b> is true, also allow IPv6 addresses
+ * without brackets.
+ *
+ * Always rejects IPv4 addresses with brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+static int
+tor_addr_parse_impl(tor_addr_t *addr, const char *src,
+                    bool allow_ipv6_without_brackets)
 {
   /* Holds substring of IPv6 address after removing square brackets */
   char *tmp = NULL;
-  int result;
+  int result = -1;
   struct in_addr in_tmp;
   struct in6_addr in6_tmp;
+  int brackets_detected = 0;
+
   tor_assert(addr && src);
-  if (src[0] == '[' && src[1])
+
+  size_t len = strlen(src);
+
+  if (len && src[0] == '[' && src[len - 1] == ']') {
+    brackets_detected = 1;
     src = tmp = tor_strndup(src+1, strlen(src)-2);
+  }
 
-  if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
-    result = AF_INET6;
-    tor_addr_from_in6(addr, &in6_tmp);
-  } else if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
-    result = AF_INET;
-    tor_addr_from_in(addr, &in_tmp);
-  } else {
-    result = -1;
+  /* Try to parse an IPv6 address if it has brackets, or if IPv6 addresses
+   * without brackets are allowed */
+  if (brackets_detected || allow_ipv6_without_brackets) {
+    if (tor_inet_pton(AF_INET6, src, &in6_tmp) > 0) {
+      result = AF_INET6;
+      tor_addr_from_in6(addr, &in6_tmp);
+    }
+  }
+
+  /* Try to parse an IPv4 address without brackets */
+  if (!brackets_detected) {
+    if (tor_inet_pton(AF_INET, src, &in_tmp) > 0) {
+      result = AF_INET;
+      tor_addr_from_in(addr, &in_tmp);
+    }
+  }
+
+  /* Clear the address on error, to avoid returning uninitialised or partly
+   * parsed data.
+   */
+  if (result == -1) {
+    memset(addr, 0, sizeof(tor_addr_t));
   }
 
   tor_free(tmp);
   return result;
 }
 
+/** Convert the string in <b>src</b> to a tor_addr_t <b>addr</b>.  The string
+ * may be an IPv4 address, an IPv6 address, or an IPv6 address surrounded by
+ * square brackets.
+ *
+ * Returns an address family on success, or -1 if an invalid address string is
+ * provided. */
+int
+tor_addr_parse(tor_addr_t *addr, const char *src)
+{
+  return tor_addr_parse_impl(addr, src, 1);
+}
+
 #ifdef HAVE_IFADDRS_TO_SMARTLIST
 /*
  * Convert a linked list consisting of <b>ifaddrs</b> structures
@@ -1352,7 +1455,7 @@ get_interface_addresses_win32(int severity, sa_family_t family)
 
 /* This is defined on Mac OS X */
 #ifndef _SIZEOF_ADDR_IFREQ
-#define _SIZEOF_ADDR_IFREQ sizeof
+#define _SIZEOF_ADDR_IFREQ(x) sizeof(x)
 #endif
 
 /* Free ifc->ifc_buf safely. */
@@ -1371,10 +1474,10 @@ ifconf_free_ifc_buf(struct ifconf *ifc)
  * into smartlist of <b>tor_addr_t</b> structures.
  */
 STATIC smartlist_t *
-ifreq_to_smartlist(char *buf, size_t buflen)
+ifreq_to_smartlist(const uint8_t *buf, size_t buflen)
 {
   smartlist_t *result = smartlist_new();
-  char *end = buf + buflen;
+  const uint8_t *end = buf + buflen;
 
   /* These acrobatics are due to alignment issues which trigger
    * undefined behaviour traps on OSX. */
@@ -1448,7 +1551,7 @@ get_interface_addresses_ioctl(int severity, sa_family_t family)
     /* Ensure we have least IFREQ_SIZE bytes unused at the end. Otherwise, we
      * don't know if we got everything during ioctl. */
   } while (mult * IFREQ_SIZE - ifc.ifc_len <= IFREQ_SIZE);
-  result = ifreq_to_smartlist(ifc.ifc_buf, ifc.ifc_len);
+  result = ifreq_to_smartlist((const uint8_t *)ifc.ifc_buf, ifc.ifc_len);
 
  done:
   if (fd >= 0)
@@ -1601,11 +1704,15 @@ get_interface_address6,(int severity, sa_family_t family, tor_addr_t *addr))
    * Ideally, we want the default route, see #12377 for details */
   SMARTLIST_FOREACH_BEGIN(addrs, tor_addr_t *, a) {
     tor_addr_copy(addr, a);
+    const bool is_internal = tor_addr_is_internal(a, 0);
     rv = 0;
 
+    log_debug(LD_NET, "Found %s interface address '%s'",
+              (is_internal ? "internal" : "external"), fmt_addr(addr));
+
     /* If we found a non-internal address, declare success.  Otherwise,
      * keep looking. */
-    if (!tor_addr_is_internal(a, 0))
+    if (!is_internal)
       break;
   } SMARTLIST_FOREACH_END(a);
 
@@ -1709,6 +1816,11 @@ get_interface_address6_list,(int severity,
  * form "ip" or "ip:0".  Otherwise, accept those forms, and set
  * *<b>port_out</b> to <b>default_port</b>.
  *
+ * This function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
  * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_parse(int severity, const char *addrport,
@@ -1718,6 +1830,7 @@ tor_addr_port_parse(int severity, const char *addrport,
   int retval = -1;
   int r;
   char *addr_tmp = NULL;
+  bool has_port;
 
   tor_assert(addrport);
   tor_assert(address_out);
@@ -1727,28 +1840,47 @@ tor_addr_port_parse(int severity, const char *addrport,
   if (r < 0)
     goto done;
 
-  if (!*port_out) {
+  has_port = !! *port_out;
+  /* If there's no port, use the default port, or fail if there is no default
+   */
+  if (!has_port) {
     if (default_port >= 0)
       *port_out = default_port;
     else
       goto done;
   }
 
-  /* make sure that address_out is an IP address */
-  if (tor_addr_parse(address_out, addr_tmp) < 0)
+  /* Make sure that address_out is an IP address.
+   * If there is no port in addrport, allow IPv6 addresses without brackets. */
+  if (tor_addr_parse_impl(address_out, addr_tmp, !has_port) < 0)
     goto done;
 
   retval = 0;
 
  done:
+  /* Clear the address and port on error, to avoid returning uninitialised or
+   * partly parsed data.
+   */
+  if (retval == -1) {
+    memset(address_out, 0, sizeof(tor_addr_t));
+    *port_out = 0;
+  }
   tor_free(addr_tmp);
   return retval;
 }
 
 /** Given an address of the form "host[:port]", try to divide it into its host
- * and port portions, setting *<b>address_out</b> to a newly allocated string
- * holding the address portion and *<b>port_out</b> to the port (or 0 if no
- * port is given).  Return 0 on success, -1 on failure. */
+ * and port portions.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Sets *<b>address_out</b> to a newly allocated string holding the address
+ * portion, and *<b>port_out</b> to the port (or 0 if no port is given).
+ *
+ * Return 0 on success, -1 on failure. */
 int
 tor_addr_port_split(int severity, const char *addrport,
                     char **address_out, uint16_t *port_out)
@@ -1757,8 +1889,11 @@ tor_addr_port_split(int severity, const char *addrport,
   tor_assert(addrport);
   tor_assert(address_out);
   tor_assert(port_out);
+
   /* We need to check for IPv6 manually because the logic below doesn't
-   * do a good job on IPv6 addresses that lack a port. */
+   * do a good job on IPv6 addresses that lack a port.
+   * If an IPv6 address without square brackets is ambiguous, it gets parsed
+   * here as an address, rather than address:port. */
   if (tor_addr_parse(&a_tmp, addrport) == AF_INET6) {
     *port_out = 0;
     *address_out = tor_strdup(addrport);
@@ -1798,8 +1933,7 @@ tor_addr_port_split(int severity, const char *addrport,
     tor_free(address_);
   }
 
-  if (port_out)
-    *port_out = ok ? ((uint16_t) port_) : 0;
+  *port_out = ok ? ((uint16_t) port_) : 0;
 
   return ok ? 0 : -1;
 }
@@ -1875,17 +2009,24 @@ parse_port_range(const char *port, uint16_t *port_min_out,
 }
 
 /** Given a host-order <b>addr</b>, call tor_inet_ntop() on it
- *  and return a strdup of the resulting address.
+ *  and return a strdup of the resulting address. Return NULL if
+ *  tor_inet_ntop() fails.
  */
 char *
 tor_dup_ip(uint32_t addr)
 {
+  const char *ip_str;
   char buf[TOR_ADDR_BUF_LEN];
   struct in_addr in;
 
   in.s_addr = htonl(addr);
-  tor_inet_ntop(AF_INET, &in, buf, sizeof(buf));
-  return tor_strdup(buf);
+  ip_str = tor_inet_ntop(AF_INET, &in, buf, sizeof(buf));
+
+  tor_assertf_nonfatal(ip_str, "Failed to duplicate IP %08X", addr);
+  if (ip_str)
+    return tor_strdup(buf);
+
+  return NULL;
 }
 
 /**
@@ -1934,7 +2075,7 @@ tor_addr_port_new(const tor_addr_t *addr, uint16_t port)
   return ap;
 }
 
-/** Return true iff <a>a</b> and <b>b</b> are the same address and port */
+/** Return true iff <b>a</b> and <b>b</b> are the same address and port */
 int
 tor_addr_port_eq(const tor_addr_port_t *a,
                  const tor_addr_port_t *b)
@@ -2018,8 +2159,12 @@ string_is_valid_nonrfc_hostname(const char *string)
 
   smartlist_split_string(components,string,".",0,0);
 
-  if (BUG(smartlist_len(components) == 0))
-    return 0; // LCOV_EXCL_LINE should be impossible given the earlier checks.
+  if (BUG(smartlist_len(components) == 0)) {
+    // LCOV_EXCL_START should be impossible given the earlier checks.
+    smartlist_free(components);
+    return 0;
+    // LCOV_EXCL_STOP
+  }
 
   /* Allow a single terminating '.' used rarely to indicate domains
    * are FQDNs rather than relative. */
diff --git a/src/lib/net/address.h b/src/lib/net/address.h
index 9b826c8359..e5016ee4fe 100644
--- a/src/lib/net/address.h
+++ b/src/lib/net/address.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -62,6 +62,7 @@
 typedef uint8_t maskbits_t;
 
 struct in_addr;
+
 /** Holds an IPv4 or IPv6 address.  (Uses less memory than struct
  * sockaddr_storage.) */
 typedef struct tor_addr_t
@@ -103,6 +104,10 @@ int tor_addr_from_sockaddr(tor_addr_t *a, const struct sockaddr *sa,
                            uint16_t *port_out);
 void tor_addr_make_unspec(tor_addr_t *a);
 void tor_addr_make_null(tor_addr_t *a, sa_family_t family);
+#define tor_addr_port_make_null(addr, port, family) \
+  (void)(tor_addr_make_null(addr, family), (port) = 0)
+#define tor_addr_port_make_null_ap(ap, family) \
+  tor_addr_port_make_null(&(ap)->addr, (ap)->port, family)
 char *tor_sockaddr_to_str(const struct sockaddr *sa);
 
 /** Return an in6_addr* equivalent to <b>a</b>, or NULL if <b>a</b> is not
@@ -133,6 +138,7 @@ tor_addr_to_in6_assert(const tor_addr_t *a)
  * Requires that <b>x</b> is actually an IPv6 address.
  */
 #define tor_addr_to_in6_addr16(x) S6_ADDR16(*tor_addr_to_in6_assert(x))
+
 /** Given an IPv6 address <b>x</b>, yield it as an array of uint32_t.
  *
  * Requires that <b>x</b> is actually an IPv6 address.
@@ -146,6 +152,7 @@ tor_addr_to_ipv4n(const tor_addr_t *a)
 {
   return a->family == AF_INET ? a->addr.in_addr.s_addr : 0;
 }
+
 /** Return an IPv4 address in host order for <b>a</b>, or 0 if
  * <b>a</b> is not an IPv4 address. */
 static inline uint32_t
@@ -153,10 +160,11 @@ tor_addr_to_ipv4h(const tor_addr_t *a)
 {
   return ntohl(tor_addr_to_ipv4n(a));
 }
+
 /** Given an IPv6 address, return its mapped IPv4 address in host order, or
  * 0 if <b>a</b> is not an IPv6 address.
  *
- * (Does not check whether the address is really a mapped address */
+ * (Does not check whether the address is really a mapped address.) */
 static inline uint32_t
 tor_addr_to_mapped_ipv4h(const tor_addr_t *a)
 {
@@ -165,21 +173,21 @@ tor_addr_to_mapped_ipv4h(const tor_addr_t *a)
     // Work around an incorrect NULL pointer dereference warning in
     // "clang --analyze" due to limited analysis depth
     addr32 = tor_addr_to_in6_addr32(a);
-    // To improve performance, wrap this assertion in:
-    // #if !defined(__clang_analyzer__) || PARANOIA
     tor_assert(addr32);
     return ntohl(addr32[3]);
   } else {
     return 0;
   }
 }
+
 /** Return the address family of <b>a</b>.  Possible values are:
- * AF_INET6, AF_INET, AF_UNSPEC. */
+ * AF_INET6, AF_INET, AF_UNSPEC, AF_UNIX. */
 static inline sa_family_t
 tor_addr_family(const tor_addr_t *a)
 {
   return a->family;
 }
+
 /** Return an in_addr* equivalent to <b>a</b>, or NULL if <b>a</b> is not
  * an IPv4 address. */
 static inline const struct in_addr *
@@ -187,6 +195,7 @@ tor_addr_to_in(const tor_addr_t *a)
 {
   return a->family == AF_INET ? &a->addr.in_addr : NULL;
 }
+
 /** Return true iff <b>a</b> is an IPv4 address equal to the host-ordered
  * address in <b>u</b>. */
 static inline int
@@ -204,24 +213,39 @@ tor_addr_eq_ipv4h(const tor_addr_t *a, uint32_t u)
  */
 #define TOR_ADDR_BUF_LEN 48
 
+/** Length of a buffer containing an IP address along with a port number and
+ * a seperating colon.
+ *
+ * This allows enough space for
+ *   "[ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255]:12345",
+ * plus a terminating NUL.
+ */
+#define TOR_ADDRPORT_BUF_LEN (TOR_ADDR_BUF_LEN + 6)
+
 char *tor_addr_to_str_dup(const tor_addr_t *addr) ATTR_MALLOC;
 
 /** Wrapper function of fmt_addr_impl(). It does not decorate IPv6
  *  addresses. */
 #define fmt_addr(a) fmt_addr_impl((a), 0)
+
 /** Wrapper function of fmt_addr_impl(). It decorates IPv6
  *  addresses. */
 #define fmt_and_decorate_addr(a) fmt_addr_impl((a), 1)
+
 const char *fmt_addr_impl(const tor_addr_t *addr, int decorate);
 const char *fmt_addrport(const tor_addr_t *addr, uint16_t port);
-const char * fmt_addr32(uint32_t addr);
+#define fmt_addrport_ap(ap) fmt_addrport(&(ap)->addr, (ap)->port)
+const char *fmt_addr32(uint32_t addr);
+const char *fmt_addr_family(const tor_addr_t *addr);
 
 MOCK_DECL(int,get_interface_address6,(int severity, sa_family_t family,
 tor_addr_t *addr));
+
 struct smartlist_t;
-void interface_address6_list_free_(struct smartlist_t * addrs);// XXXX
+void interface_address6_list_free_(struct smartlist_t * addrs);
 #define interface_address6_list_free(addrs) \
   FREE_AND_NULL(struct smartlist_t, interface_address6_list_free_, (addrs))
+
 MOCK_DECL(struct smartlist_t *,get_interface_address6_list,(int severity,
                                                      sa_family_t family,
                                                      int include_internal));
@@ -246,6 +270,7 @@ int tor_addr_compare_masked(const tor_addr_t *addr1, const tor_addr_t *addr2,
 uint64_t tor_addr_hash(const tor_addr_t *addr);
 struct sipkey;
 uint64_t tor_addr_keyed_hash(const struct sipkey *key, const tor_addr_t *addr);
+
 int tor_addr_is_v4(const tor_addr_t *addr);
 int tor_addr_is_internal_(const tor_addr_t *ip, int for_listening,
                           const char *filename, int lineno);
@@ -276,21 +301,25 @@ int tor_addr_parse_PTR_name(tor_addr_t *result, const char *address,
 int tor_addr_parse_mask_ports(const char *s, unsigned flags,
                               tor_addr_t *addr_out, maskbits_t *mask_out,
                               uint16_t *port_min_out, uint16_t *port_max_out);
+
 const char * tor_addr_to_str(char *dest, const tor_addr_t *addr, size_t len,
                              int decorate);
 int tor_addr_parse(tor_addr_t *addr, const char *src);
 void tor_addr_copy(tor_addr_t *dest, const tor_addr_t *src);
 void tor_addr_copy_tight(tor_addr_t *dest, const tor_addr_t *src);
+
 void tor_addr_from_ipv4n(tor_addr_t *dest, uint32_t v4addr);
 /** Set <b>dest</b> to the IPv4 address encoded in <b>v4addr</b> in host
  * order. */
 #define tor_addr_from_ipv4h(dest, v4addr)       \
   tor_addr_from_ipv4n((dest), htonl(v4addr))
-void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const char *bytes);
+void tor_addr_from_ipv6_bytes(tor_addr_t *dest, const uint8_t *bytes);
 /** Set <b>dest</b> to the IPv4 address incoded in <b>in</b>. */
 #define tor_addr_from_in(dest, in) \
   tor_addr_from_ipv4n((dest), (in)->s_addr);
 void tor_addr_from_in6(tor_addr_t *dest, const struct in6_addr *in6);
+void tor_addr_copy_ipv6_bytes(uint8_t *dest, const tor_addr_t *src);
+
 int tor_addr_is_null(const tor_addr_t *addr);
 int tor_addr_is_loopback(const tor_addr_t *addr);
 
@@ -299,6 +328,7 @@ int tor_addr_is_valid_ipv4n(uint32_t v4n_addr, int for_listening);
 #define tor_addr_is_valid_ipv4h(v4h_addr, for_listening) \
         tor_addr_is_valid_ipv4n(htonl(v4h_addr), (for_listening))
 int tor_port_is_valid(uint16_t port, int for_listening);
+
 /* Are addr and port both valid? */
 #define tor_addr_port_is_valid(addr, port, for_listening) \
         (tor_addr_is_valid((addr), (for_listening)) &&    \
@@ -329,9 +359,11 @@ int parse_port_range(const char *port, uint16_t *port_min_out,
                      uint16_t *port_max_out);
 int addr_mask_get_bits(uint32_t mask);
 char *tor_dup_ip(uint32_t addr) ATTR_MALLOC;
+
 MOCK_DECL(int,get_interface_address,(int severity, uint32_t *addr));
 #define interface_address_list_free(lst)\
   interface_address6_list_free(lst)
+
 /** Return a smartlist of the IPv4 addresses of all interfaces on the server.
  * Excludes loopback and multicast addresses. Only includes internal addresses
  * if include_internal is true. (Note that a relay behind NAT may use an
@@ -377,8 +409,8 @@ STATIC struct smartlist_t *get_interface_addresses_win32(int severity,
 #endif /* defined(HAVE_IP_ADAPTER_TO_SMARTLIST) */
 
 #ifdef HAVE_IFCONF_TO_SMARTLIST
-STATIC struct smartlist_t *ifreq_to_smartlist(char *ifr,
-                                       size_t buflen);
+STATIC struct smartlist_t *ifreq_to_smartlist(const uint8_t *ifr,
+                                              size_t buflen);
 STATIC struct smartlist_t *get_interface_addresses_ioctl(int severity,
                                                   sa_family_t family);
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
diff --git a/src/lib/net/alertsock.c b/src/lib/net/alertsock.c
index cc59d7d893..537fdcaee4 100644
--- a/src/lib/net/alertsock.c
+++ b/src/lib/net/alertsock.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/net/alertsock.h b/src/lib/net/alertsock.h
index c45f42be81..dab4273cf1 100644
--- a/src/lib/net/alertsock.h
+++ b/src/lib/net/alertsock.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -42,4 +42,4 @@ typedef struct alert_sockets_t {
 int alert_sockets_create(alert_sockets_t *socks_out, uint32_t flags);
 void alert_sockets_close(alert_sockets_t *socks);
 
-#endif
+#endif /* !defined(TOR_ALERTSOCK_H) */
diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c
index 3eb0a033d5..4dbf491e1a 100644
--- a/src/lib/net/buffers_net.c
+++ b/src/lib/net/buffers_net.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/net/buffers_net.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/net/nettypes.h"
@@ -22,6 +22,10 @@
 
 #include <stdlib.h>
 
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
 #ifdef PARANOIA
 /** Helper: If PARANOIA is defined, assert that the buffer in local variable
  * <b>buf</b> is well-formed. */
@@ -30,27 +34,36 @@
 #define check() STMT_NIL
 #endif /* defined(PARANOIA) */
 
-/** Read up to <b>at_most</b> bytes from the socket <b>fd</b> into
+/** Read up to <b>at_most</b> bytes from the file descriptor <b>fd</b> into
  * <b>chunk</b> (which must be on <b>buf</b>). If we get an EOF, set
- * *<b>reached_eof</b> to 1.  Return -1 on error, 0 on eof or blocking,
- * and the number of bytes read otherwise. */
+ * *<b>reached_eof</b> to 1. Uses <b>tor_socket_recv()</b> iff <b>is_socket</b>
+ * is true, otherwise it uses <b>read()</b>.  Return -1 on error (and sets
+ * *<b>error</b> to errno), 0 on eof or blocking, and the number of bytes read
+ * otherwise. */
 static inline int
 read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
-              int *reached_eof, int *socket_error)
+              int *reached_eof, int *error, bool is_socket)
 {
   ssize_t read_result;
   if (at_most > CHUNK_REMAINING_CAPACITY(chunk))
     at_most = CHUNK_REMAINING_CAPACITY(chunk);
-  read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+
+  if (is_socket)
+    read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0);
+  else
+    read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most);
 
   if (read_result < 0) {
-    int e = tor_socket_errno(fd);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
-        log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?");
+        log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?",
+                 is_socket ? "recv" : "read");
 #endif
-      *socket_error = e;
+      if (error)
+        *error = e;
       return -1;
     }
     return 0; /* would block. */
@@ -63,21 +76,22 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most,
     chunk->datalen += read_result;
     log_debug(LD_NET,"Read %ld bytes. %d on inbuf.", (long)read_result,
               (int)buf->datalen);
-    tor_assert(read_result < INT_MAX);
+    tor_assert(read_result <= BUF_MAX_LEN);
     return (int)read_result;
   }
 }
 
-/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
- * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
- * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
- * error; else return the number of bytes read.
+/** Read from file descriptor <b>fd</b>, writing onto end of <b>buf</b>.  Read
+ * at most <b>at_most</b> bytes, growing the buffer as necessary.  If recv()
+ * returns 0 (because of EOF), set *<b>reached_eof</b> to 1 and return 0.
+ * Return -1 on error; else return the number of bytes read.
  */
 /* XXXX indicate "read blocked" somehow? */
-int
-buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
-                     int *reached_eof,
-                     int *socket_error)
+static int
+buf_read_from_fd(buf_t *buf, int fd, size_t at_most,
+                 int *reached_eof,
+                 int *socket_error,
+                 bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes read" are not mutually exclusive.
@@ -87,11 +101,11 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 
   check();
   tor_assert(reached_eof);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
 
-  if (BUG(buf->datalen >= INT_MAX))
+  if (BUG(buf->datalen > BUF_MAX_LEN))
     return -1;
-  if (BUG(buf->datalen >= INT_MAX - at_most))
+  if (BUG(buf->datalen > BUF_MAX_LEN - at_most))
     return -1;
 
   while (at_most > total_read) {
@@ -108,11 +122,12 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
         readlen = cap;
     }
 
-    r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error);
+    r = read_to_chunk(buf, chunk, fd, readlen,
+                      reached_eof, socket_error, is_socket);
     check();
     if (r < 0)
       return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
+    tor_assert(total_read+r <= BUF_MAX_LEN);
     total_read += r;
     if ((size_t)r < readlen) { /* eof, block, or no more to read. */
       break;
@@ -122,22 +137,27 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
 }
 
 /** Helper for buf_flush_to_socket(): try to write <b>sz</b> bytes from chunk
- * <b>chunk</b> of buffer <b>buf</b> onto socket <b>s</b>.  On success, deduct
- * the bytes written from *<b>buf_flushlen</b>.  Return the number of bytes
- * written on success, 0 on blocking, -1 on failure.
+ * <b>chunk</b> of buffer <b>buf</b> onto file descriptor <b>fd</b>.  On
+ * success, deduct the bytes written from *<b>buf_flushlen</b>.  Return the
+ * number of bytes written on success, 0 on blocking, -1 on failure.
  */
 static inline int
-flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
-            size_t *buf_flushlen)
+flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz,
+            size_t *buf_flushlen, bool is_socket)
 {
   ssize_t write_result;
 
   if (sz > chunk->datalen)
     sz = chunk->datalen;
-  write_result = tor_socket_send(s, chunk->data, sz, 0);
+
+  if (is_socket)
+    write_result = tor_socket_send(fd, chunk->data, sz, 0);
+  else
+    write_result = write(fd, chunk->data, sz);
 
   if (write_result < 0) {
-    int e = tor_socket_errno(s);
+    int e = is_socket ? tor_socket_errno(fd) : errno;
+
     if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */
 #ifdef _WIN32
       if (e == WSAENOBUFS)
@@ -150,20 +170,20 @@ flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz,
   } else {
     *buf_flushlen -= write_result;
     buf_drain(buf, write_result);
-    tor_assert(write_result < INT_MAX);
+    tor_assert(write_result <= BUF_MAX_LEN);
     return (int)write_result;
   }
 }
 
-/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+/** Write data from <b>buf</b> to the file descriptor <b>fd</b>.  Write at most
  * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
  * the number of bytes actually written, and remove the written bytes
  * from the buffer.  Return the number of bytes written on success,
  * -1 on failure.  Return 0 if write() would block.
  */
-int
-buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
-                    size_t *buf_flushlen)
+static int
+buf_flush_to_fd(buf_t *buf, int fd, size_t sz,
+                size_t *buf_flushlen, bool is_socket)
 {
   /* XXXX It's stupid to overload the return values for these functions:
    * "error status" and "number of bytes flushed" are not mutually exclusive.
@@ -171,7 +191,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
   int r;
   size_t flushed = 0;
   tor_assert(buf_flushlen);
-  tor_assert(SOCKET_OK(s));
+  tor_assert(SOCKET_OK(fd));
   if (BUG(*buf_flushlen > buf->datalen)) {
     *buf_flushlen = buf->datalen;
   }
@@ -188,7 +208,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
     else
       flushlen0 = buf->head->datalen;
 
-    r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen);
+    r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket);
     check();
     if (r < 0)
       return r;
@@ -197,6 +217,58 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
     if (r == 0 || (size_t)r < flushlen0) /* can't flush any more now. */
       break;
   }
-  tor_assert(flushed < INT_MAX);
+  tor_assert(flushed <= BUF_MAX_LEN);
   return (int)flushed;
 }
+
+/** Write data from <b>buf</b> to the socket <b>s</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz,
+                    size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, s, sz, buf_flushlen, true);
+}
+
+/** Read from socket <b>s</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If recv() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most,
+                     int *reached_eof,
+                     int *socket_error)
+{
+  return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true);
+}
+
+/** Write data from <b>buf</b> to the pipe <b>fd</b>.  Write at most
+ * <b>sz</b> bytes, decrement *<b>buf_flushlen</b> by
+ * the number of bytes actually written, and remove the written bytes
+ * from the buffer.  Return the number of bytes written on success,
+ * -1 on failure.  Return 0 if write() would block.
+ */
+int
+buf_flush_to_pipe(buf_t *buf, int fd, size_t sz,
+                  size_t *buf_flushlen)
+{
+  return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false);
+}
+
+/** Read from pipe <b>fd</b>, writing onto end of <b>buf</b>.  Read at most
+ * <b>at_most</b> bytes, growing the buffer as necessary.  If read() returns 0
+ * (because of EOF), set *<b>reached_eof</b> to 1 and return 0. Return -1 on
+ * error; else return the number of bytes read.
+ */
+int
+buf_read_from_pipe(buf_t *buf, int fd, size_t at_most,
+                   int *reached_eof,
+                   int *socket_error)
+{
+  return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false);
+}
diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h
index 5f69bebedf..a45c23a273 100644
--- a/src/lib/net/buffers_net.h
+++ b/src/lib/net/buffers_net.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most,
 int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz,
                         size_t *buf_flushlen);
 
-#endif /* !defined(TOR_BUFFERS_H) */
+int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most,
+                       int *reached_eof,
+                       int *socket_error);
+
+int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz,
+                      size_t *buf_flushlen);
+
+#endif /* !defined(TOR_BUFFERS_NET_H) */
diff --git a/src/lib/net/gethostname.c b/src/lib/net/gethostname.c
index e54a1ea16e..001d95391d 100644
--- a/src/lib/net/gethostname.c
+++ b/src/lib/net/gethostname.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/net/gethostname.h b/src/lib/net/gethostname.h
index 69b0528bc0..90f8056779 100644
--- a/src/lib/net/gethostname.h
+++ b/src/lib/net/gethostname.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,4 +16,4 @@
 
 MOCK_DECL(int,tor_gethostname,(char *name, size_t namelen));
 
-#endif
+#endif /* !defined(TOR_GETHOSTNAME_H) */
diff --git a/src/lib/net/inaddr.c b/src/lib/net/inaddr.c
index 1a2406ce5f..d50ac2440c 100644
--- a/src/lib/net/inaddr.c
+++ b/src/lib/net/inaddr.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,7 +11,9 @@
 #include "lib/net/inaddr.h"
 
 #include "lib/cc/torint.h"
+#include "lib/container/smartlist.h"
 #include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
 #include "lib/net/inaddr_st.h"
 #include "lib/string/compat_ctype.h"
 #include "lib/string/compat_string.h"
@@ -35,12 +37,31 @@
  * (Like inet_aton(str,addr), but works on Windows and Solaris.)
  */
 int
-tor_inet_aton(const char *str, struct in_addr* addr)
+tor_inet_aton(const char *str, struct in_addr *addr)
 {
-  unsigned a,b,c,d;
+  unsigned a, b, c, d;
   char more;
-  if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a,&b,&c,&d,&more) != 4)
+  bool is_octal = false;
+  smartlist_t *sl = NULL;
+
+  if (tor_sscanf(str, "%3u.%3u.%3u.%3u%c", &a, &b, &c, &d, &more) != 4)
+    return 0;
+
+  /* Parse the octets and check them for leading zeros. */
+  sl = smartlist_new();
+  smartlist_split_string(sl, str, ".", 0, 0);
+  SMARTLIST_FOREACH(sl, const char *, octet, {
+    is_octal = (strlen(octet) > 1 && octet[0] == '0');
+    if (is_octal) {
+        break;
+    }
+  });
+  SMARTLIST_FOREACH(sl, char *, octet, tor_free(octet));
+  smartlist_free(sl);
+
+  if (is_octal)
     return 0;
+
   if (a > 255) return 0;
   if (b > 255) return 0;
   if (c > 255) return 0;
@@ -168,6 +189,13 @@ tor_inet_pton(int af, const char *src, void *dst)
   if (af == AF_INET) {
     return tor_inet_aton(src, dst);
   } else if (af == AF_INET6) {
+    ssize_t len = strlen(src);
+
+    /* Reject if src has needless trailing ':'. */
+    if (len > 2 && src[len - 1] == ':' && src[len - 2] != ':') {
+      return 0;
+    }
+
     struct in6_addr *out = dst;
     uint16_t words[8];
     int gapPos = -1, i, setWords=0;
@@ -207,7 +235,6 @@ tor_inet_pton(int af, const char *src, void *dst)
         return 0;
       if (TOR_ISXDIGIT(*src)) {
         char *next;
-        ssize_t len;
         long r = strtol(src, &next, 16);
         if (next == NULL || next == src) {
           /* The 'next == src' error case can happen on versions of openbsd
diff --git a/src/lib/net/inaddr.h b/src/lib/net/inaddr.h
index 36352b65ea..8d6766eb5d 100644
--- a/src/lib/net/inaddr.h
+++ b/src/lib/net/inaddr.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,4 +24,4 @@ int tor_inet_ntoa(const struct in_addr *in, char *buf, size_t buf_len);
 const char *tor_inet_ntop(int af, const void *src, char *dst, size_t len);
 int tor_inet_pton(int af, const char *src, void *dst);
 
-#endif
+#endif /* !defined(TOR_INADDR_H) */
diff --git a/src/lib/net/inaddr_st.h b/src/lib/net/inaddr_st.h
index 806f2c096a..b9ee2b86cf 100644
--- a/src/lib/net/inaddr_st.h
+++ b/src/lib/net/inaddr_st.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -104,4 +104,4 @@ struct sockaddr_in6 {
 };
 #endif /* !defined(HAVE_STRUCT_SOCKADDR_IN6) */
 
-#endif /* TOR_INADDR_ST_H */
+#endif /* !defined(TOR_INADDR_ST_H) */
diff --git a/src/lib/net/include.am b/src/lib/net/include.am
index ff0967e786..485019f4b7 100644
--- a/src/lib/net/include.am
+++ b/src/lib/net/include.am
@@ -5,12 +5,14 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-net-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_net_a_SOURCES =			\
 	src/lib/net/address.c			\
 	src/lib/net/alertsock.c                 \
 	src/lib/net/buffers_net.c		\
 	src/lib/net/gethostname.c		\
 	src/lib/net/inaddr.c			\
+	src/lib/net/network_sys.c		\
 	src/lib/net/resolve.c			\
 	src/lib/net/socket.c			\
 	src/lib/net/socketpair.c
@@ -20,6 +22,7 @@ src_lib_libtor_net_testing_a_SOURCES = \
 src_lib_libtor_net_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_net_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/net/address.h			\
 	src/lib/net/alertsock.h                 \
@@ -28,6 +31,7 @@ noinst_HEADERS +=				\
 	src/lib/net/inaddr.h			\
 	src/lib/net/inaddr_st.h			\
 	src/lib/net/nettypes.h			\
+	src/lib/net/network_sys.h		\
 	src/lib/net/resolve.h			\
 	src/lib/net/socket.h			\
 	src/lib/net/socketpair.h		\
diff --git a/src/lib/net/lib_net.md b/src/lib/net/lib_net.md
new file mode 100644
index 0000000000..b61878d827
--- /dev/null
+++ b/src/lib/net/lib_net.md
@@ -0,0 +1,6 @@
+@dir /lib/net
+@brief lib/net: Low-level network-related code.
+
+This module includes address manipulation, compatibility wrappers,
+convenience functions, and so on.
+
diff --git a/src/lib/net/nettypes.h b/src/lib/net/nettypes.h
index 6209bbe18a..953673d4c3 100644
--- a/src/lib/net/nettypes.h
+++ b/src/lib/net/nettypes.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -31,7 +31,7 @@ typedef int socklen_t;
 #define TOR_SOCKET_T_FORMAT "%"PRIuPTR
 #define SOCKET_OK(s) ((SOCKET)(s) != INVALID_SOCKET)
 #define TOR_INVALID_SOCKET INVALID_SOCKET
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 /** Type used for a network socket. */
 #define tor_socket_t int
 #define TOR_SOCKET_T_FORMAT "%d"
@@ -41,4 +41,4 @@ typedef int socklen_t;
 #define TOR_INVALID_SOCKET (-1)
 #endif /* defined(_WIN32) */
 
-#endif
+#endif /* !defined(TOR_NET_TYPES_H) */
diff --git a/src/lib/net/network_sys.c b/src/lib/net/network_sys.c
new file mode 100644
index 0000000000..e95c3ba819
--- /dev/null
+++ b/src/lib/net/network_sys.c
@@ -0,0 +1,47 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file network_sys.c
+ * \brief Subsystem object for networking setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/net/network_sys.h"
+#include "lib/net/resolve.h"
+#include "lib/net/socket.h"
+
+#ifdef _WIN32
+#include <winsock2.h>
+#include <windows.h>
+#endif
+
+static int
+subsys_network_initialize(void)
+{
+  if (network_init() < 0)
+    return -1;
+
+  return 0;
+}
+
+static void
+subsys_network_shutdown(void)
+{
+#ifdef _WIN32
+  WSACleanup();
+#endif
+  tor_free_getaddrinfo_cache();
+}
+
+const subsys_fns_t sys_network = {
+  .name = "network",
+  SUBSYS_DECLARE_LOCATION(),
+  /* Network depends on logging, and a lot of other modules depend on network.
+   */
+  .level = -55,
+  .supported = true,
+  .initialize = subsys_network_initialize,
+  .shutdown = subsys_network_shutdown,
+};
diff --git a/src/lib/net/network_sys.h b/src/lib/net/network_sys.h
new file mode 100644
index 0000000000..734533c7e8
--- /dev/null
+++ b/src/lib/net/network_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file network_sys.h
+ * \brief Declare subsystem object for the network module.
+ **/
+
+#ifndef TOR_NETWORK_SYS_H
+#define TOR_NETWORK_SYS_H
+
+extern const struct subsys_fns_t sys_network;
+
+#endif /* !defined(TOR_NETWORK_SYS_H) */
diff --git a/src/lib/net/resolve.c b/src/lib/net/resolve.c
index 8cee29df37..68a8c01ef4 100644
--- a/src/lib/net/resolve.c
+++ b/src/lib/net/resolve.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,6 +8,7 @@
  * \brief Use the libc DNS resolver to convert hostnames into addresses.
  **/
 
+#define RESOLVE_PRIVATE
 #include "lib/net/resolve.h"
 
 #include "lib/net/address.h"
@@ -16,8 +17,8 @@
 #include "lib/string/parse_int.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
-#include "ht.h"
+#include "ext/siphash.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
@@ -35,6 +36,8 @@
  * *<b>addr</b> to the proper IP address, in host byte order.  Returns 0
  * on success, -1 on failure; 1 on transient failure.
  *
+ * This function only accepts IPv4 addresses.
+ *
  * (This function exists because standard windows gethostbyname
  * doesn't treat raw IP addresses properly.)
  */
@@ -45,6 +48,11 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   tor_addr_t myaddr;
   int ret;
 
+  if (BUG(!addr))
+    return -1;
+
+  *addr = 0;
+
   if ((ret = tor_addr_lookup(name, AF_INET, &myaddr)))
     return ret;
 
@@ -56,183 +64,257 @@ tor_lookup_hostname,(const char *name, uint32_t *addr))
   return -1;
 }
 
-/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
- * *<b>addr</b> to the proper IP address and family. The <b>family</b>
- * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
- * <i>preferred</i> family, though another one may be returned if only one
- * family is implemented for this address.
+#ifdef HAVE_GETADDRINFO
+
+/* Host lookup helper for tor_addr_lookup(), when getaddrinfo() is
+ * available on this system.
  *
- * Return 0 on success, -1 on failure; 1 on transient failure.
+ * See tor_addr_lookup() for details.
  */
-MOCK_IMPL(int,
-tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                          tor_addr_t *addr))
 {
-  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
-   * something.
-   */
-  struct in_addr iaddr;
-  struct in6_addr iaddr6;
-  tor_assert(name);
-  tor_assert(addr);
-  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
-  if (!*name) {
-    /* Empty address is an error. */
-    return -1;
-  } else if (tor_inet_pton(AF_INET, name, &iaddr)) {
-    /* It's an IPv4 IP. */
-    if (family == AF_INET6)
-      return -1;
-    tor_addr_from_in(addr, &iaddr);
-    return 0;
-  } else if (tor_inet_pton(AF_INET6, name, &iaddr6)) {
-    if (family == AF_INET)
-      return -1;
-    tor_addr_from_in6(addr, &iaddr6);
-    return 0;
-  } else {
-#ifdef HAVE_GETADDRINFO
-    int err;
-    struct addrinfo *res=NULL, *res_p;
-    struct addrinfo *best=NULL;
-    struct addrinfo hints;
-    int result = -1;
-    memset(&hints, 0, sizeof(hints));
-    hints.ai_family = family;
-    hints.ai_socktype = SOCK_STREAM;
-    err = tor_getaddrinfo(name, NULL, &hints, &res);
-    /* The check for 'res' here shouldn't be necessary, but it makes static
-     * analysis tools happy. */
-    if (!err && res) {
-      best = NULL;
-      for (res_p = res; res_p; res_p = res_p->ai_next) {
-        if (family == AF_UNSPEC) {
-          if (res_p->ai_family == AF_INET) {
-            best = res_p;
-            break;
-          } else if (res_p->ai_family == AF_INET6 && !best) {
-            best = res_p;
-          }
-        } else if (family == res_p->ai_family) {
+  int err;
+  struct addrinfo *res=NULL, *res_p;
+  struct addrinfo *best=NULL;
+  struct addrinfo hints;
+  int result = -1;
+  memset(&hints, 0, sizeof(hints));
+  hints.ai_family = family;
+  hints.ai_socktype = SOCK_STREAM;
+  err = tor_getaddrinfo(name, NULL, &hints, &res);
+  /* The check for 'res' here shouldn't be necessary, but it makes static
+   * analysis tools happy. */
+  if (!err && res) {
+    best = NULL;
+    for (res_p = res; res_p; res_p = res_p->ai_next) {
+      if (family == AF_UNSPEC) {
+        if (res_p->ai_family == AF_INET) {
           best = res_p;
           break;
+        } else if (res_p->ai_family == AF_INET6 && !best) {
+          best = res_p;
         }
+      } else if (family == res_p->ai_family) {
+        best = res_p;
+        break;
       }
-      if (!best)
-        best = res;
-      if (best->ai_family == AF_INET) {
-        tor_addr_from_in(addr,
-                         &((struct sockaddr_in*)best->ai_addr)->sin_addr);
-        result = 0;
-      } else if (best->ai_family == AF_INET6) {
-        tor_addr_from_in6(addr,
-                          &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
-        result = 0;
-      }
-      tor_freeaddrinfo(res);
-      return result;
     }
-    return (err == EAI_AGAIN) ? 1 : -1;
-#else /* !(defined(HAVE_GETADDRINFO)) */
-    struct hostent *ent;
-    int err;
+    if (!best)
+      best = res;
+    if (best->ai_family == AF_INET) {
+      tor_addr_from_in(addr,
+                       &((struct sockaddr_in*)best->ai_addr)->sin_addr);
+      result = 0;
+    } else if (best->ai_family == AF_INET6) {
+      tor_addr_from_in6(addr,
+                        &((struct sockaddr_in6*)best->ai_addr)->sin6_addr);
+      result = 0;
+    }
+    tor_freeaddrinfo(res);
+    return result;
+  }
+  return (err == EAI_AGAIN) ? 1 : -1;
+}
+
+#else /* !defined(HAVE_GETADDRINFO) */
+
+/* Host lookup helper for tor_addr_lookup(), which calls gethostbyname().
+ * Used when getaddrinfo() is not available on this system.
+ *
+ * See tor_addr_lookup() for details.
+ */
+MOCK_IMPL(STATIC int,
+tor_addr_lookup_host_impl,(const char *name,
+                          uint16_t family,
+                           tor_addr_t *addr))
+{
+  (void) family;
+  struct hostent *ent;
+  int err;
 #ifdef HAVE_GETHOSTBYNAME_R_6_ARG
-    char buf[2048];
-    struct hostent hostent;
-    int r;
-    r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
+  char buf[2048];
+  struct hostent hostent;
+  int r;
+  r = gethostbyname_r(name, &hostent, buf, sizeof(buf), &ent, &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_5_ARG)
-    char buf[2048];
-    struct hostent hostent;
-    ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
+  char buf[2048];
+  struct hostent hostent;
+  ent = gethostbyname_r(name, &hostent, buf, sizeof(buf), &err);
 #elif defined(HAVE_GETHOSTBYNAME_R_3_ARG)
-    struct hostent_data data;
-    struct hostent hent;
-    memset(&data, 0, sizeof(data));
-    err = gethostbyname_r(name, &hent, &data);
-    ent = err ? NULL : &hent;
+  struct hostent_data data;
+  struct hostent hent;
+  memset(&data, 0, sizeof(data));
+  err = gethostbyname_r(name, &hent, &data);
+  ent = err ? NULL : &hent;
 #else
-    ent = gethostbyname(name);
+  ent = gethostbyname(name);
 #ifdef _WIN32
-    err = WSAGetLastError();
+  err = WSAGetLastError();
 #else
-    err = h_errno;
-#endif
+  err = h_errno;
+#endif /* defined(_WIN32) */
 #endif /* defined(HAVE_GETHOSTBYNAME_R_6_ARG) || ... */
-    if (ent) {
-      if (ent->h_addrtype == AF_INET) {
-        tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
-      } else if (ent->h_addrtype == AF_INET6) {
-        tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
-      } else {
-        tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
-      }
-      return 0;
+  if (ent) {
+    if (ent->h_addrtype == AF_INET) {
+      tor_addr_from_in(addr, (struct in_addr*) ent->h_addr);
+    } else if (ent->h_addrtype == AF_INET6) {
+      tor_addr_from_in6(addr, (struct in6_addr*) ent->h_addr);
+    } else {
+      tor_assert(0); // LCOV_EXCL_LINE: gethostbyname() returned bizarre type
     }
+    return 0;
+  }
 #ifdef _WIN32
-    return (err == WSATRY_AGAIN) ? 1 : -1;
+  return (err == WSATRY_AGAIN) ? 1 : -1;
 #else
-    return (err == TRY_AGAIN) ? 1 : -1;
+  return (err == TRY_AGAIN) ? 1 : -1;
 #endif
+}
 #endif /* defined(HAVE_GETADDRINFO) */
+
+/** Similar behavior to Unix gethostbyname: resolve <b>name</b>, and set
+ * *<b>addr</b> to the proper IP address and family. The <b>family</b>
+ * argument (which must be AF_INET, AF_INET6, or AF_UNSPEC) declares a
+ * <i>preferred</i> family, though another one may be returned if only one
+ * family is implemented for this address.
+ *
+ * Like tor_addr_parse(), this function accepts IPv6 addresses with or without
+ * square brackets.
+ *
+ * Return 0 on success, -1 on failure; 1 on transient failure.
+ */
+MOCK_IMPL(int,
+tor_addr_lookup,(const char *name, uint16_t family, tor_addr_t *addr))
+{
+  /* Perhaps eventually this should be replaced by a tor_getaddrinfo or
+   * something.
+   */
+  int parsed_family = 0;
+  int result = -1;
+
+  tor_assert(name);
+  tor_assert(addr);
+  tor_assert(family == AF_INET || family == AF_INET6 || family == AF_UNSPEC);
+
+  if (!*name) {
+    /* Empty address is an error. */
+    goto permfail;
+  }
+
+  /* Is it an IP address? */
+  parsed_family = tor_addr_parse(addr, name);
+
+  if (parsed_family >= 0) {
+    /* If the IP address family matches, or was unspecified */
+    if (parsed_family == family || family == AF_UNSPEC) {
+      goto success;
+    } else {
+      goto permfail;
+    }
+  } else {
+    /* Clear the address after a failed tor_addr_parse(). */
+    memset(addr, 0, sizeof(tor_addr_t));
+    result = tor_addr_lookup_host_impl(name, family, addr);
+    goto done;
   }
+
+ /* If we weren't successful, and haven't already set the result,
+  * assume it's a permanent failure */
+ permfail:
+  result = -1;
+  goto done;
+ success:
+  result = 0;
+
+ /* We have set the result, now it's time to clean up */
+ done:
+  if (result) {
+    /* Clear the address on error */
+    memset(addr, 0, sizeof(tor_addr_t));
+  }
+  return result;
 }
 
 /** Parse an address or address-port combination from <b>s</b>, resolve the
  * address as needed, and put the result in <b>addr_out</b> and (optionally)
- * <b>port_out</b>.  Return 0 on success, negative on failure. */
+ * <b>port_out</b>.
+ *
+ * Like tor_addr_port_parse(), this function accepts:
+ *  - IPv6 address and port, when the IPv6 address is in square brackets,
+ *  - IPv6 address with square brackets,
+ *  - IPv6 address without square brackets.
+ *
+ * Return 0 on success, negative on failure. */
 int
 tor_addr_port_lookup(const char *s, tor_addr_t *addr_out, uint16_t *port_out)
 {
-  const char *port;
   tor_addr_t addr;
-  uint16_t portval;
+  uint16_t portval = 0;
   char *tmp = NULL;
+  int rv = 0;
+  int result;
 
   tor_assert(s);
   tor_assert(addr_out);
 
   s = eat_whitespace(s);
 
-  if (*s == '[') {
-    port = strstr(s, "]");
-    if (!port)
-      goto err;
-    tmp = tor_strndup(s+1, port-(s+1));
-    port = port+1;
-    if (*port == ':')
-      port++;
-    else
-      port = NULL;
-  } else {
-    port = strchr(s, ':');
-    if (port)
-      tmp = tor_strndup(s, port-s);
-    else
-      tmp = tor_strdup(s);
-    if (port)
-      ++port;
+  /* Try parsing s as an address:port first, so we don't have to duplicate
+   * the logic that rejects IPv6:Port with no square brackets. */
+  rv = tor_addr_port_parse(LOG_WARN, s, &addr, &portval, 0);
+  /* That was easy, no DNS required. */
+  if (rv == 0)
+    goto success;
+
+  /* Now let's check for malformed IPv6 addresses and ports:
+   * tor_addr_port_parse() requires squared brackes if there is a port,
+   * and we want tor_addr_port_lookup() to have the same requirement.
+   * But we strip the port using tor_addr_port_split(), so tor_addr_lookup()
+   * only sees the address, and will accept it without square brackets. */
+  int family = tor_addr_parse(&addr, s);
+  /* If tor_addr_parse() succeeds where tor_addr_port_parse() failed, we need
+   * to reject this address as malformed. */
+  if (family >= 0) {
+    /* Double-check it's an IPv6 address. If not, we have a parsing bug.
+     */
+    tor_assertf_nonfatal(family == AF_INET6,
+                         "Wrong family: %d (should be IPv6: %d) which "
+                         "failed IP:port parsing, but passed IP parsing. "
+                         "input string: '%s'; parsed address: '%s'.",
+                         family, AF_INET6, s, fmt_addr(&addr));
+    goto err;
   }
 
-  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+  /* Now we have a hostname. Let's split off the port, if any. */
+  rv = tor_addr_port_split(LOG_WARN, s, &tmp, &portval);
+  if (rv < 0)
     goto err;
-  tor_free(tmp);
 
-  if (port) {
-    portval = (int) tor_parse_long(port, 10, 1, 65535, NULL, NULL);
-    if (!portval)
-      goto err;
-  } else {
-    portval = 0;
-  }
+  /* And feed the hostname to the lookup function. */
+  if (tor_addr_lookup(tmp, AF_UNSPEC, &addr) != 0)
+    goto err;
 
+ success:
   if (port_out)
     *port_out = portval;
   tor_addr_copy(addr_out, &addr);
+  result = 0;
+  goto done;
 
-  return 0;
  err:
+  /* Clear the address and port on error */
+  memset(addr_out, 0, sizeof(tor_addr_t));
+  if (port_out)
+    *port_out = 0;
+  result = -1;
+
+ /* We have set the result, now it's time to clean up */
+ done:
   tor_free(tmp);
-  return -1;
+  return result;
 }
 
 #ifdef USE_SANDBOX_GETADDRINFO
@@ -290,11 +372,11 @@ static HT_HEAD(getaddrinfo_cache, cached_getaddrinfo_item_t)
 
 HT_PROTOTYPE(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
              cached_getaddrinfo_item_hash,
-             cached_getaddrinfo_items_eq)
+             cached_getaddrinfo_items_eq);
 HT_GENERATE2(getaddrinfo_cache, cached_getaddrinfo_item_t, node,
              cached_getaddrinfo_item_hash,
              cached_getaddrinfo_items_eq,
-             0.6, tor_reallocarray_, tor_free_)
+             0.6, tor_reallocarray_, tor_free_);
 
 /** If true, don't try to cache getaddrinfo results. */
 static int sandbox_getaddrinfo_cache_disabled = 0;
@@ -421,4 +503,13 @@ tor_make_getaddrinfo_cache_active(void)
 {
   sandbox_getaddrinfo_is_active = 1;
 }
-#endif
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
+void
+sandbox_disable_getaddrinfo_cache(void)
+{
+}
+void
+tor_make_getaddrinfo_cache_active(void)
+{
+}
+#endif /* defined(USE_SANDBOX_GETADDRINFO) */
diff --git a/src/lib/net/resolve.h b/src/lib/net/resolve.h
index 47a283c81c..ef3d9fa176 100644
--- a/src/lib/net/resolve.h
+++ b/src/lib/net/resolve.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -24,12 +24,18 @@
 
 struct tor_addr_t;
 
+/*
+ * Primary lookup functions.
+ */
 MOCK_DECL(int, tor_lookup_hostname,(const char *name, uint32_t *addr));
 MOCK_DECL(int, tor_addr_lookup,(const char *name, uint16_t family,
                                 struct tor_addr_t *addr_out));
 int tor_addr_port_lookup(const char *s, struct tor_addr_t *addr_out,
                          uint16_t *port_out);
 
+/*
+ * Sandbox helpers
+ */
 struct addrinfo;
 #ifdef USE_SANDBOX_GETADDRINFO
 /** Pre-calls getaddrinfo in order to pre-record result. */
@@ -42,8 +48,7 @@ int tor_getaddrinfo(const char *name, const char *servname,
                         struct addrinfo **res);
 void tor_freeaddrinfo(struct addrinfo *addrinfo);
 void tor_free_getaddrinfo_cache(void);
-void tor_make_getaddrinfo_cache_active(void);
-#else /* !(defined(USE_SANDBOX_GETADDRINFO)) */
+#else /* !defined(USE_SANDBOX_GETADDRINFO) */
 #define tor_getaddrinfo(name, servname, hints, res)  \
   getaddrinfo((name),(servname), (hints),(res))
 #define tor_add_addrinfo(name) \
@@ -54,5 +59,15 @@ void tor_make_getaddrinfo_cache_active(void);
 #endif /* defined(USE_SANDBOX_GETADDRINFO) */
 
 void sandbox_disable_getaddrinfo_cache(void);
+void tor_make_getaddrinfo_cache_active(void);
 
+/*
+ * Internal resolver wrapper; exposed for mocking.
+ */
+#ifdef RESOLVE_PRIVATE
+MOCK_DECL(STATIC int, tor_addr_lookup_host_impl, (const char *name,
+                                                  uint16_t family,
+                                                  struct tor_addr_t *addr));
 #endif
+
+#endif /* !defined(TOR_RESOLVE_H) */
diff --git a/src/lib/net/socket.c b/src/lib/net/socket.c
index fba90b7506..adc060a735 100644
--- a/src/lib/net/socket.c
+++ b/src/lib/net/socket.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,7 +9,6 @@
  *    sockets.
  **/
 
-#define SOCKET_PRIVATE
 #include "lib/net/socket.h"
 #include "lib/net/socketpair.h"
 #include "lib/net/address.h"
@@ -31,6 +30,9 @@
 #endif
 #include <stddef.h>
 #include <string.h>
+#ifdef __FreeBSD__
+#include <sys/sysctl.h>
+#endif
 
 /** Called before we make any calls to network-related functions.
  * (Some operating systems require their network libraries to be
@@ -60,6 +62,32 @@ network_init(void)
   return 0;
 }
 
+/**
+ * Warn the user if any system network parameters should be changed.
+ */
+void
+check_network_configuration(bool server_mode)
+{
+#ifdef __FreeBSD__
+  if (server_mode) {
+    int random_id_state;
+    size_t state_size = sizeof(random_id_state);
+
+    if (sysctlbyname("net.inet.ip.random_id", &random_id_state,
+                     &state_size, NULL, 0)) {
+      log_warn(LD_CONFIG,
+               "Failed to figure out if IP ids are randomized.");
+    } else if (random_id_state == 0) {
+      log_warn(LD_CONFIG, "Looks like IP ids are not randomized. "
+               "Please consider setting the net.inet.ip.random_id sysctl, "
+               "so your relay makes it harder to figure out how busy it is.");
+    }
+  }
+#else /* !defined(__FreeBSD__) */
+  (void) server_mode;
+#endif /* defined(__FreeBSD__) */
+}
+
 /* When set_max_file_sockets() is called, update this with the max file
  * descriptor value so we can use it to check the limit when opening a new
  * socket. Default value is what Debian sets as the default hard limit. */
@@ -177,7 +205,7 @@ mark_socket_closed(tor_socket_t s)
     bitarray_clear(open_sockets, s);
   }
 }
-#else /* !(defined(DEBUG_SOCKET_COUNTING)) */
+#else /* !defined(DEBUG_SOCKET_COUNTING) */
 #define mark_socket_open(s) ((void) (s))
 #define mark_socket_closed(s) ((void) (s))
 #endif /* defined(DEBUG_SOCKET_COUNTING) */
@@ -279,7 +307,7 @@ tor_open_socket_with_extensions(int domain, int type, int protocol,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -389,7 +417,7 @@ tor_accept_socket_with_extensions(tor_socket_t sockfd, struct sockaddr *addr,
       return TOR_INVALID_SOCKET;
     }
   }
-#else /* !(defined(FD_CLOEXEC)) */
+#else /* !defined(FD_CLOEXEC) */
   (void)cloexec;
 #endif /* defined(FD_CLOEXEC) */
 
@@ -429,7 +457,9 @@ get_n_open_sockets(void)
  * localhost is inaccessible (for example, if the networking
  * stack is down). And even if it succeeds, the socket pair will not
  * be able to read while localhost is down later (the socket pair may
- * even close, depending on OS-specific timeouts).
+ * even close, depending on OS-specific timeouts). The socket pair
+ * should work on IPv4-only, IPv6-only, and dual-stack systems, as long
+ * as they have the standard localhost addresses.
  *
  * Returns 0 on success and -errno on failure; do not rely on the value
  * of errno or WSAGetLastError().
@@ -456,11 +486,11 @@ tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
   r = socketpair(family, type, protocol, fd);
   if (r < 0)
     return -errno;
-#else
+#else /* !(defined(HAVE_SOCKETPAIR) && !defined(_WIN32)) */
   r = tor_ersatz_socketpair(family, type, protocol, fd);
   if (r < 0)
     return -r;
-#endif
+#endif /* defined(HAVE_SOCKETPAIR) && !defined(_WIN32) */
 
 #if defined(FD_CLOEXEC)
   if (SOCKET_OK(fd[0])) {
diff --git a/src/lib/net/socket.h b/src/lib/net/socket.h
index 0909619510..46735fdef0 100644
--- a/src/lib/net/socket.h
+++ b/src/lib/net/socket.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -54,6 +54,7 @@ int tor_addr_from_getsockname(struct tor_addr_t *addr_out, tor_socket_t sock);
 int set_socket_nonblocking(tor_socket_t socket);
 int tor_socketpair(int family, int type, int protocol, tor_socket_t fd[2]);
 int network_init(void);
+void check_network_configuration(bool server_mode);
 
 int get_max_sockets(void);
 void set_max_sockets(int);
@@ -91,7 +92,7 @@ ssize_t read_all_from_socket(tor_socket_t fd, char *buf, size_t count);
 #define ERRNO_IS_EINTR(e)            ((e) == WSAEINTR || 0)
 int tor_socket_errno(tor_socket_t sock);
 const char *tor_socket_strerror(int e);
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
 #define SOCK_ERRNO(e) e
 #if EAGAIN == EWOULDBLOCK
 /* || 0 is for -Wparentheses-equality (-Wall?) appeasement under clang */
@@ -115,4 +116,4 @@ const char *tor_socket_strerror(int e);
 #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747b
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKET_H) */
diff --git a/src/lib/net/socketpair.c b/src/lib/net/socketpair.c
index 10eb749735..d4310020cb 100644
--- a/src/lib/net/socketpair.c
+++ b/src/lib/net/socketpair.c
@@ -1,6 +1,11 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+
+/**
+ * @file socketpair.c
+ * @brief Replacement socketpair() for systems that lack it
+ **/
 
 #include "lib/cc/torint.h"
 #include "lib/net/socketpair.h"
@@ -22,11 +27,11 @@
 #include <windows.h>
 #define socket_errno() (WSAGetLastError())
 #define SOCKET_EPROTONOSUPPORT WSAEPROTONOSUPPORT
-#else
+#else /* !defined(_WIN32) */
 #define closesocket(x) close(x)
 #define socket_errno() (errno)
 #define SOCKET_EPROTONOSUPPORT EPROTONOSUPPORT
-#endif
+#endif /* defined(_WIN32) */
 
 #ifdef NEED_ERSATZ_SOCKETPAIR
 
@@ -105,7 +110,12 @@ sockaddr_eq(struct sockaddr *sa1, struct sockaddr *sa2)
 /**
  * Helper used to implement socketpair on systems that lack it, by
  * making a direct connection to localhost.
- */
+ *
+ * See tor_socketpair() for details.
+ *
+ * The direct connection defaults to IPv4, but falls back to IPv6 if
+ * IPv4 is not supported.
+ **/
 int
 tor_ersatz_socketpair(int family, int type, int protocol, tor_socket_t fd[2])
 {
diff --git a/src/lib/net/socketpair.h b/src/lib/net/socketpair.h
index 6be0803881..b07016ab94 100644
--- a/src/lib/net/socketpair.h
+++ b/src/lib/net/socketpair.h
@@ -1,11 +1,16 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_SOCKETPAIR_H
 #define TOR_SOCKETPAIR_H
 
+/**
+ * @file socketpair.h
+ * @brief Header for socketpair.c
+ **/
+
 #include "orconfig.h"
 #include "lib/testsupport/testsupport.h"
 #include "lib/net/nettypes.h"
@@ -16,4 +21,4 @@ int tor_ersatz_socketpair(int family, int type, int protocol,
                           tor_socket_t fd[2]);
 #endif
 
-#endif
+#endif /* !defined(TOR_SOCKETPAIR_H) */
diff --git a/src/lib/net/socks5_status.h b/src/lib/net/socks5_status.h
index e55242ce66..90c4305d29 100644
--- a/src/lib/net/socks5_status.h
+++ b/src/lib/net/socks5_status.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,6 +27,17 @@ typedef enum {
   SOCKS5_TTL_EXPIRED                = 0x06,
   SOCKS5_COMMAND_NOT_SUPPORTED      = 0x07,
   SOCKS5_ADDRESS_TYPE_NOT_SUPPORTED = 0x08,
+
+  /* Extended error code (see prop304). Only used if the SocksPort flag
+   * "ExtendedErrors" is set. */
+  SOCKS5_HS_NOT_FOUND               = 0xF0,
+  SOCKS5_HS_IS_INVALID              = 0xF1,
+  SOCKS5_HS_INTRO_FAILED            = 0xF2,
+  SOCKS5_HS_REND_FAILED             = 0xF3,
+  SOCKS5_HS_MISSING_CLIENT_AUTH     = 0xF4,
+  SOCKS5_HS_BAD_CLIENT_AUTH         = 0xF5,
+  SOCKS5_HS_BAD_ADDRESS             = 0xF6,
+  SOCKS5_HS_INTRO_TIMEDOUT          = 0xF7,
 } socks5_reply_status_t;
 
-#endif
+#endif /* !defined(TOR_SOCKS5_STATUS_H) */
diff --git a/src/lib/osinfo/include.am b/src/lib/osinfo/include.am
index 16c5812604..84bd7feb00 100644
--- a/src/lib/osinfo/include.am
+++ b/src/lib/osinfo/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-osinfo-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_osinfo_a_SOURCES =			\
 	src/lib/osinfo/uname.c
 
@@ -13,5 +14,6 @@ src_lib_libtor_osinfo_testing_a_SOURCES = \
 src_lib_libtor_osinfo_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_osinfo_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/osinfo/uname.h
diff --git a/src/lib/osinfo/lib_osinfo.md b/src/lib/osinfo/lib_osinfo.md
new file mode 100644
index 0000000000..0678ecc21e
--- /dev/null
+++ b/src/lib/osinfo/lib_osinfo.md
@@ -0,0 +1,8 @@
+@dir /lib/osinfo
+@brief lib/osinfo: For inspecting the OS version and capabilities.
+
+In general, we use this module when we're telling the user what operating
+system they are running.  We shouldn't make decisions based on the output of
+these checks: instead, we should have more specific checks, either at compile
+time or run time, based on the observed system behavior.
+
diff --git a/src/lib/osinfo/uname.c b/src/lib/osinfo/uname.c
index 2b37ff136c..f7f5ede307 100644
--- a/src/lib/osinfo/uname.c
+++ b/src/lib/osinfo/uname.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,6 +27,40 @@ static char uname_result[256];
 /** True iff uname_result is set. */
 static int uname_result_is_set = 0;
 
+#ifdef _WIN32
+/** Table to map claimed windows versions into human-readable windows
+ * versions. */
+static struct {
+  unsigned major;
+  unsigned minor;
+  const char *client_version;
+  const char *server_version;
+} win_version_table[] = {
+  /* This table must be sorted in descending order.
+   * Sources:
+   *   https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions
+   *   https://docs.microsoft.com/en-us/windows/desktop/api/winnt/
+   *     ns-winnt-_osversioninfoexa#remarks
+   */
+   /* Windows Server 2019 is indistinguishable from Windows Server 2016
+    * using GetVersionEx().
+   { 10,  0, NULL,                        "Windows Server 2019" }, */
+   // clang-format off
+   { 10,  0, "Windows 10",                "Windows Server 2016" },
+   {  6,  3, "Windows 8.1",               "Windows Server 2012 R2" },
+   {  6,  2, "Windows 8",                 "Windows Server 2012" },
+   {  6,  1, "Windows 7",                 "Windows Server 2008 R2" },
+   {  6,  0, "Windows Vista",             "Windows Server 2008" },
+   {  5,  2, "Windows XP Professional",   "Windows Server 2003" },
+   /* Windows XP did not have a server version, but we need something here */
+   {  5,  1, "Windows XP",                "Windows XP Server" },
+   {  5,  0, "Windows 2000 Professional", "Windows 2000 Server" },
+   /* Earlier versions are not supported by GetVersionEx(). */
+   {  0,  0, NULL,                        NULL }
+   // clang-format on
+};
+#endif /* defined(_WIN32) */
+
 /** Return a pointer to a description of our platform.
  */
 MOCK_IMPL(const char *,
@@ -49,31 +83,6 @@ get_uname,(void))
         int is_client = 0;
         int is_server = 0;
         const char *plat = NULL;
-        static struct {
-          unsigned major; unsigned minor;
-          const char *client_version; const char *server_version;
-        } win_version_table[] = {
-    /* This table must be sorted in descending order.
-     * Sources:
-     *   https://en.wikipedia.org/wiki/List_of_Microsoft_Windows_versions
-     *   https://docs.microsoft.com/en-us/windows/desktop/api/winnt/
-     *     ns-winnt-_osversioninfoexa#remarks
-     */
-    /* Windows Server 2019 is indistinguishable from Windows Server 2016
-     * using GetVersionEx().
-    { 10,  0, NULL,                        "Windows Server 2019" }, */
-    { 10,  0, "Windows 10",                "Windows Server 2016" },
-    {  6,  3, "Windows 8.1",               "Windows Server 2012 R2" },
-    {  6,  2, "Windows 8",                 "Windows Server 2012" },
-    {  6,  1, "Windows 7",                 "Windows Server 2008 R2" },
-    {  6,  0, "Windows Vista",             "Windows Server 2008" },
-    {  5,  2, "Windows XP Professional",   "Windows Server 2003" },
-    /* Windows XP did not have a server version, but we need something here */
-    {  5,  1, "Windows XP",                "Windows XP Server" },
-    {  5,  0, "Windows 2000 Professional", "Windows 2000 Server" },
-    /* Earlier versions are not supported by GetVersionEx(). */
-    {  0,  0, NULL,                        NULL }
-        };
         memset(&info, 0, sizeof(info));
         info.dwOSVersionInfoSize = sizeof(info);
         if (! GetVersionEx((LPOSVERSIONINFO)&info)) {
@@ -137,7 +146,7 @@ get_uname,(void))
         if (!is_server && !is_client) {
           strlcat(uname_result, " [client or server]", sizeof(uname_result));
         }
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
         /* LCOV_EXCL_START -- can't provoke uname failure */
         strlcpy(uname_result, "Unknown platform", sizeof(uname_result));
         /* LCOV_EXCL_STOP */
diff --git a/src/lib/osinfo/uname.h b/src/lib/osinfo/uname.h
index fcce629074..c6b1f43235 100644
--- a/src/lib/osinfo/uname.h
+++ b/src/lib/osinfo/uname.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,4 +15,4 @@
 
 MOCK_DECL(const char *, get_uname,(void));
 
-#endif
+#endif /* !defined(HAVE_TOR_UNAME_H) */
diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include
index 05414d2a96..ce1b6ecf59 100644
--- a/src/lib/process/.may_include
+++ b/src/lib/process/.may_include
@@ -1,17 +1,20 @@
 orconfig.h
 
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h
 lib/err/*.h
-lib/intmath/*.h
+lib/evloop/*.h
 lib/fs/*.h
+lib/intmath/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/process/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 
-ht.h
\ No newline at end of file
+ext/ht.h
diff --git a/src/lib/process/daemon.c b/src/lib/process/daemon.c
index 3b90bef671..b3b98a297e 100644
--- a/src/lib/process/daemon.c
+++ b/src/lib/process/daemon.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -165,7 +165,7 @@ finish_daemon(const char *desired_cwd)
 
   return 0;
 }
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
 /* defined(_WIN32) */
 int
 start_daemon(void)
diff --git a/src/lib/process/daemon.h b/src/lib/process/daemon.h
index 20920e0aae..23f3117898 100644
--- a/src/lib/process/daemon.h
+++ b/src/lib/process/daemon.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -18,4 +18,4 @@ int finish_daemon(const char *desired_cwd);
 
 bool start_daemon_has_been_called(void);
 
-#endif
+#endif /* !defined(TOR_DAEMON_H) */
diff --git a/src/lib/process/env.c b/src/lib/process/env.c
index 0060200ba1..517b655a5e 100644
--- a/src/lib/process/env.c
+++ b/src/lib/process/env.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,6 @@
 #include "lib/container/smartlist.h"
 #include "lib/log/util_bug.h"
 #include "lib/log/log.h"
-#include "lib/malloc/malloc.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -47,7 +46,7 @@ get_environment(void)
    * when we do a mostly-static build on OSX 10.7, the resulting binary won't
    * work on OSX 10.6. */
   return *_NSGetEnviron();
-#else /* !(defined(HAVE__NSGETENVIRON)) */
+#else /* !defined(HAVE__NSGETENVIRON) */
   return environ;
 #endif /* defined(HAVE__NSGETENVIRON) */
 }
diff --git a/src/lib/process/env.h b/src/lib/process/env.h
index 15d59351e0..7838dcaa90 100644
--- a/src/lib/process/env.h
+++ b/src/lib/process/env.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,4 +38,4 @@ void set_environment_variable_in_smartlist(struct smartlist_t *env_vars,
                                            const char *new_var,
                                            void (*free_old)(void*),
                                            int free_p);
-#endif
+#endif /* !defined(TOR_ENV_H) */
diff --git a/src/lib/process/include.am b/src/lib/process/include.am
index c6cc3a6699..18876b3f54 100644
--- a/src/lib/process/include.am
+++ b/src/lib/process/include.am
@@ -5,13 +5,17 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-process-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_process_a_SOURCES =		\
 	src/lib/process/daemon.c		\
 	src/lib/process/env.c			\
 	src/lib/process/pidfile.c		\
+	src/lib/process/process.c		\
+	src/lib/process/process_sys.c		\
+	src/lib/process/process_unix.c		\
+	src/lib/process/process_win32.c		\
 	src/lib/process/restrict.c		\
 	src/lib/process/setuid.c		\
-	src/lib/process/subprocess.c		\
 	src/lib/process/waitpid.c
 
 src_lib_libtor_process_testing_a_SOURCES = \
@@ -19,11 +23,15 @@ src_lib_libtor_process_testing_a_SOURCES = \
 src_lib_libtor_process_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_process_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/process/daemon.h		\
 	src/lib/process/env.h			\
 	src/lib/process/pidfile.h		\
+	src/lib/process/process.h		\
+	src/lib/process/process_sys.h		\
+	src/lib/process/process_unix.h		\
+	src/lib/process/process_win32.h		\
 	src/lib/process/restrict.h		\
 	src/lib/process/setuid.h		\
-	src/lib/process/subprocess.h		\
 	src/lib/process/waitpid.h
diff --git a/src/lib/process/lib_process.md b/src/lib/process/lib_process.md
new file mode 100644
index 0000000000..354129e70e
--- /dev/null
+++ b/src/lib/process/lib_process.md
@@ -0,0 +1,2 @@
+@dir /lib/process
+@brief lib/process: Launch and manage subprocesses.
diff --git a/src/lib/process/pidfile.c b/src/lib/process/pidfile.c
index 1b9d1c6d25..e7d9d2c47a 100644
--- a/src/lib/process/pidfile.c
+++ b/src/lib/process/pidfile.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/process/pidfile.h b/src/lib/process/pidfile.h
index dfeb39e046..d04302df3a 100644
--- a/src/lib/process/pidfile.h
+++ b/src/lib/process/pidfile.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,4 +13,4 @@
 
 int write_pidfile(const char *filename);
 
-#endif
+#endif /* !defined(TOR_PIDFILE_H) */
diff --git a/src/lib/process/process.c b/src/lib/process/process.c
new file mode 100644
index 0000000000..12c1f9a772
--- /dev/null
+++ b/src/lib/process/process.c
@@ -0,0 +1,798 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.c
+ * \brief Module for working with other processes.
+ **/
+
+#define PROCESS_PRIVATE
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_STDDEF_H
+#include <stddef.h>
+#endif
+
+/** A list of all <b>process_t</b> instances currently allocated. */
+static smartlist_t *processes;
+
+/**
+ * Boolean.  If true, then Tor may call execve or CreateProcess via
+ * tor_spawn_background.
+ **/
+static int may_spawn_background_process = 1;
+
+/** Structure to represent a child process. */
+struct process_t {
+  /** Process status. */
+  process_status_t status;
+
+  /** Which protocol is the process using? */
+  process_protocol_t protocol;
+
+  /** Which function to call when we have data ready from stdout? */
+  process_read_callback_t stdout_read_callback;
+
+  /** Which function to call when we have data ready from stderr? */
+  process_read_callback_t stderr_read_callback;
+
+  /** Which function call when our process terminated? */
+  process_exit_callback_t exit_callback;
+
+  /** Our exit code when the process have terminated. */
+  process_exit_code_t exit_code;
+
+  /** Name of the command we want to execute (for example: /bin/ls). */
+  char *command;
+
+  /** The arguments used for the new process. The format here is one argument
+   * per element of the smartlist_t. On Windows these arguments are combined
+   * together using the <b>tor_join_win_cmdline</b> function. On Unix the
+   * process name (argv[0]) and the trailing NULL is added automatically before
+   * the process is executed. */
+  smartlist_t *arguments;
+
+  /** The environment used for the new process. */
+  smartlist_t *environment;
+
+  /** Buffer to store data from stdout when it is read. */
+  buf_t *stdout_buffer;
+
+  /** Buffer to store data from stderr when it is read. */
+  buf_t *stderr_buffer;
+
+  /** Buffer to store data to stdin before it is written. */
+  buf_t *stdin_buffer;
+
+  /** Do we need to store some custom data with the process? */
+  void *data;
+
+#ifndef _WIN32
+  /** Our Unix process handle. */
+  process_unix_t *unix_process;
+#else
+  /** Our Win32 process handle. */
+  process_win32_t *win32_process;
+#endif /* !defined(_WIN32) */
+};
+
+/** Convert a given process status in <b>status</b> to its string
+ * representation. */
+const char *
+process_status_to_string(process_status_t status)
+{
+  switch (status) {
+  case PROCESS_STATUS_NOT_RUNNING:
+    return "not running";
+  case PROCESS_STATUS_RUNNING:
+    return "running";
+  case PROCESS_STATUS_ERROR:
+    return "error";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/** Convert a given process protocol in <b>protocol</b> to its string
+ * representation. */
+const char *
+process_protocol_to_string(process_protocol_t protocol)
+{
+  switch (protocol) {
+  case PROCESS_PROTOCOL_LINE:
+    return "Line";
+  case PROCESS_PROTOCOL_RAW:
+    return "Raw";
+  }
+
+  /* LCOV_EXCL_START */
+  tor_assert_unreached();
+  return NULL;
+  /* LCOV_EXCL_STOP */
+}
+
+/**
+ * Turn off may_spawn_background_process, so that all future calls to
+ * tor_spawn_background are guaranteed to fail.
+ **/
+void
+tor_disable_spawning_background_processes(void)
+{
+  may_spawn_background_process = 0;
+}
+
+/** Initialize the Process subsystem.  This function initializes the Process
+ * subsystem's global state. For cleaning up, <b>process_free_all()</b> should
+ * be called. */
+void
+process_init(void)
+{
+  processes = smartlist_new();
+
+#ifdef _WIN32
+  process_win32_init();
+#endif
+}
+
+/** Free up all resources that is handled by the Process subsystem. Note that
+ * this call does not terminate already running processes. */
+void
+process_free_all(void)
+{
+#ifdef _WIN32
+  process_win32_deinit();
+#endif
+
+  SMARTLIST_FOREACH(processes, process_t *, x, process_free(x));
+  smartlist_free(processes);
+}
+
+/** Get a list of all processes.  This function returns a smartlist of
+ * <b>process_t</b> containing all the currently allocated processes. */
+const smartlist_t *
+process_get_all_processes(void)
+{
+  return processes;
+}
+
+/** Allocate and initialize a new process.  This function returns a newly
+ * allocated and initialized process data, which can be used to configure and
+ * later run a subprocess of Tor. Use the various <b>process_set_*()</b>
+ * methods to configure it and run the process using <b>process_exec()</b>. Use
+ * <b>command</b> to specify the path to the command to run. You can either
+ * specify an absolute path to the command or relative where Tor will use the
+ * underlying operating system's functionality for finding the command to run.
+ * */
+process_t *
+process_new(const char *command)
+{
+  tor_assert(command);
+
+  process_t *process;
+  process = tor_malloc_zero(sizeof(process_t));
+
+  /* Set our command. */
+  process->command = tor_strdup(command);
+
+  /* By default we are not running. */
+  process->status = PROCESS_STATUS_NOT_RUNNING;
+
+  /* Prepare process environment. */
+  process->arguments = smartlist_new();
+  process->environment = smartlist_new();
+
+  /* Prepare the buffers. */
+  process->stdout_buffer = buf_new();
+  process->stderr_buffer = buf_new();
+  process->stdin_buffer = buf_new();
+
+#ifndef _WIN32
+  /* Prepare our Unix process handle. */
+  process->unix_process = process_unix_new();
+#else
+  /* Prepare our Win32 process handle. */
+  process->win32_process = process_win32_new();
+#endif /* !defined(_WIN32) */
+
+  smartlist_add(processes, process);
+
+  return process;
+}
+
+/** Deallocate the given process in <b>process</b>. */
+void
+process_free_(process_t *process)
+{
+  if (! process)
+    return;
+
+  /* Cleanup parameters. */
+  tor_free(process->command);
+
+  /* Cleanup arguments and environment. */
+  SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x));
+  smartlist_free(process->arguments);
+
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+
+  /* Cleanup the buffers. */
+  buf_free(process->stdout_buffer);
+  buf_free(process->stderr_buffer);
+  buf_free(process->stdin_buffer);
+
+#ifndef _WIN32
+  /* Cleanup our Unix process handle. */
+  process_unix_free(process->unix_process);
+#else
+  /* Cleanup our Win32 process handle. */
+  process_win32_free(process->win32_process);
+#endif /* !defined(_WIN32) */
+
+  smartlist_remove(processes, process);
+
+  tor_free(process);
+}
+
+/** Execute the given process.  This function executes the given process as a
+ * subprocess of Tor. Returns <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_exec(process_t *process)
+{
+  tor_assert(process);
+
+  if (BUG(may_spawn_background_process == 0))
+    return PROCESS_STATUS_ERROR;
+
+  process_status_t status = PROCESS_STATUS_NOT_RUNNING;
+
+  log_info(LD_PROCESS, "Starting new process: %s", process->command);
+
+#ifndef _WIN32
+  status = process_unix_exec(process);
+#else
+  status = process_win32_exec(process);
+#endif
+
+  /* Update our state. */
+  process_set_status(process, status);
+
+  if (status != PROCESS_STATUS_RUNNING) {
+    log_warn(LD_PROCESS, "Failed to start process: %s",
+             process_get_command(process));
+  }
+
+  return status;
+}
+
+/** Terminate the given process. Returns true on success,
+ * otherwise false. */
+bool
+process_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  /* Terminating a non-running process isn't going to work. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  log_debug(LD_PROCESS, "Terminating process");
+
+#ifndef _WIN32
+  return process_unix_terminate(process);
+#else
+  return process_win32_terminate(process);
+#endif
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+#ifndef _WIN32
+  return process_unix_get_pid(process);
+#else
+  return process_win32_get_pid(process);
+#endif
+}
+
+/** Set the callback function for output from the child process's standard out
+ * handle.  This function sets the callback function which is called every time
+ * the child process have written output to its standard out file handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stdout_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stdout_read_callback = callback;
+}
+
+/** Set the callback function for output from the child process's standard
+ * error handle.  This function sets the callback function which is called
+ * every time the child process have written output to its standard error file
+ * handle.
+ *
+ * Use <b>process_set_protocol(process, PROCESS_PROTOCOL_LINE)</b> if you want
+ * the callback to only contain complete "\n" or "\r\n" terminated lines. */
+void
+process_set_stderr_read_callback(process_t *process,
+                                 process_read_callback_t callback)
+{
+  tor_assert(process);
+  process->stderr_read_callback = callback;
+}
+
+/** Set the callback function for process exit notification.  The
+ * <b>callback</b> function will be called every time your child process have
+ * terminated. */
+void
+process_set_exit_callback(process_t *process,
+                          process_exit_callback_t callback)
+{
+  tor_assert(process);
+  process->exit_callback = callback;
+}
+
+/** Get the current command of the given process. */
+const char *
+process_get_command(const process_t *process)
+{
+  tor_assert(process);
+  return process->command;
+}
+
+void
+process_set_protocol(process_t *process, process_protocol_t protocol)
+{
+  tor_assert(process);
+  process->protocol = protocol;
+}
+
+/** Get the currently used protocol of the given process. */
+process_protocol_t
+process_get_protocol(const process_t *process)
+{
+  tor_assert(process);
+  return process->protocol;
+}
+
+/** Set opague pointer to data.  This function allows you to store a pointer to
+ * your own data in the given process. Use <b>process_get_data()</b> in the
+ * various callback functions to retrieve the data again.
+ *
+ * Note that the given process does NOT take ownership of the data and you are
+ * responsible for freeing up any resources allocated by the given data.
+ * */
+void
+process_set_data(process_t *process, void *data)
+{
+  tor_assert(process);
+  process->data = data;
+}
+
+/** Get the opaque pointer to callback data from the given process.  This
+ * function allows you get the data you stored with <b>process_set_data()</b>
+ * in the different callback functions. */
+void *
+process_get_data(const process_t *process)
+{
+  tor_assert(process);
+  return process->data;
+}
+
+/** Set the status of a given process. */
+void
+process_set_status(process_t *process, process_status_t status)
+{
+  tor_assert(process);
+  process->status = status;
+}
+
+/** Get the status of the given process. */
+process_status_t
+process_get_status(const process_t *process)
+{
+  tor_assert(process);
+  return process->status;
+}
+
+/** Append an argument to the list of arguments in the given process. */
+void
+process_append_argument(process_t *process, const char *argument)
+{
+  tor_assert(process);
+  tor_assert(argument);
+
+  smartlist_add(process->arguments, tor_strdup(argument));
+}
+
+/** Returns a list of arguments (excluding the command itself) from the
+ * given process. */
+const smartlist_t *
+process_get_arguments(const process_t *process)
+{
+  tor_assert(process);
+  return process->arguments;
+}
+
+/** Returns a newly allocated Unix style argument vector. Use <b>tor_free()</b>
+ * to deallocate it after use. */
+char **
+process_get_argv(const process_t *process)
+{
+  tor_assert(process);
+
+  /** Generate a Unix style process argument vector from our process's
+   * arguments smartlist_t. */
+  char **argv = NULL;
+
+  char *filename = process->command;
+  const smartlist_t *arguments = process->arguments;
+  const size_t size = smartlist_len(arguments);
+
+  /* Make space for the process filename as argv[0] and a trailing NULL. */
+  argv = tor_malloc_zero(sizeof(char *) * (size + 2));
+
+  /* Set our filename as first argument. */
+  argv[0] = filename;
+
+  /* Put in the rest of the values from arguments. */
+  SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) {
+    tor_assert(arg_val != NULL);
+
+    argv[arg_val_sl_idx + 1] = arg_val;
+  } SMARTLIST_FOREACH_END(arg_val);
+
+  return argv;
+}
+
+/** This function clears the internal environment and copies over every string
+ * from <b>env</b> as the new environment. */
+void
+process_reset_environment(process_t *process, const smartlist_t *env)
+{
+  tor_assert(process);
+  tor_assert(env);
+
+  /* Cleanup old environment. */
+  SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x));
+  smartlist_free(process->environment);
+  process->environment = smartlist_new();
+
+  SMARTLIST_FOREACH(env, char *, x,
+                    smartlist_add(process->environment, tor_strdup(x)));
+}
+
+/** Set the given <b>key</b>/<b>value</b> pair as environment variable in the
+ * given process. */
+void
+process_set_environment(process_t *process,
+                        const char *key,
+                        const char *value)
+{
+  tor_assert(process);
+  tor_assert(key);
+  tor_assert(value);
+
+  smartlist_add_asprintf(process->environment, "%s=%s", key, value);
+}
+
+/** Returns a newly allocated <b>process_environment_t</b> containing the
+ * environment variables for the given process. */
+process_environment_t *
+process_get_environment(const process_t *process)
+{
+  tor_assert(process);
+  return process_environment_make(process->environment);
+}
+
+#ifndef _WIN32
+/** Get the internal handle for the Unix backend. */
+process_unix_t *
+process_get_unix_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->unix_process);
+  return process->unix_process;
+}
+#else /* defined(_WIN32) */
+/** Get the internal handle for Windows backend. */
+process_win32_t *
+process_get_win32_process(const process_t *process)
+{
+  tor_assert(process);
+  tor_assert(process->win32_process);
+  return process->win32_process;
+}
+#endif /* !defined(_WIN32) */
+
+/** Write <b>size</b> bytes of <b>data</b> to the given process's standard
+ * input. */
+void
+process_write(process_t *process,
+              const uint8_t *data, size_t size)
+{
+  tor_assert(process);
+  tor_assert(data);
+
+  buf_add(process->stdin_buffer, (char *)data, size);
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** As tor_vsnprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_vprintf(process_t *process,
+                const char *format, va_list args)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  int size;
+  char *data;
+
+  size = tor_vasprintf(&data, format, args);
+  tor_assert(data != NULL);
+  process_write(process, (uint8_t *)data, size);
+  tor_free(data);
+}
+
+/** As tor_snprintf(), but write the data to the given process's standard
+ * input. */
+void
+process_printf(process_t *process,
+               const char *format, ...)
+{
+  tor_assert(process);
+  tor_assert(format);
+
+  va_list ap;
+  va_start(ap, format);
+  process_vprintf(process, format, ap);
+  va_end(ap);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard output
+ * handle. */
+void
+process_notify_event_stdout(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stdout(process, process->stdout_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stdout_buffer,
+                      process->stdout_read_callback);
+}
+
+/** This function is called by the Process backend when a given process have
+ * data that is ready to be read from the child process's standard error
+ * handle. */
+void
+process_notify_event_stderr(process_t *process)
+{
+  tor_assert(process);
+
+  int ret;
+  ret = process_read_stderr(process, process->stderr_buffer);
+
+  if (ret > 0)
+    process_read_data(process,
+                      process->stderr_buffer,
+                      process->stderr_read_callback);
+}
+
+/** This function is called by the Process backend when a given process is
+ * allowed to begin writing data to the standard input of the child process. */
+void
+process_notify_event_stdin(process_t *process)
+{
+  tor_assert(process);
+
+  process_write_stdin(process, process->stdin_buffer);
+}
+
+/** This function is called by the Process backend when a given process have
+ * terminated. The exit status code is passed in <b>exit_code</b>. We mark the
+ * process as no longer running and calls the <b>exit_callback</b> with
+ * information about the process termination. The given <b>process</b> is
+ * free'd iff the exit_callback returns true. */
+void
+process_notify_event_exit(process_t *process, process_exit_code_t exit_code)
+{
+  tor_assert(process);
+
+  log_debug(LD_PROCESS,
+            "Process terminated with exit code: %"PRIu64, exit_code);
+
+  /* Update our state. */
+  process_set_status(process, PROCESS_STATUS_NOT_RUNNING);
+  process->exit_code = exit_code;
+
+  /* Call our exit callback, if it exists. */
+  bool free_process_handle = false;
+
+  /* The exit callback will tell us if we should process_free() our handle. */
+  if (process->exit_callback)
+    free_process_handle = process->exit_callback(process, exit_code);
+
+  if (free_process_handle)
+    process_free(process);
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard out handle. Returns the number of
+ * bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stdout(process, buffer);
+#else
+  return process_win32_read_stdout(process, buffer);
+#endif
+}
+
+/** This function is called whenever the Process backend have notified us that
+ * there is data to be read from its standard error handle. Returns the number
+ * of bytes that have been put into the given buffer. */
+MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  return process_unix_read_stderr(process, buffer);
+#else
+  return process_win32_read_stderr(process, buffer);
+#endif
+}
+
+/** This function calls the backend function for the given process whenever
+ * there is data to be written to the backends' file handles. */
+MOCK_IMPL(STATIC void, process_write_stdin,
+          (process_t *process, buf_t *buffer))
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+#ifndef _WIN32
+  process_unix_write(process, buffer);
+#else
+  process_win32_write(process, buffer);
+#endif
+}
+
+/** This function calls the protocol handlers based on the value of
+ * <b>process_get_protocol(process)</b>. Currently we call
+ * <b>process_read_buffer()</b> for <b>PROCESS_PROTOCOL_RAW</b> and
+ * <b>process_read_lines()</b> for <b>PROCESS_PROTOCOL_LINE</b>. */
+STATIC void
+process_read_data(process_t *process,
+                  buf_t *buffer,
+                  process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  switch (process_get_protocol(process)) {
+  case PROCESS_PROTOCOL_RAW:
+    process_read_buffer(process, buffer, callback);
+    break;
+  case PROCESS_PROTOCOL_LINE:
+    process_read_lines(process, buffer, callback);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_unreached();
+    return;
+    /* LCOV_EXCL_STOP */
+  }
+}
+
+/** This function takes the content of the given <b>buffer</b> and passes it to
+ * the given <b>callback</b> function, but ensures that an additional zero byte
+ * is added to the end of the data such that the given callback implementation
+ * can threat the content as a ASCIIZ string. */
+STATIC void
+process_read_buffer(process_t *process,
+                    buf_t *buffer,
+                    process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer);
+
+  /* We allocate an extra byte for the zero byte in the end. */
+  char *data = tor_malloc_zero(size + 1);
+
+  buf_get_bytes(buffer, data, size);
+  log_debug(LD_PROCESS, "Read data from process");
+
+  if (callback)
+    callback(process, data, size);
+
+  tor_free(data);
+}
+
+/** This function tries to extract complete lines from the given <b>buffer</b>
+ * and calls the given <b>callback</b> function whenever it has a complete
+ * line. Before calling <b>callback</b> we remove the trailing "\n" or "\r\n"
+ * from the line. If we are unable to extract a complete line we leave the data
+ * in the buffer for next call. */
+STATIC void
+process_read_lines(process_t *process,
+                   buf_t *buffer,
+                   process_read_callback_t callback)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  const size_t size = buf_datalen(buffer) + 1;
+  size_t line_size = 0;
+  char *data = tor_malloc_zero(size);
+  int ret;
+
+  while (true) {
+    line_size = size;
+    ret = buf_get_line(buffer, data, &line_size);
+
+    /* A complete line should always be smaller than the size of our
+     * buffer. */
+    tor_assert(ret != -1);
+
+    /* Remove \n from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\n') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    /* Remove \r from the end of the line. */
+    if (line_size >= 1 && data[line_size - 1] == '\r') {
+      data[line_size - 1] = '\0';
+      --line_size;
+    }
+
+    if (ret == 1) {
+      log_debug(LD_PROCESS, "Read line from process: \"%s\"", data);
+
+      if (callback)
+        callback(process, data, line_size);
+
+      /* We have read a whole line, let's see if there is more lines to read.
+       * */
+      continue;
+    }
+
+    /* No complete line for us to read. We are done for now. */
+    tor_assert_nonfatal(ret == 0);
+    break;
+  }
+
+  tor_free(data);
+}
diff --git a/src/lib/process/process.h b/src/lib/process/process.h
new file mode 100644
index 0000000000..8879ec4f21
--- /dev/null
+++ b/src/lib/process/process.h
@@ -0,0 +1,148 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process.h
+ * \brief Header for process.c
+ **/
+
+#ifndef TOR_PROCESS_H
+#define TOR_PROCESS_H
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdbool.h>
+
+/** Maximum number of bytes to write to a process' stdin. */
+#define PROCESS_MAX_WRITE (1024)
+
+/** Maximum number of bytes to read from a process' stdout/stderr. */
+#define PROCESS_MAX_READ  (1024)
+
+typedef enum {
+  /** The process is not running. */
+  PROCESS_STATUS_NOT_RUNNING,
+
+  /** The process is running. */
+  PROCESS_STATUS_RUNNING,
+
+  /** The process is in an erroneous state. */
+  PROCESS_STATUS_ERROR
+} process_status_t;
+
+const char *process_status_to_string(process_status_t status);
+
+typedef enum {
+  /** Pass complete newline-terminated lines to the
+   *  callback (with the LF or CRLF removed). */
+  PROCESS_PROTOCOL_LINE,
+
+  /** Pass the raw response from read() to the callback. */
+  PROCESS_PROTOCOL_RAW
+} process_protocol_t;
+
+const char *process_protocol_to_string(process_protocol_t protocol);
+
+void tor_disable_spawning_background_processes(void);
+
+struct smartlist_t;
+
+struct process_t;
+typedef struct process_t process_t;
+
+typedef uint64_t process_exit_code_t;
+typedef uint64_t process_pid_t;
+
+typedef void (*process_read_callback_t)(process_t *,
+                                        const char *,
+                                        size_t);
+typedef bool
+(*process_exit_callback_t)(process_t *, process_exit_code_t);
+
+void process_init(void);
+void process_free_all(void);
+const struct smartlist_t *process_get_all_processes(void);
+
+process_t *process_new(const char *command);
+void process_free_(process_t *process);
+#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s))
+
+process_status_t process_exec(process_t *process);
+bool process_terminate(process_t *process);
+
+process_pid_t process_get_pid(process_t *process);
+
+void process_set_stdout_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_stderr_read_callback(process_t *,
+                                      process_read_callback_t);
+void process_set_exit_callback(process_t *,
+                               process_exit_callback_t);
+
+const char *process_get_command(const process_t *process);
+
+void process_append_argument(process_t *process, const char *argument);
+const struct smartlist_t *process_get_arguments(const process_t *process);
+char **process_get_argv(const process_t *process);
+
+void process_reset_environment(process_t *process,
+                               const struct smartlist_t *env);
+void process_set_environment(process_t *process,
+                             const char *key,
+                             const char *value);
+
+struct process_environment_t;
+struct process_environment_t *process_get_environment(const process_t *);
+
+void process_set_protocol(process_t *process, process_protocol_t protocol);
+process_protocol_t process_get_protocol(const process_t *process);
+
+void process_set_data(process_t *process, void *data);
+void *process_get_data(const process_t *process);
+
+void process_set_status(process_t *process, process_status_t status);
+process_status_t process_get_status(const process_t *process);
+
+#ifndef _WIN32
+struct process_unix_t;
+struct process_unix_t *process_get_unix_process(const process_t *process);
+#else
+struct process_win32_t;
+struct process_win32_t *process_get_win32_process(const process_t *process);
+#endif /* !defined(_WIN32) */
+
+void process_write(process_t *process,
+                   const uint8_t *data, size_t size);
+void process_vprintf(process_t *process,
+                     const char *format, va_list args) CHECK_PRINTF(2, 0);
+void process_printf(process_t *process,
+                    const char *format, ...) CHECK_PRINTF(2, 3);
+
+void process_notify_event_stdout(process_t *process);
+void process_notify_event_stderr(process_t *process);
+void process_notify_event_stdin(process_t *process);
+void process_notify_event_exit(process_t *process,
+                               process_exit_code_t);
+
+#ifdef PROCESS_PRIVATE
+struct buf_t;
+MOCK_DECL(STATIC int, process_read_stdout, (process_t *, struct buf_t *));
+MOCK_DECL(STATIC int, process_read_stderr, (process_t *, struct buf_t *));
+MOCK_DECL(STATIC void, process_write_stdin, (process_t *, struct buf_t *));
+
+STATIC void process_read_data(process_t *process,
+                              struct buf_t *buffer,
+                              process_read_callback_t callback);
+STATIC void process_read_buffer(process_t *process,
+                                struct buf_t *buffer,
+                                process_read_callback_t callback);
+STATIC void process_read_lines(process_t *process,
+                               struct buf_t *buffer,
+                               process_read_callback_t callback);
+#endif /* defined(PROCESS_PRIVATE) */
+
+#endif /* !defined(TOR_PROCESS_H) */
diff --git a/src/lib/process/process_sys.c b/src/lib/process/process_sys.c
new file mode 100644
index 0000000000..c8332ba91e
--- /dev/null
+++ b/src/lib/process/process_sys.c
@@ -0,0 +1,34 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.c
+ * \brief Subsystem object for process setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/process/process_sys.h"
+#include "lib/process/process.h"
+
+static int
+subsys_process_initialize(void)
+{
+  process_init();
+  return 0;
+}
+
+static void
+subsys_process_shutdown(void)
+{
+  process_free_all();
+}
+
+const subsys_fns_t sys_process = {
+  .name = "process",
+  SUBSYS_DECLARE_LOCATION(),
+  .level = -18,
+  .supported = true,
+  .initialize = subsys_process_initialize,
+  .shutdown = subsys_process_shutdown
+};
diff --git a/src/lib/process/process_sys.h b/src/lib/process/process_sys.h
new file mode 100644
index 0000000000..97b3aaebd0
--- /dev/null
+++ b/src/lib/process/process_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_sys.h
+ * \brief Declare subsystem object for the process module.
+ **/
+
+#ifndef TOR_PROCESS_SYS_H
+#define TOR_PROCESS_SYS_H
+
+extern const struct subsys_fns_t sys_process;
+
+#endif /* !defined(TOR_PROCESS_SYS_H) */
diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c
new file mode 100644
index 0000000000..2b47e1874d
--- /dev/null
+++ b/src/lib/process/process_unix.c
@@ -0,0 +1,698 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.c
+ * \brief Module for working with Unix processes.
+ **/
+
+#define PROCESS_UNIX_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/process/process.h"
+#include "lib/process/process_unix.h"
+#include "lib/process/waitpid.h"
+#include "lib/process/env.h"
+
+#include <stdio.h>
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef HAVE_ERRNO_H
+#include <errno.h>
+#endif
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+#include <sys/prctl.h>
+#endif
+
+#if HAVE_SIGNAL_H
+#include <signal.h>
+#endif
+
+#ifndef _WIN32
+
+/** Maximum number of file descriptors, if we cannot get it via sysconf() */
+#define DEFAULT_MAX_FD 256
+
+/** Internal state for Unix handles. */
+struct process_unix_handle_t {
+  /** Unix File Descriptor. */
+  int fd;
+
+  /** Have we reached end of file? */
+  bool reached_eof;
+
+  /** Event structure for libevent. */
+  struct event *event;
+
+  /** Are we writing? */
+  bool is_writing;
+};
+
+/** Internal state for our Unix process. */
+struct process_unix_t {
+  /** Standard in handle. */
+  process_unix_handle_t stdin_handle;
+
+  /** Standard out handle. */
+  process_unix_handle_t stdout_handle;
+
+  /** Standard error handle. */
+  process_unix_handle_t stderr_handle;
+
+  /** The process identifier of our process. */
+  pid_t pid;
+
+  /** Waitpid Callback structure. */
+  waitpid_callback_t *waitpid;
+};
+
+/** Returns a newly allocated <b>process_unix_t</b>. */
+process_unix_t *
+process_unix_new(void)
+{
+  process_unix_t *unix_process;
+  unix_process = tor_malloc_zero(sizeof(process_unix_t));
+
+  unix_process->stdin_handle.fd = -1;
+  unix_process->stderr_handle.fd = -1;
+  unix_process->stdout_handle.fd = -1;
+
+  return unix_process;
+}
+
+/** Deallocates the given <b>unix_process</b>. */
+void
+process_unix_free_(process_unix_t *unix_process)
+{
+  if (! unix_process)
+    return;
+
+  /* Clean up our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+
+  /* FIXME(ahf): Refactor waitpid code? */
+  unix_process->waitpid = NULL;
+
+  /* Close all our file descriptors. */
+  process_unix_close_file_descriptors(unix_process);
+
+  tor_event_free(unix_process->stdout_handle.event);
+  tor_event_free(unix_process->stderr_handle.event);
+  tor_event_free(unix_process->stdin_handle.event);
+
+  tor_free(unix_process);
+}
+
+/** Executes the given process as a child process of Tor.  This function is
+ * responsible for setting up the child process and run it. This includes
+ * setting up pipes for interprocess communication, initialize the waitpid
+ * callbacks, and finally run fork() followed by execve(). Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_unix_exec(process_t *process)
+{
+  static int max_fd = -1;
+
+  process_unix_t *unix_process;
+  pid_t pid;
+  int stdin_pipe[2];
+  int stdout_pipe[2];
+  int stderr_pipe[2];
+  int retval, fd;
+
+  unix_process = process_get_unix_process(process);
+
+  /* Create standard in pipe. */
+  retval = pipe(stdin_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdin "
+             "communication with process: %s",
+             strerror(errno));
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard out pipe. */
+  retval = pipe(stdout_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stdout "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create standard error pipe. */
+  retval = pipe(stderr_pipe);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Unable to create pipe for stderr "
+             "communication with process: %s",
+             strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+#ifdef _SC_OPEN_MAX
+  if (-1 == max_fd) {
+    max_fd = (int)sysconf(_SC_OPEN_MAX);
+
+    if (max_fd == -1) {
+      max_fd = DEFAULT_MAX_FD;
+      log_warn(LD_PROCESS,
+               "Cannot find maximum file descriptor, assuming: %d", max_fd);
+    }
+  }
+#else /* !defined(_SC_OPEN_MAX) */
+  max_fd = DEFAULT_MAX_FD;
+#endif /* defined(_SC_OPEN_MAX) */
+
+  pid = fork();
+
+  if (0 == pid) {
+    /* This code is running in the child process context. */
+
+#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
+    /* Attempt to have the kernel issue a SIGTERM if the parent
+     * goes away. Certain attributes of the binary being execve()ed
+     * will clear this during the execve() call, but it's better
+     * than nothing.
+     */
+    prctl(PR_SET_PDEATHSIG, SIGTERM);
+#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
+
+    /* Link process stdout to the write end of the pipe. */
+    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stderr to the write end of the pipe. */
+    retval = dup2(stderr_pipe[1], STDERR_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Link process stdin to the read end of the pipe */
+    retval = dup2(stdin_pipe[0], STDIN_FILENO);
+    if (-1 == retval)
+      goto error;
+
+    /* Close our pipes now after they have been dup2()'ed. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /* Close all other fds, including the read end of the pipe.  XXX: We should
+     * now be doing enough FD_CLOEXEC setting to make this needless.
+     */
+    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++)
+      close(fd);
+
+    /* Create the argv value for our new process. */
+    char **argv = process_get_argv(process);
+
+    /* Create the env value for our new process. */
+    process_environment_t *env = process_get_environment(process);
+
+    /* Call the requested program. */
+    execve(argv[0], argv, env->unixoid_environment_block);
+
+    /* If we made it here it is because execve failed :-( */
+    tor_free(argv);
+    process_environment_free(env);
+
+ error:
+    fprintf(stderr, "Error from child process: %s", strerror(errno));
+    _exit(1);
+  }
+
+  /* We are in the parent process. */
+  if (-1 == pid) {
+    log_warn(LD_PROCESS,
+             "Failed to create child process: %s", strerror(errno));
+
+    /** Cleanup standard in pipe. */
+    close(stdin_pipe[0]);
+    close(stdin_pipe[1]);
+
+    /** Cleanup standard out pipe. */
+    close(stdout_pipe[0]);
+    close(stdout_pipe[1]);
+
+    /** Cleanup standard error pipe. */
+    close(stderr_pipe[0]);
+    close(stderr_pipe[1]);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Register our PID. */
+  unix_process->pid = pid;
+
+  /* Setup waitpid callbacks. */
+  unix_process->waitpid = set_waitpid_callback(pid,
+                                               process_unix_waitpid_callback,
+                                               process);
+
+  /* Handle standard out. */
+  unix_process->stdout_handle.fd = stdout_pipe[0];
+  retval = close(stdout_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard error. */
+  unix_process->stderr_handle.fd = stderr_pipe[0];
+  retval = close(stderr_pipe[1]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS,
+             "Failed to close write end of standard error pipe: %s",
+             strerror(errno));
+  }
+
+  /* Handle standard in. */
+  unix_process->stdin_handle.fd = stdin_pipe[1];
+  retval = close(stdin_pipe[0]);
+
+  if (-1 == retval) {
+    log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s",
+             strerror(errno));
+  }
+
+  /* Setup our handles. */
+  process_unix_setup_handle(process,
+                            &unix_process->stdout_handle,
+                            EV_READ|EV_PERSIST,
+                            stdout_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stderr_handle,
+                            EV_READ|EV_PERSIST,
+                            stderr_read_callback);
+
+  process_unix_setup_handle(process,
+                            &unix_process->stdin_handle,
+                            EV_WRITE|EV_PERSIST,
+                            stdin_write_callback);
+
+  /* Start reading from standard out and standard error. */
+  process_unix_start_reading(&unix_process->stdout_handle);
+  process_unix_start_reading(&unix_process->stderr_handle);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_unix_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* All running processes should have a waitpid. */
+  if (BUG(unix_process->waitpid == NULL))
+    return false;
+
+  bool success = true;
+
+  /* Send a SIGTERM to our child process. */
+  int ret;
+
+  ret = kill(unix_process->pid, SIGTERM);
+
+  if (ret == -1) {
+    log_warn(LD_PROCESS, "Unable to terminate process: %s",
+             strerror(errno));
+    success = false;
+  }
+
+  /* Close all our FD's. */
+  if (! process_unix_close_file_descriptors(unix_process))
+    success = false;
+
+  return success;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_unix_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+  return (process_pid_t)unix_process->pid;
+}
+
+/** Write the given <b>buffer</b> as input to the given <b>process</b>'s
+ * standard input. Returns the number of bytes written. */
+int
+process_unix_write(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  size_t buffer_flush_len = buf_datalen(buffer);
+  const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len);
+
+  /* If we have data to write (when buffer_flush_len > 0) and we are not
+   * currently getting file descriptor events from the kernel, we tell the
+   * kernel to start notifying us about when we can write to our file
+   * descriptor and return. */
+  if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) {
+    process_unix_start_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We don't have any data to write, but the kernel is currently notifying us
+   * about whether we are able to write or not. Tell the kernel to stop
+   * notifying us until we have data to write. */
+  if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) {
+    process_unix_stop_writing(&unix_process->stdin_handle);
+    return 0;
+  }
+
+  /* We have data to write and the kernel have told us to write it. */
+  return buf_flush_to_pipe(buffer,
+                           process_get_unix_process(process)->stdin_handle.fd,
+                           max_to_write, &buffer_flush_len);
+}
+
+/** Read data from the given process's standard output and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stdout(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stdout_handle,
+                                  buffer);
+}
+
+/** Read data from the given process's standard error and put it into
+ * <b>buffer</b>. Returns the number of bytes read. */
+int
+process_unix_read_stderr(process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  return process_unix_read_handle(process,
+                                  &unix_process->stderr_handle,
+                                  buffer);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard output. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stdout_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdout(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * read from the child process's standard error. We notify the Process
+ * subsystem, which is then responsible for calling back to us for doing the
+ * actual reading of the data. */
+STATIC void
+stderr_read_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stderr(process);
+}
+
+/** This function is called whenever libevent thinks we have data that could be
+ * written the child process's standard input. We notify the Process subsystem,
+ * which is then responsible for calling back to us for doing the actual write
+ * of the data. */
+STATIC void
+stdin_write_callback(evutil_socket_t fd, short event, void *data)
+{
+  (void)fd;
+  (void)event;
+
+  process_t *process = data;
+  tor_assert(process);
+
+  process_notify_event_stdin(process);
+}
+
+/** This function tells libevent that we are interested in receiving read
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * read events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_reading(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+}
+
+/** This function tells libevent that we are interested in receiving write
+ * events from the given <b>handle</b>. */
+STATIC void
+process_unix_start_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (event_add(handle->event, NULL))
+    log_warn(LD_PROCESS,
+             "Unable to add libevent event for handle.");
+
+  handle->is_writing = true;
+}
+
+/** This function tells libevent that we are no longer interested in receiving
+ * write events from the given <b>handle</b>. */
+STATIC void
+process_unix_stop_writing(process_unix_handle_t *handle)
+{
+  tor_assert(handle);
+
+  if (handle->event == NULL)
+    return;
+
+  if (event_del(handle->event))
+    log_warn(LD_PROCESS,
+             "Unable to delete libevent event for handle.");
+
+  handle->is_writing = false;
+}
+
+/** This function is called when the waitpid system have detected that our
+ * process have terminated. We disable the waitpid system and notify the
+ * Process subsystem that we have terminated. */
+STATIC void
+process_unix_waitpid_callback(int status, void *data)
+{
+  tor_assert(data);
+
+  process_t *process = data;
+  process_unix_t *unix_process = process_get_unix_process(process);
+
+  /* Remove our waitpid callback. */
+  clear_waitpid_callback(unix_process->waitpid);
+  unix_process->waitpid = NULL;
+
+  /* Notify our process. */
+  process_notify_event_exit(process, status);
+
+  /* Make sure you don't modify the process after we have called
+   * process_notify_event_exit() on it, to allow users to process_free() it in
+   * the exit callback. */
+}
+
+/** This function sets the file descriptor in the <b>handle</b> as non-blocking
+ * and configures the libevent event structure based on the given <b>flags</b>
+ * to ensure that <b>callback</b> is called whenever we have events on the
+ * given <b>handle</b>. */
+STATIC void
+process_unix_setup_handle(process_t *process,
+                          process_unix_handle_t *handle,
+                          short flags,
+                          event_callback_fn callback)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(callback);
+
+  /* Put our file descriptor into non-blocking mode. */
+  if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) {
+    log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s",
+             strerror(errno));
+  }
+
+  /* Setup libevent event. */
+  handle->event = tor_event_new(tor_libevent_get_base(),
+                                handle->fd,
+                                flags,
+                                callback,
+                                process);
+}
+
+/** This function reads data from the given <b>handle</b> and puts it into
+ * <b>buffer</b>. Returns the number of bytes read this way. */
+STATIC int
+process_unix_read_handle(process_t *process,
+                         process_unix_handle_t *handle,
+                         buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(handle);
+  tor_assert(buffer);
+
+  int ret = 0;
+  int eof = 0;
+  int error = 0;
+
+  ret = buf_read_from_pipe(buffer,
+                           handle->fd,
+                           PROCESS_MAX_READ,
+                           &eof,
+                           &error);
+
+  if (error)
+    log_warn(LD_PROCESS,
+             "Unable to read data: %s", strerror(error));
+
+  if (eof) {
+    handle->reached_eof = true;
+    process_unix_stop_reading(handle);
+  }
+
+  return ret;
+}
+
+/** Close the standard in, out, and error handles of the given
+ * <b>unix_process</b>. */
+STATIC bool
+process_unix_close_file_descriptors(process_unix_t *unix_process)
+{
+  tor_assert(unix_process);
+
+  int ret;
+  bool success = true;
+
+  /* Stop reading and writing before we close() our
+   * file descriptors. */
+  if (! unix_process->stdout_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stdout_handle);
+
+  if (! unix_process->stderr_handle.reached_eof)
+    process_unix_stop_reading(&unix_process->stderr_handle);
+
+  if (unix_process->stdin_handle.is_writing)
+    process_unix_stop_writing(&unix_process->stdin_handle);
+
+  if (unix_process->stdin_handle.fd != -1) {
+    ret = close(unix_process->stdin_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard in");
+      success = false;
+    }
+
+    unix_process->stdin_handle.fd = -1;
+  }
+
+  if (unix_process->stdout_handle.fd != -1) {
+    ret = close(unix_process->stdout_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard out");
+      success = false;
+    }
+
+    unix_process->stdout_handle.fd = -1;
+  }
+
+  if (unix_process->stderr_handle.fd != -1) {
+    ret = close(unix_process->stderr_handle.fd);
+    if (ret == -1) {
+      log_warn(LD_PROCESS, "Unable to close standard error");
+      success = false;
+    }
+
+    unix_process->stderr_handle.fd = -1;
+  }
+
+  return success;
+}
+
+#endif /* !defined(_WIN32) */
diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h
new file mode 100644
index 0000000000..a6b8304d48
--- /dev/null
+++ b/src/lib/process/process_unix.h
@@ -0,0 +1,68 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_unix.h
+ * \brief Header for process_unix.c
+ **/
+
+#ifndef TOR_PROCESS_UNIX_H
+#define TOR_PROCESS_UNIX_H
+
+#ifndef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+
+#include <event2/event.h>
+
+struct process_t;
+
+struct process_unix_t;
+typedef struct process_unix_t process_unix_t;
+
+process_unix_t *process_unix_new(void);
+void process_unix_free_(process_unix_t *unix_process);
+#define process_unix_free(s) \
+  FREE_AND_NULL(process_unix_t, process_unix_free_, (s))
+
+process_status_t process_unix_exec(struct process_t *process);
+bool process_unix_terminate(struct process_t *process);
+
+process_pid_t process_unix_get_pid(struct process_t *process);
+
+int process_unix_write(struct process_t *process, buf_t *buffer);
+int process_unix_read_stdout(struct process_t *process, buf_t *buffer);
+int process_unix_read_stderr(struct process_t *process, buf_t *buffer);
+
+#ifdef PROCESS_UNIX_PRIVATE
+struct process_unix_handle_t;
+typedef struct process_unix_handle_t process_unix_handle_t;
+
+STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data);
+STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data);
+
+STATIC void process_unix_start_reading(process_unix_handle_t *);
+STATIC void process_unix_stop_reading(process_unix_handle_t *);
+
+STATIC void process_unix_start_writing(process_unix_handle_t *);
+STATIC void process_unix_stop_writing(process_unix_handle_t *);
+
+STATIC void process_unix_waitpid_callback(int status, void *data);
+
+STATIC void process_unix_setup_handle(process_t *process,
+                                      process_unix_handle_t *handle,
+                                      short flags,
+                                      event_callback_fn callback);
+STATIC int process_unix_read_handle(process_t *,
+                                    process_unix_handle_t *,
+                                    buf_t *);
+STATIC bool process_unix_close_file_descriptors(process_unix_t *);
+#endif /* defined(PROCESS_UNIX_PRIVATE) */
+
+#endif /* !defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_UNIX_H) */
diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c
new file mode 100644
index 0000000000..8683e17fc2
--- /dev/null
+++ b/src/lib/process/process_win32.c
@@ -0,0 +1,1105 @@
+/* Copyright (c) 2003, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.c
+ * \brief Module for working with Windows processes.
+ **/
+
+#define PROCESS_WIN32_PRIVATE
+#include "lib/intmath/cmp.h"
+#include "lib/buf/buffers.h"
+#include "lib/net/buffers_net.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/log.h"
+#include "lib/log/util_bug.h"
+#include "lib/log/win32err.h"
+#include "lib/process/process.h"
+#include "lib/process/process_win32.h"
+#include "lib/process/env.h"
+
+#ifdef HAVE_SYS_TIME_H
+#include <sys/time.h>
+#endif
+
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#ifdef _WIN32
+
+/** The size of our intermediate buffers. */
+#define BUFFER_SIZE (1024)
+
+/** Timer that ticks once a second and calls the process_win32_timer_callback()
+ * function. */
+static periodic_timer_t *periodic_timer;
+
+/** Structure to represent the state around the pipe HANDLE.
+ *
+ * This structure is used to store state about a given HANDLE, including
+ * whether we have reached end of file, its intermediate buffers, and how much
+ * data that is available in the intermediate buffer. */
+struct process_win32_handle_t {
+  /** Standard out pipe handle. */
+  HANDLE pipe;
+
+  /** True iff we have reached EOF from the pipe. */
+  bool reached_eof;
+
+  /** How much data is available in buffer. */
+  size_t data_available;
+
+  /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */
+  char buffer[BUFFER_SIZE];
+
+  /** Overlapped structure for ReadFileEx() and WriteFileEx(). */
+  OVERLAPPED overlapped;
+
+  /** Are we waiting for another I/O operation to complete? */
+  bool busy;
+};
+
+/** Structure to represent the Windows specific implementation details of this
+ * Process backend.
+ *
+ * This structure is attached to <b>process_t</b> (see process.h) and is
+ * reachable from <b>process_t</b> via the <b>process_get_win32_process()</b>
+ * method. */
+struct process_win32_t {
+  /** Standard in state. */
+  process_win32_handle_t stdin_handle;
+
+  /** Standard out state. */
+  process_win32_handle_t stdout_handle;
+
+  /** Standard error state. */
+  process_win32_handle_t stderr_handle;
+
+  /** Process Information. */
+  PROCESS_INFORMATION process_information;
+};
+
+/** Create a new <b>process_win32_t</b>.
+ *
+ * This function constructs a new <b>process_win32_t</b> and initializes the
+ * default values. */
+process_win32_t *
+process_win32_new(void)
+{
+  process_win32_t *win32_process;
+  win32_process = tor_malloc_zero(sizeof(process_win32_t));
+
+  win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE;
+  win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE;
+
+  return win32_process;
+}
+
+/** Free a given <b>process_win32_t</b>.
+ *
+ * This function deinitializes and frees up the resources allocated for the
+ * given <b>process_win32_t</b>. */
+void
+process_win32_free_(process_win32_t *win32_process)
+{
+  if (! win32_process)
+    return;
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  tor_free(win32_process);
+}
+
+/** Initialize the Windows backend of the Process subsystem. */
+void
+process_win32_init(void)
+{
+  /* We don't start the periodic timer here because it makes no sense to have
+   * the timer running until we have some processes that benefits from the
+   * timer timer ticks. */
+}
+
+/** Deinitialize the Windows backend of the Process subsystem. */
+void
+process_win32_deinit(void)
+{
+  /* Stop our timer, but only if it's running. */
+  if (process_win32_timer_running())
+    process_win32_timer_stop();
+}
+
+/** Execute the given process. This function is responsible for setting up
+ * named pipes for I/O between the child process and the Tor process. Returns
+ * <b>PROCESS_STATUS_RUNNING</b> upon success. */
+process_status_t
+process_win32_exec(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  HANDLE stdout_pipe_read = NULL;
+  HANDLE stdout_pipe_write = NULL;
+  HANDLE stderr_pipe_read = NULL;
+  HANDLE stderr_pipe_write = NULL;
+  HANDLE stdin_pipe_read = NULL;
+  HANDLE stdin_pipe_write = NULL;
+  BOOL ret = FALSE;
+
+  /* Setup our security attributes. */
+  SECURITY_ATTRIBUTES security_attributes;
+  security_attributes.nLength = sizeof(security_attributes);
+  security_attributes.bInheritHandle = TRUE;
+  /* FIXME: should we set explicit security attributes?
+   * (See Ticket #2046, comment 5) */
+  security_attributes.lpSecurityDescriptor = NULL;
+
+  /* Create our standard out pipe. */
+  if (! process_win32_create_pipe(&stdout_pipe_read,
+                                  &stdout_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create our standard error pipe. */
+  if (! process_win32_create_pipe(&stderr_pipe_read,
+                                  &stderr_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_READER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Create out standard in pipe. */
+  if (! process_win32_create_pipe(&stdin_pipe_read,
+                                  &stdin_pipe_write,
+                                  &security_attributes,
+                                  PROCESS_WIN32_PIPE_TYPE_WRITER)) {
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* Configure startup info for our child process. */
+  STARTUPINFOA startup_info;
+
+  memset(&startup_info, 0, sizeof(startup_info));
+  startup_info.cb = sizeof(startup_info);
+  startup_info.hStdError = stderr_pipe_write;
+  startup_info.hStdOutput = stdout_pipe_write;
+  startup_info.hStdInput = stdin_pipe_read;
+  startup_info.dwFlags |= STARTF_USESTDHANDLES;
+
+  /* Create the env value for our new process. */
+  process_environment_t *env = process_get_environment(process);
+
+  /* Create the argv value for our new process. */
+  char **argv = process_get_argv(process);
+
+  /* Windows expects argv to be a whitespace delimited string, so join argv up
+   */
+  char *joined_argv = tor_join_win_cmdline((const char **)argv);
+
+  /* Create the child process */
+  ret = CreateProcessA(NULL,
+                       joined_argv,
+                       NULL,
+                       NULL,
+                       TRUE,
+                       CREATE_NO_WINDOW,
+                       env->windows_environment_block[0] == '\0' ?
+                         NULL : env->windows_environment_block,
+                       NULL,
+                       &startup_info,
+                       &win32_process->process_information);
+
+  tor_free(argv);
+  tor_free(joined_argv);
+  process_environment_free(env);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "CreateProcessA() failed: %s",
+      format_win32_error(GetLastError()));
+
+    /* Cleanup our handles. */
+    CloseHandle(stdout_pipe_read);
+    CloseHandle(stdout_pipe_write);
+    CloseHandle(stderr_pipe_read);
+    CloseHandle(stderr_pipe_write);
+    CloseHandle(stdin_pipe_read);
+    CloseHandle(stdin_pipe_write);
+
+    /* In the Unix backend, we do not get an error in the Tor process when a
+     * child process fails to spawn its target executable since we need to
+     * first do the fork() call in the Tor process and then the child process
+     * is responsible for doing the call to execve().
+     *
+     * This means that the user of the process_exec() API must check for
+     * whether it returns PROCESS_STATUS_ERROR, which will rarely happen on
+     * Unix, but will happen for error cases on Windows where it does not
+     * happen on Unix. For example: when the target executable does not exist
+     * on the file system.
+     *
+     * To have somewhat feature compatibility between the Unix and the Windows
+     * backend, we here notify the process_t owner that the process have exited
+     * (even though it never managed to run) to ensure that the exit callback
+     * is executed.
+     */
+    process_notify_event_exit(process, 0);
+
+    return PROCESS_STATUS_ERROR;
+  }
+
+  /* TODO: Should we close hProcess and hThread in
+   * process_handle->process_information? */
+  win32_process->stdout_handle.pipe = stdout_pipe_read;
+  win32_process->stderr_handle.pipe = stderr_pipe_read;
+  win32_process->stdin_handle.pipe = stdin_pipe_write;
+
+  /* Close our ends of the pipes that is now owned by the child process. */
+  CloseHandle(stdout_pipe_write);
+  CloseHandle(stderr_pipe_write);
+  CloseHandle(stdin_pipe_read);
+
+  /* Used by the callback functions from ReadFileEx() and WriteFileEx() such
+   * that we can figure out which process_t that was responsible for the event.
+   *
+   * Warning, here be dragons:
+   *
+   *   MSDN says that the hEvent member of the overlapped structure is unused
+   *   for ReadFileEx() and WriteFileEx, which allows us to store a pointer to
+   *   our process state there.
+   */
+  win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process;
+  win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process;
+
+  /* Start our timer if it is not already running. */
+  if (! process_win32_timer_running())
+    process_win32_timer_start();
+
+  /* We use Windows Extended I/O functions, so our completion callbacks are
+   * called automatically for us when there is data to read. Because of this
+   * we start the read of standard out and error right away. */
+  process_notify_event_stdout(process);
+  process_notify_event_stderr(process);
+
+  return PROCESS_STATUS_RUNNING;
+}
+
+/** Terminate the given process. Returns true on success, otherwise false. */
+bool
+process_win32_terminate(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Terminate our process. */
+  BOOL ret;
+
+  ret = TerminateProcess(win32_process->process_information.hProcess, 0);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "TerminateProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Cleanup our handles. */
+  process_win32_cleanup_handle(&win32_process->stdin_handle);
+  process_win32_cleanup_handle(&win32_process->stdout_handle);
+  process_win32_cleanup_handle(&win32_process->stderr_handle);
+
+  return true;
+}
+
+/** Returns the unique process identifier for the given <b>process</b>. */
+process_pid_t
+process_win32_get_pid(process_t *process)
+{
+  tor_assert(process);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  return (process_pid_t)win32_process->process_information.dwProcessId;
+}
+
+/** Schedule an async write of the data found in <b>buffer</b> for the given
+ * process.  This function runs an async write operation of the content of
+ * buffer, if we are not already waiting for a pending I/O request. Returns the
+ * number of bytes that Windows will hopefully write for us in the background.
+ * */
+int
+process_win32_write(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD error_code = 0;
+  const size_t buffer_size = buf_datalen(buffer);
+
+  /* Windows is still writing our buffer. */
+  if (win32_process->stdin_handle.busy)
+    return 0;
+
+  /* Nothing for us to do right now. */
+  if (buffer_size == 0)
+    return 0;
+
+  /* We have reached end of file already? */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return 0;
+
+  /* Figure out how much data we should read. */
+  const size_t write_size = MIN(buffer_size,
+                                sizeof(win32_process->stdin_handle.buffer));
+
+  /* Read data from the process_t buffer into our intermediate buffer. */
+  buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size);
+
+  /* Because of the slightly weird API for WriteFileEx() we must set this to 0
+   * before we call WriteFileEx() because WriteFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Schedule our write. */
+  ret = WriteFileEx(win32_process->stdin_handle.pipe,
+                    win32_process->stdin_handle.buffer,
+                    write_size,
+                    &win32_process->stdin_handle.overlapped,
+                    process_win32_stdin_write_done);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "WriteFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "WriteFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for WriteFileEx() we
+   * should check GetLastError() after a call to WriteFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "WriteFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+    return 0;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* This cast should be safe since our buffer can maximum be BUFFER_SIZE
+   * large. */
+  return (int)write_size;
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard output of the child process
+ * and asks Windows to call process_win32_stdout_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stdout(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stdout_handle,
+                                        buffer,
+                                        process_win32_stdout_read_done);
+}
+
+/** This function is called from the Process subsystem whenever the Windows
+ * backend says it has data ready. This function also ensures that we are
+ * starting a new background read from the standard error of the child process
+ * and asks Windows to call process_win32_stderr_read_done() when that
+ * operation is finished. Returns the number of bytes moved into <b>buffer</b>.
+ * */
+int
+process_win32_read_stderr(struct process_t *process, buf_t *buffer)
+{
+  tor_assert(process);
+  tor_assert(buffer);
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  return process_win32_read_from_handle(&win32_process->stderr_handle,
+                                        buffer,
+                                        process_win32_stderr_read_done);
+}
+
+/** This function is responsible for moving the Tor process into what Microsoft
+ * calls an "alertable" state. Once the process is in an alertable state the
+ * Windows kernel will notify us when our background I/O requests have finished
+ * and the callbacks will be executed. */
+void
+process_win32_trigger_completion_callbacks(void)
+{
+  DWORD ret;
+
+  /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep
+   * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause
+   * the process to enter alertable state, where the Windows kernel will notify
+   * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which
+   * will cause our completion callbacks to be executed.
+   *
+   * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION
+   * if one or more I/O callbacks were executed. */
+  ret = SleepEx(0, TRUE);
+
+  /* Warn us if the function returned something we did not anticipate. */
+  if (ret != 0 && ret != WAIT_IO_COMPLETION) {
+    log_warn(LD_PROCESS, "SleepEx() returned %lu", ret);
+  }
+}
+
+/** Start the periodic timer which is reponsible for checking whether processes
+ * are still alive and to make sure that the Tor process is periodically being
+ * moved into an alertable state. */
+void
+process_win32_timer_start(void)
+{
+  /* Make sure we never start our timer if it's already running. */
+  if (BUG(process_win32_timer_running()))
+    return;
+
+  /* Wake up once a second. */
+  static const struct timeval interval = {1, 0};
+
+  log_info(LD_PROCESS, "Starting Windows Process I/O timer");
+  periodic_timer = periodic_timer_new(tor_libevent_get_base(),
+                                      &interval,
+                                      process_win32_timer_callback,
+                                      NULL);
+}
+
+/** Stops the periodic timer. */
+void
+process_win32_timer_stop(void)
+{
+  if (BUG(periodic_timer == NULL))
+    return;
+
+  log_info(LD_PROCESS, "Stopping Windows Process I/O timer");
+  periodic_timer_free(periodic_timer);
+}
+
+/** Returns true iff the periodic timer is running. */
+bool
+process_win32_timer_running(void)
+{
+  return periodic_timer != NULL;
+}
+
+/** This function is called whenever the periodic_timer ticks. The function is
+ * responsible for moving the Tor process into an alertable state once a second
+ * and checking for whether our child processes have terminated since the last
+ * tick. */
+STATIC void
+process_win32_timer_callback(periodic_timer_t *timer, void *data)
+{
+  tor_assert(timer == periodic_timer);
+  tor_assert(data == NULL);
+
+  /* Move the process into an alertable state. */
+  process_win32_trigger_completion_callbacks();
+
+  /* Check if our processes are still alive. */
+
+  /* Since the call to process_win32_timer_test_process() might call
+   * process_notify_event_exit() which again might call process_free() which
+   * updates the list of processes returned by process_get_all_processes() it
+   * is important here that we make sure to not touch the list of processes if
+   * the call to process_win32_timer_test_process() returns true. */
+  bool done;
+
+  do {
+    const smartlist_t *processes = process_get_all_processes();
+    done = true;
+
+    SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) {
+      /* If process_win32_timer_test_process() returns true, it means that
+       * smartlist_remove() might have been called on the list returned by
+       * process_get_all_processes(). We start the loop over again until we
+       * have a succesful run over the entire list where the list was not
+       * modified. */
+      if (process_win32_timer_test_process(process)) {
+        done = false;
+        break;
+      }
+    } SMARTLIST_FOREACH_END(process);
+  } while (! done);
+}
+
+/** Test whether a given process is still alive. Notify the Process subsystem
+ * if our process have died. Returns true iff the given process have
+ * terminated. */
+STATIC bool
+process_win32_timer_test_process(process_t *process)
+{
+  tor_assert(process);
+
+  /* No need to look at processes that don't claim they are running. */
+  if (process_get_status(process) != PROCESS_STATUS_RUNNING)
+    return false;
+
+  process_win32_t *win32_process = process_get_win32_process(process);
+  BOOL ret = FALSE;
+  DWORD exit_code = 0;
+
+  /* Sometimes the Windows kernel wont give us the EOF/Broken Pipe error
+   * message until some time after the process have actually terminated. We
+   * make sure that our ReadFileEx() calls for the process have *all* returned
+   * and both standard out and error have been marked as EOF before we try to
+   * see if the process terminated.
+   *
+   * This ensures that we *never* call the exit callback of the `process_t`,
+   * which potentially ends up calling `process_free()` on our `process_t`,
+   * before all data have been received from the process.
+   *
+   * We do NOT have a check here for whether standard in reached EOF since
+   * standard in's WriteFileEx() function is only called on-demand when we have
+   * something to write and is thus usually not awaiting to finish any
+   * operations. If we WriteFileEx() to a file that has terminated we'll simply
+   * get an error from ReadFileEx() or its completion routine and move on with
+   * life.  */
+  if (! win32_process->stdout_handle.reached_eof)
+    return false;
+
+  if (! win32_process->stderr_handle.reached_eof)
+    return false;
+
+  /* We start by testing whether our process is still running. */
+  ret = GetExitCodeProcess(win32_process->process_information.hProcess,
+                           &exit_code);
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Notify our process_t that our process have terminated. Since our
+   * exit_callback might decide to process_free() our process handle it is very
+   * important that we do not touch the process_t after the call to
+   * process_notify_event_exit(). */
+  if (exit_code != STILL_ACTIVE) {
+    process_notify_event_exit(process, exit_code);
+    return true;
+  }
+
+  return false;
+}
+
+/** Create a new overlapped named pipe. This function creates a new connected,
+ * named, pipe in <b>*read_pipe</b> and <b>*write_pipe</b> if the function is
+ * succesful. Returns true on sucess, false on failure. */
+STATIC bool
+process_win32_create_pipe(HANDLE *read_pipe,
+                          HANDLE *write_pipe,
+                          SECURITY_ATTRIBUTES *attributes,
+                          process_win32_pipe_type_t pipe_type)
+{
+  tor_assert(read_pipe);
+  tor_assert(write_pipe);
+  tor_assert(attributes);
+
+  BOOL ret = FALSE;
+
+  /* Buffer size. */
+  const size_t size = 4096;
+
+  /* Our additional read/write modes that depends on which pipe type we are
+   * creating. */
+  DWORD read_mode = 0;
+  DWORD write_mode = 0;
+
+  /* Generate the unique pipe name. */
+  char pipe_name[MAX_PATH];
+  static DWORD process_id = 0;
+  static DWORD counter = 0;
+
+  if (process_id == 0)
+    process_id = GetCurrentProcessId();
+
+  tor_snprintf(pipe_name, sizeof(pipe_name),
+               "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu",
+               process_id, counter++);
+
+  /* Only one of our handles can be overlapped. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    read_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    write_mode = FILE_FLAG_OVERLAPPED;
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* Setup our read and write handles. */
+  HANDLE read_handle;
+  HANDLE write_handle;
+
+  /* Create our named pipe. */
+  read_handle = CreateNamedPipeA(pipe_name,
+                                 (PIPE_ACCESS_INBOUND|read_mode),
+                                 (PIPE_TYPE_BYTE|PIPE_WAIT),
+                                 1,
+                                 size,
+                                 size,
+                                 1000,
+                                 attributes);
+
+  if (read_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s",
+             format_win32_error(GetLastError()));
+    return false;
+  }
+
+  /* Create our file in the pipe namespace. */
+  write_handle = CreateFileA(pipe_name,
+                             GENERIC_WRITE,
+                             0,
+                             attributes,
+                             OPEN_EXISTING,
+                             (FILE_ATTRIBUTE_NORMAL|write_mode),
+                             NULL);
+
+  if (write_handle == INVALID_HANDLE_VALUE) {
+    log_warn(LD_PROCESS, "CreateFileA() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+
+    return false;
+  }
+
+  /* Set the inherit flag for our pipe. */
+  switch (pipe_type) {
+  case PROCESS_WIN32_PIPE_TYPE_READER:
+    ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  case PROCESS_WIN32_PIPE_TYPE_WRITER:
+    ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0);
+    break;
+  default:
+    /* LCOV_EXCL_START */
+    tor_assert_nonfatal_unreached_once();
+    /* LCOV_EXCL_STOP */
+  }
+
+  if (! ret) {
+    log_warn(LD_PROCESS, "SetHandleInformation() failed: %s",
+             format_win32_error(GetLastError()));
+
+    CloseHandle(read_handle);
+    CloseHandle(write_handle);
+
+    return false;
+  }
+
+  /* Everything is good. */
+  *read_pipe = read_handle;
+  *write_pipe = write_handle;
+
+  return true;
+}
+
+/** Cleanup a given <b>handle</b>. */
+STATIC void
+process_win32_cleanup_handle(process_win32_handle_t *handle)
+{
+  tor_assert(handle);
+
+#if 0
+  BOOL ret;
+  DWORD error_code;
+
+  /* Cancel any pending I/O requests: This means that instead of getting
+   * ERROR_BROKEN_PIPE we get ERROR_OPERATION_ABORTED, but it doesn't seem
+   * like this is needed. */
+  ret = CancelIo(handle->pipe);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* There was no pending I/O requests for our handle. */
+    if (error_code != ERROR_NOT_FOUND) {
+      log_warn(LD_PROCESS, "CancelIo() failed: %s",
+               format_win32_error(error_code));
+    }
+  }
+#endif /* 0 */
+
+  /* Close our handle. */
+  if (handle->pipe != INVALID_HANDLE_VALUE) {
+    CloseHandle(handle->pipe);
+    handle->pipe = INVALID_HANDLE_VALUE;
+    handle->reached_eof = true;
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard output. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stdout_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stdout_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stdout(process);
+  }
+}
+
+/** This function is called when ReadFileEx() completes its background read
+ * from the child process's standard error. We notify the Process subsystem if
+ * there is data available for it to read from us. */
+STATIC VOID WINAPI
+process_win32_stderr_read_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  /* Extract our process_t from the hEvent member of OVERLAPPED. */
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  if (process_win32_handle_read_completion(&win32_process->stderr_handle,
+                                           error_code,
+                                           byte_count)) {
+    /* Schedule our next read. */
+    process_notify_event_stderr(process);
+  }
+}
+
+/** This function is called when WriteFileEx() completes its background write
+ * to the child process's standard input. We notify the Process subsystem that
+ * it can write data to us again. */
+STATIC VOID WINAPI
+process_win32_stdin_write_done(DWORD error_code,
+                               DWORD byte_count,
+                               LPOVERLAPPED overlapped)
+{
+  tor_assert(overlapped);
+  tor_assert(overlapped->hEvent);
+
+  (void)byte_count;
+
+  process_t *process = (process_t *)overlapped->hEvent;
+  process_win32_t *win32_process = process_get_win32_process(process);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  win32_process->stdin_handle.busy = false;
+
+  /* Check if we have been asked to write to the handle that have been marked
+   * as having reached EOF. */
+  if (BUG(win32_process->stdin_handle.reached_eof))
+    return;
+
+  if (error_code == 0) {
+    /** Our data have been succesfully written. Clear our state and schedule
+     * the next write. */
+    win32_process->stdin_handle.data_available = 0;
+    memset(win32_process->stdin_handle.buffer, 0,
+           sizeof(win32_process->stdin_handle.buffer));
+
+    /* Schedule the next write. */
+    process_notify_event_stdin(process);
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our WriteFileEx() call was succesful, but we reached the end of our
+     * file.  We mark our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    win32_process->stdin_handle.reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from WriteFileEx(): %s",
+             format_win32_error(error_code));
+    win32_process->stdin_handle.reached_eof = true;
+  }
+}
+
+/** This function reads data from the given <b>handle</b>'s internal buffer and
+ * moves it into the given <b>buffer</b>. Additionally, we start the next
+ * ReadFileEx() background operation with the given <b>callback</b> as
+ * completion callback. Returns the number of bytes written to the buffer. */
+STATIC int
+process_win32_read_from_handle(process_win32_handle_t *handle,
+                               buf_t *buffer,
+                               LPOVERLAPPED_COMPLETION_ROUTINE callback)
+{
+  tor_assert(handle);
+  tor_assert(buffer);
+  tor_assert(callback);
+
+  BOOL ret = FALSE;
+  int bytes_available = 0;
+  DWORD error_code = 0;
+
+  /* We already have a request to read data that isn't complete yet. */
+  if (BUG(handle->busy))
+    return 0;
+
+  /* Check if we have been asked to read from a handle that have already told
+   * us that we have reached the end of the file. */
+  if (BUG(handle->reached_eof))
+    return 0;
+
+  /* This cast should be safe since our buffer can be at maximum up to
+   * BUFFER_SIZE in size. */
+  bytes_available = (int)handle->data_available;
+
+  if (handle->data_available > 0) {
+    /* Read data from our intermediate buffer into the process_t buffer. */
+    buf_add(buffer, handle->buffer, handle->data_available);
+
+    /* Reset our read state. */
+    handle->data_available = 0;
+    memset(handle->buffer, 0, sizeof(handle->buffer));
+  }
+
+  /* Because of the slightly weird API for ReadFileEx() we must set this to 0
+   * before we call ReadFileEx() because ReadFileEx() does not reset the last
+   * error itself when it's succesful. See comment below after the call to
+   * GetLastError(). */
+  SetLastError(0);
+
+  /* Ask the Windows kernel to read data from our pipe into our buffer and call
+   * the callback function when it is done. */
+  ret = ReadFileEx(handle->pipe,
+                   handle->buffer,
+                   sizeof(handle->buffer),
+                   &handle->overlapped,
+                   callback);
+
+  if (! ret) {
+    error_code = GetLastError();
+
+    /* No need to log at warning level for these two. */
+    if (error_code == ERROR_HANDLE_EOF || error_code == ERROR_BROKEN_PIPE) {
+      log_debug(LD_PROCESS, "ReadFileEx() returned EOF from pipe: %s",
+                format_win32_error(error_code));
+    } else {
+      log_warn(LD_PROCESS, "ReadFileEx() failed: %s",
+               format_win32_error(error_code));
+    }
+
+    handle->reached_eof = true;
+    return bytes_available;
+  }
+
+  /* Here be dragons: According to MSDN's documentation for ReadFileEx() we
+   * should check GetLastError() after a call to ReadFileEx() even though the
+   * `ret` return value was successful. If everything is good, GetLastError()
+   * returns `ERROR_SUCCESS` and nothing happens.
+   *
+   * XXX(ahf): I have not managed to trigger this code while stress-testing
+   * this code. */
+  error_code = GetLastError();
+
+  if (error_code != ERROR_SUCCESS) {
+    /* LCOV_EXCL_START */
+    log_warn(LD_PROCESS, "ReadFileEx() failed after returning success: %s",
+             format_win32_error(error_code));
+    handle->reached_eof = true;
+    return bytes_available;
+    /* LCOV_EXCL_STOP */
+  }
+
+  /* We mark our handle as having a pending I/O request. */
+  handle->busy = true;
+
+  return bytes_available;
+}
+
+/** This function checks the callback values from ReadFileEx() in
+ * <b>error_code</b> and <b>byte_count</b> if we have read data. Returns true
+ * iff our caller should request more data from ReadFileEx(). */
+STATIC bool
+process_win32_handle_read_completion(process_win32_handle_t *handle,
+                                     DWORD error_code,
+                                     DWORD byte_count)
+{
+  tor_assert(handle);
+
+  /* Mark our handle as not having any outstanding I/O requests. */
+  handle->busy = false;
+
+  if (error_code == 0) {
+    /* Our ReadFileEx() call was succesful and there is data for us. */
+
+    /* This cast should be safe since byte_count should never be larger than
+     * BUFFER_SIZE. */
+    tor_assert(byte_count <= BUFFER_SIZE);
+    handle->data_available = (size_t)byte_count;
+
+    /* Tell our caller to schedule the next read. */
+    return true;
+  } else if (error_code == ERROR_HANDLE_EOF ||
+             error_code == ERROR_BROKEN_PIPE) {
+    /* Our ReadFileEx() finished, but we reached the end of our file.  We mark
+     * our handle as having reached EOF and returns. */
+    tor_assert(byte_count == 0);
+
+    handle->reached_eof = true;
+  } else {
+    /* An error happened: We warn the user and mark our handle as having
+     * reached EOF */
+    log_warn(LD_PROCESS,
+             "Error in I/O completion routine from ReadFileEx(): %s",
+             format_win32_error(error_code));
+
+    handle->reached_eof = true;
+  }
+
+  /* Our caller should NOT schedule the next read. */
+  return false;
+}
+
+/** Format a single argument for being put on a Windows command line.
+ * Returns a newly allocated string */
+STATIC char *
+format_win_cmdline_argument(const char *arg)
+{
+  char *formatted_arg;
+  char need_quotes;
+  const char *c;
+  int i;
+  int bs_counter = 0;
+  /* Backslash we can point to when one is inserted into the string */
+  const char backslash = '\\';
+
+  /* Smartlist of *char */
+  smartlist_t *arg_chars;
+  arg_chars = smartlist_new();
+
+  /* Quote string if it contains whitespace or is empty */
+  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
+
+  /* Build up smartlist of *chars */
+  for (c=arg; *c != '\0'; c++) {
+    if ('"' == *c) {
+      /* Double up backslashes preceding a quote */
+      for (i=0; i<(bs_counter*2); i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      /* Escape the quote */
+      smartlist_add(arg_chars, (void*)&backslash);
+      smartlist_add(arg_chars, (void*)c);
+    } else if ('\\' == *c) {
+      /* Count backslashes until we know whether to double up */
+      bs_counter++;
+    } else {
+      /* Don't double up slashes preceding a non-quote */
+      for (i=0; i<bs_counter; i++)
+        smartlist_add(arg_chars, (void*)&backslash);
+      bs_counter = 0;
+      smartlist_add(arg_chars, (void*)c);
+    }
+  }
+  /* Don't double up trailing backslashes */
+  for (i=0; i<bs_counter; i++)
+    smartlist_add(arg_chars, (void*)&backslash);
+
+  /* Allocate space for argument, quotes (if needed), and terminator */
+  const size_t formatted_arg_len = smartlist_len(arg_chars) +
+    (need_quotes ? 2 : 0) + 1;
+  formatted_arg = tor_malloc_zero(formatted_arg_len);
+
+  /* Add leading quote */
+  i=0;
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+
+  /* Add characters */
+  SMARTLIST_FOREACH(arg_chars, char*, ch,
+  {
+    formatted_arg[i++] = *ch;
+  });
+
+  /* Add trailing quote */
+  if (need_quotes)
+    formatted_arg[i++] = '"';
+  formatted_arg[i] = '\0';
+
+  smartlist_free(arg_chars);
+  return formatted_arg;
+}
+
+/** Format a command line for use on Windows, which takes the command as a
+ * string rather than string array. Follows the rules from "Parsing C++
+ * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
+ * Python subprocess module. Returns a newly allocated string */
+STATIC char *
+tor_join_win_cmdline(const char *argv[])
+{
+  smartlist_t *argv_list;
+  char *joined_argv;
+  int i;
+
+  /* Format each argument and put the result in a smartlist */
+  argv_list = smartlist_new();
+  for (i=0; argv[i] != NULL; i++) {
+    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
+  }
+
+  /* Join the arguments with whitespace */
+  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
+
+  /* Free the newly allocated arguments, and the smartlist */
+  SMARTLIST_FOREACH(argv_list, char *, arg,
+  {
+    tor_free(arg);
+  });
+  smartlist_free(argv_list);
+
+  return joined_argv;
+}
+
+#endif /* defined(_WIN32) */
diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h
new file mode 100644
index 0000000000..0f264c8710
--- /dev/null
+++ b/src/lib/process/process_win32.h
@@ -0,0 +1,97 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file process_win32.h
+ * \brief Header for process_win32.c
+ **/
+
+#ifndef TOR_PROCESS_WIN32_H
+#define TOR_PROCESS_WIN32_H
+
+#ifdef _WIN32
+
+#include "orconfig.h"
+#include "lib/malloc/malloc.h"
+#include "lib/evloop/compat_libevent.h"
+
+#include <windows.h>
+
+struct process_t;
+
+struct process_win32_t;
+typedef struct process_win32_t process_win32_t;
+
+process_win32_t *process_win32_new(void);
+void process_win32_free_(process_win32_t *win32_process);
+#define process_win32_free(s) \
+  FREE_AND_NULL(process_win32_t, process_win32_free_, (s))
+
+void process_win32_init(void);
+void process_win32_deinit(void);
+
+process_status_t process_win32_exec(struct process_t *process);
+bool process_win32_terminate(struct process_t *process);
+
+process_pid_t process_win32_get_pid(struct process_t *process);
+
+int process_win32_write(struct process_t *process, buf_t *buffer);
+int process_win32_read_stdout(struct process_t *process, buf_t *buffer);
+int process_win32_read_stderr(struct process_t *process, buf_t *buffer);
+
+void process_win32_trigger_completion_callbacks(void);
+
+/* Timer handling. */
+void process_win32_timer_start(void);
+void process_win32_timer_stop(void);
+bool process_win32_timer_running(void);
+
+#ifdef PROCESS_WIN32_PRIVATE
+STATIC void process_win32_timer_callback(periodic_timer_t *, void *);
+STATIC bool process_win32_timer_test_process(process_t *);
+
+/* I/O pipe handling. */
+struct process_win32_handle_t;
+typedef struct process_win32_handle_t process_win32_handle_t;
+
+typedef enum process_win32_pipe_type_t {
+  /** This pipe is used for reading. */
+  PROCESS_WIN32_PIPE_TYPE_READER,
+
+  /** This pipe is used for writing. */
+  PROCESS_WIN32_PIPE_TYPE_WRITER
+} process_win32_pipe_type_t;
+
+STATIC bool process_win32_create_pipe(HANDLE *,
+                                      HANDLE *,
+                                      SECURITY_ATTRIBUTES *,
+                                      process_win32_pipe_type_t);
+
+STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle);
+
+STATIC VOID WINAPI process_win32_stdout_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stderr_read_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+STATIC VOID WINAPI process_win32_stdin_write_done(DWORD,
+                                                  DWORD,
+                                                  LPOVERLAPPED);
+
+STATIC int process_win32_read_from_handle(process_win32_handle_t *,
+                                          buf_t *,
+                                          LPOVERLAPPED_COMPLETION_ROUTINE);
+STATIC bool process_win32_handle_read_completion(process_win32_handle_t *,
+                                                 DWORD,
+                                                 DWORD);
+
+STATIC char *format_win_cmdline_argument(const char *arg);
+STATIC char *tor_join_win_cmdline(const char *argv[]);
+#endif /* defined(PROCESS_WIN32_PRIVATE) */
+
+#endif /* defined(_WIN32) */
+
+#endif /* !defined(TOR_PROCESS_WIN32_H) */
diff --git a/src/lib/process/restrict.c b/src/lib/process/restrict.c
index 534b39d101..cd2a1c57b5 100644
--- a/src/lib/process/restrict.c
+++ b/src/lib/process/restrict.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -85,7 +85,7 @@ tor_set_max_memlock(void)
 {
   /* Future consideration for Windows is probably SetProcessWorkingSetSize
    * This is similar to setting the memory rlimit of RLIMIT_MEMLOCK
-   * http://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx
+   * https://msdn.microsoft.com/en-us/library/ms686234(VS.85).aspx
    */
 
   struct rlimit limit;
@@ -128,7 +128,7 @@ tor_mlockall(void)
    * Future consideration for Windows may be VirtualLock
    * VirtualLock appears to implement mlock() but not mlockall()
    *
-   * http://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
+   * https://msdn.microsoft.com/en-us/library/aa366895(VS.85).aspx
    */
 
 #ifdef HAVE_UNIX_MLOCKALL
@@ -152,7 +152,7 @@ tor_mlockall(void)
                            "pages: %s", strerror(errno));
     return -1;
   }
-#else /* !(defined(HAVE_UNIX_MLOCKALL)) */
+#else /* !defined(HAVE_UNIX_MLOCKALL) */
   log_warn(LD_GENERAL, "Unable to lock memory pages. mlockall() unsupported?");
   return -1;
 #endif /* defined(HAVE_UNIX_MLOCKALL) */
@@ -190,7 +190,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
 
   /* Define some maximum connections values for systems where we cannot
    * automatically determine a limit. Re Cygwin, see
-   * http://archives.seul.org/or/talk/Aug-2006/msg00210.html
+   * https://archives.seul.org/or/talk/Aug-2006/msg00210.html
    * For an iPhone, 9999 should work. For Windows and all other unknown
    * systems we use 15000 as the default. */
 #ifndef HAVE_GETRLIMIT
@@ -214,7 +214,7 @@ set_max_file_descriptors(rlim_t limit, int *max_out)
     return -1;
   }
   limit = MAX_CONNECTIONS;
-#else /* !(!defined(HAVE_GETRLIMIT)) */
+#else /* defined(HAVE_GETRLIMIT) */
   struct rlimit rlim;
 
   if (getrlimit(RLIMIT_NOFILE, &rlim) != 0) {
diff --git a/src/lib/process/restrict.h b/src/lib/process/restrict.h
index 8491c99044..6173d5a438 100644
--- a/src/lib/process/restrict.h
+++ b/src/lib/process/restrict.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/process/setuid.c b/src/lib/process/setuid.c
index 6e8258f279..3cfd520a4f 100644
--- a/src/lib/process/setuid.c
+++ b/src/lib/process/setuid.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -64,7 +64,7 @@ log_credential_status(void)
 
   /* log UIDs */
 #ifdef HAVE_GETRESUID
-  if (getresuid(&ruid, &euid, &suid) != 0 ) {
+  if (getresuid(&ruid, &euid, &suid) != 0) {
     log_warn(LD_GENERAL, "Error getting changed UIDs: %s", strerror(errno));
     return -1;
   } else {
@@ -72,7 +72,7 @@ log_credential_status(void)
            "UID is %u (real), %u (effective), %u (saved)",
            (unsigned)ruid, (unsigned)euid, (unsigned)suid);
   }
-#else /* !(defined(HAVE_GETRESUID)) */
+#else /* !defined(HAVE_GETRESUID) */
   /* getresuid is not present on MacOS X, so we can't get the saved (E)UID */
   ruid = getuid();
   euid = geteuid();
@@ -85,7 +85,7 @@ log_credential_status(void)
 
   /* log GIDs */
 #ifdef HAVE_GETRESGID
-  if (getresgid(&rgid, &egid, &sgid) != 0 ) {
+  if (getresgid(&rgid, &egid, &sgid) != 0) {
     log_warn(LD_GENERAL, "Error getting changed GIDs: %s", strerror(errno));
     return -1;
   } else {
@@ -93,7 +93,7 @@ log_credential_status(void)
            "GID is %u (real), %u (effective), %u (saved)",
            (unsigned)rgid, (unsigned)egid, (unsigned)sgid);
   }
-#else /* !(defined(HAVE_GETRESGID)) */
+#else /* !defined(HAVE_GETRESGID) */
   /* getresgid is not present on MacOS X, so we can't get the saved (E)GID */
   rgid = getgid();
   egid = getegid();
@@ -154,7 +154,7 @@ have_capability_support(void)
     return 0;
   cap_free(caps);
   return 1;
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   return 0;
 #endif /* defined(HAVE_LINUX_CAPABILITIES) */
 }
@@ -265,7 +265,7 @@ switch_id(const char *user, const unsigned flags)
     if (drop_capabilities(1))
       return -1;
   }
-#else /* !(defined(HAVE_LINUX_CAPABILITIES)) */
+#else /* !defined(HAVE_LINUX_CAPABILITIES) */
   (void) keep_bindlow;
   if (warn_if_no_caps) {
     log_warn(LD_CONFIG, "KeepBindCapabilities set, but no capability support "
@@ -376,7 +376,7 @@ switch_id(const char *user, const unsigned flags)
 #endif /* defined(__linux__) && defined(HAVE_SYS_PRCTL_H) && ... */
   return 0;
 
-#else /* !(!defined(_WIN32)) */
+#else /* defined(_WIN32) */
   (void)user;
   (void)flags;
 
diff --git a/src/lib/process/setuid.h b/src/lib/process/setuid.h
index 7d03e1f025..fec35a1216 100644
--- a/src/lib/process/setuid.h
+++ b/src/lib/process/setuid.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ int have_capability_support(void);
 #define SWITCH_ID_WARN_IF_NO_CAPS (1<<1)
 int switch_id(const char *user, unsigned flags);
 
-#endif
+#endif /* !defined(TOR_SETUID_H) */
diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c
deleted file mode 100644
index f4429d7f76..0000000000
--- a/src/lib/process/subprocess.c
+++ /dev/null
@@ -1,1236 +0,0 @@
-/* Copyright (c) 2003, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.c
- * \brief Launch and monitor other processes.
- **/
-
-#define SUBPROCESS_PRIVATE
-#include "lib/process/subprocess.h"
-
-#include "lib/container/smartlist.h"
-#include "lib/err/torerr.h"
-#include "lib/log/log.h"
-#include "lib/log/util_bug.h"
-#include "lib/log/win32err.h"
-#include "lib/malloc/malloc.h"
-#include "lib/process/env.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/compat_ctype.h"
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
-#ifdef HAVE_SYS_PRCTL_H
-#include <sys/prctl.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-#ifdef HAVE_SIGNAL_H
-#include <signal.h>
-#endif
-#ifdef HAVE_FCNTL_H
-#include <fcntl.h>
-#endif
-#ifdef HAVE_SYS_WAIT_H
-#include <sys/wait.h>
-#endif
-#include <errno.h>
-#include <string.h>
-
-/** Format a single argument for being put on a Windows command line.
- * Returns a newly allocated string */
-static char *
-format_win_cmdline_argument(const char *arg)
-{
-  char *formatted_arg;
-  char need_quotes;
-  const char *c;
-  int i;
-  int bs_counter = 0;
-  /* Backslash we can point to when one is inserted into the string */
-  const char backslash = '\\';
-
-  /* Smartlist of *char */
-  smartlist_t *arg_chars;
-  arg_chars = smartlist_new();
-
-  /* Quote string if it contains whitespace or is empty */
-  need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]);
-
-  /* Build up smartlist of *chars */
-  for (c=arg; *c != '\0'; c++) {
-    if ('"' == *c) {
-      /* Double up backslashes preceding a quote */
-      for (i=0; i<(bs_counter*2); i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      /* Escape the quote */
-      smartlist_add(arg_chars, (void*)&backslash);
-      smartlist_add(arg_chars, (void*)c);
-    } else if ('\\' == *c) {
-      /* Count backslashes until we know whether to double up */
-      bs_counter++;
-    } else {
-      /* Don't double up slashes preceding a non-quote */
-      for (i=0; i<bs_counter; i++)
-        smartlist_add(arg_chars, (void*)&backslash);
-      bs_counter = 0;
-      smartlist_add(arg_chars, (void*)c);
-    }
-  }
-  /* Don't double up trailing backslashes */
-  for (i=0; i<bs_counter; i++)
-    smartlist_add(arg_chars, (void*)&backslash);
-
-  /* Allocate space for argument, quotes (if needed), and terminator */
-  const size_t formatted_arg_len = smartlist_len(arg_chars) +
-    (need_quotes ? 2 : 0) + 1;
-  formatted_arg = tor_malloc_zero(formatted_arg_len);
-
-  /* Add leading quote */
-  i=0;
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-
-  /* Add characters */
-  SMARTLIST_FOREACH(arg_chars, char*, ch,
-  {
-    formatted_arg[i++] = *ch;
-  });
-
-  /* Add trailing quote */
-  if (need_quotes)
-    formatted_arg[i++] = '"';
-  formatted_arg[i] = '\0';
-
-  smartlist_free(arg_chars);
-  return formatted_arg;
-}
-
-/** Format a command line for use on Windows, which takes the command as a
- * string rather than string array. Follows the rules from "Parsing C++
- * Command-Line Arguments" in MSDN. Algorithm based on list2cmdline in the
- * Python subprocess module. Returns a newly allocated string */
-char *
-tor_join_win_cmdline(const char *argv[])
-{
-  smartlist_t *argv_list;
-  char *joined_argv;
-  int i;
-
-  /* Format each argument and put the result in a smartlist */
-  argv_list = smartlist_new();
-  for (i=0; argv[i] != NULL; i++) {
-    smartlist_add(argv_list, (void *)format_win_cmdline_argument(argv[i]));
-  }
-
-  /* Join the arguments with whitespace */
-  joined_argv = smartlist_join_strings(argv_list, " ", 0, NULL);
-
-  /* Free the newly allocated arguments, and the smartlist */
-  SMARTLIST_FOREACH(argv_list, char *, arg,
-  {
-    tor_free(arg);
-  });
-  smartlist_free(argv_list);
-
-  return joined_argv;
-}
-
-#ifndef _WIN32
-/** Format <b>child_state</b> and <b>saved_errno</b> as a hex string placed in
- * <b>hex_errno</b>.  Called between fork and _exit, so must be signal-handler
- * safe.
- *
- * <b>hex_errno</b> must have at least HEX_ERRNO_SIZE+1 bytes available.
- *
- * The format of <b>hex_errno</b> is: "CHILD_STATE/ERRNO\n", left-padded
- * with spaces. CHILD_STATE indicates where
- * in the process of starting the child process did the failure occur (see
- * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of
- * errno when the failure occurred.
- *
- * On success return the number of characters added to hex_errno, not counting
- * the terminating NUL; return -1 on error.
- */
-STATIC int
-format_helper_exit_status(unsigned char child_state, int saved_errno,
-                          char *hex_errno)
-{
-  unsigned int unsigned_errno;
-  int written, left;
-  char *cur;
-  size_t i;
-  int res = -1;
-
-  /* Fill hex_errno with spaces, and a trailing newline (memset may
-     not be signal handler safe, so we can't use it) */
-  for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++)
-    hex_errno[i] = ' ';
-  hex_errno[HEX_ERRNO_SIZE - 1] = '\n';
-
-  /* Convert errno to be unsigned for hex conversion */
-  if (saved_errno < 0) {
-    // Avoid overflow on the cast to unsigned int when result is INT_MIN
-    // by adding 1 to the signed int negative value,
-    // then, after it has been negated and cast to unsigned,
-    // adding the original 1 back (the double-addition is intentional).
-    // Otherwise, the cast to signed could cause a temporary int
-    // to equal INT_MAX + 1, which is undefined.
-    unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1;
-  } else {
-    unsigned_errno = (unsigned int) saved_errno;
-  }
-
-  /*
-   * Count how many chars of space we have left, and keep a pointer into the
-   * current point in the buffer.
-   */
-  left = HEX_ERRNO_SIZE+1;
-  cur = hex_errno;
-
-  /* Emit child_state */
-  written = format_hex_number_sigsafe(child_state, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-  if (left <= 0)
-    goto err;
-
-  /* Now the '/' */
-  *cur = '/';
-
-  /* Adjust left and cur */
-  ++cur;
-  --left;
-  if (left <= 0)
-    goto err;
-
-  /* Need minus? */
-  if (saved_errno < 0) {
-    *cur = '-';
-    ++cur;
-    --left;
-    if (left <= 0)
-      goto err;
-  }
-
-  /* Emit unsigned_errno */
-  written = format_hex_number_sigsafe(unsigned_errno, cur, left);
-
-  if (written <= 0)
-    goto err;
-
-  /* Adjust left and cur */
-  left -= written;
-  cur += written;
-
-  /* Check that we have enough space left for a newline and a NUL */
-  if (left <= 1)
-    goto err;
-
-  /* Emit the newline and NUL */
-  *cur++ = '\n';
-  *cur++ = '\0';
-
-  res = (int)(cur - hex_errno - 1);
-
-  goto done;
-
- err:
-  /*
-   * In error exit, just write a '\0' in the first char so whatever called
-   * this at least won't fall off the end.
-   */
-  *hex_errno = '\0';
-
- done:
-  return res;
-}
-#endif /* !defined(_WIN32) */
-
-/* Maximum number of file descriptors, if we cannot get it via sysconf() */
-#define DEFAULT_MAX_FD 256
-
-/** Terminate the process of <b>process_handle</b>, if that process has not
- * already exited.
- *
- * Return 0 if we succeeded in terminating the process (or if the process
- * already exited), and -1 if we tried to kill the process but failed.
- *
- * Based on code originally borrowed from Python's os.kill. */
-int
-tor_terminate_process(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) {
-    HANDLE handle = process_handle->pid.hProcess;
-
-    if (!TerminateProcess(handle, 0))
-      return -1;
-    else
-      return 0;
-  }
-#else /* !(defined(_WIN32)) */
-  if (process_handle->waitpid_cb) {
-    /* We haven't got a waitpid yet, so we can just kill off the process. */
-    return kill(process_handle->pid, SIGTERM);
-  }
-#endif /* defined(_WIN32) */
-
-  return 0; /* We didn't need to kill the process, so report success */
-}
-
-/** Return the Process ID of <b>process_handle</b>. */
-int
-tor_process_get_pid(process_handle_t *process_handle)
-{
-#ifdef _WIN32
-  return (int) process_handle->pid.dwProcessId;
-#else
-  return (int) process_handle->pid;
-#endif
-}
-
-#ifdef _WIN32
-HANDLE
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#else /* !(defined(_WIN32)) */
-/* DOCDOC tor_process_get_stdout_pipe */
-int
-tor_process_get_stdout_pipe(process_handle_t *process_handle)
-{
-  return process_handle->stdout_pipe;
-}
-#endif /* defined(_WIN32) */
-
-/* DOCDOC process_handle_new */
-static process_handle_t *
-process_handle_new(void)
-{
-  process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t));
-
-#ifdef _WIN32
-  out->stdin_pipe = INVALID_HANDLE_VALUE;
-  out->stdout_pipe = INVALID_HANDLE_VALUE;
-  out->stderr_pipe = INVALID_HANDLE_VALUE;
-#else
-  out->stdin_pipe = -1;
-  out->stdout_pipe = -1;
-  out->stderr_pipe = -1;
-#endif /* defined(_WIN32) */
-
-  return out;
-}
-
-#ifndef _WIN32
-/** Invoked when a process that we've launched via tor_spawn_background() has
- * been found to have terminated.
- */
-static void
-process_handle_waitpid_cb(int status, void *arg)
-{
-  process_handle_t *process_handle = arg;
-
-  process_handle->waitpid_exit_status = status;
-  clear_waitpid_callback(process_handle->waitpid_cb);
-  if (process_handle->status == PROCESS_STATUS_RUNNING)
-    process_handle->status = PROCESS_STATUS_NOTRUNNING;
-  process_handle->waitpid_cb = 0;
-}
-#endif /* !defined(_WIN32) */
-
-/**
- * @name child-process states
- *
- * Each of these values represents a possible state that a child process can
- * be in.  They're used to determine what to say when telling the parent how
- * far along we were before failure.
- *
- * @{
- */
-#define CHILD_STATE_INIT 0
-#define CHILD_STATE_PIPE 1
-#define CHILD_STATE_MAXFD 2
-#define CHILD_STATE_FORK 3
-#define CHILD_STATE_DUPOUT 4
-#define CHILD_STATE_DUPERR 5
-#define CHILD_STATE_DUPIN 6
-#define CHILD_STATE_CLOSEFD 7
-#define CHILD_STATE_EXEC 8
-#define CHILD_STATE_FAILEXEC 9
-/** @} */
-/**
- * Boolean.  If true, then Tor may call execve or CreateProcess via
- * tor_spawn_background.
- **/
-static int may_spawn_background_process = 1;
-/**
- * Turn off may_spawn_background_process, so that all future calls to
- * tor_spawn_background are guaranteed to fail.
- **/
-void
-tor_disable_spawning_background_processes(void)
-{
-  may_spawn_background_process = 0;
-}
-/** Start a program in the background. If <b>filename</b> contains a '/', then
- * it will be treated as an absolute or relative path.  Otherwise, on
- * non-Windows systems, the system path will be searched for <b>filename</b>.
- * On Windows, only the current directory will be searched. Here, to search the
- * system path (as well as the application directory, current working
- * directory, and system directories), set filename to NULL.
- *
- * The strings in <b>argv</b> will be passed as the command line arguments of
- * the child program (following convention, argv[0] should normally be the
- * filename of the executable, and this must be the case if <b>filename</b> is
- * NULL). The last element of argv must be NULL. A handle to the child process
- * will be returned in process_handle (which must be non-NULL). Read
- * process_handle.status to find out if the process was successfully launched.
- * For convenience, process_handle.status is returned by this function.
- *
- * Some parts of this code are based on the POSIX subprocess module from
- * Python, and example code from
- * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx.
- */
-int
-tor_spawn_background(const char *const filename, const char **argv,
-                     process_environment_t *env,
-                     process_handle_t **process_handle_out)
-{
-  if (BUG(may_spawn_background_process == 0)) {
-    /* We should never reach this point if we're forbidden to spawn
-     * processes. Instead we should have caught the attempt earlier. */
-    return PROCESS_STATUS_ERROR;
-  }
-
-#ifdef _WIN32
-  HANDLE stdout_pipe_read = NULL;
-  HANDLE stdout_pipe_write = NULL;
-  HANDLE stderr_pipe_read = NULL;
-  HANDLE stderr_pipe_write = NULL;
-  HANDLE stdin_pipe_read = NULL;
-  HANDLE stdin_pipe_write = NULL;
-  process_handle_t *process_handle;
-  int status;
-
-  STARTUPINFOA siStartInfo;
-  BOOL retval = FALSE;
-
-  SECURITY_ATTRIBUTES saAttr;
-  char *joined_argv;
-
-  saAttr.nLength = sizeof(SECURITY_ATTRIBUTES);
-  saAttr.bInheritHandle = TRUE;
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-  saAttr.lpSecurityDescriptor = NULL;
-
-  /* Assume failure to start process */
-  status = PROCESS_STATUS_ERROR;
-
-  /* Set up pipe for stdout */
-  if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdout communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdout communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stderr */
-  if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stderr communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stderr communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Set up pipe for stdin */
-  if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to create pipe for stdin communication with child process: %s",
-      format_win32_error(GetLastError()));
-    return status;
-  }
-  if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) {
-    log_warn(LD_GENERAL,
-      "Failed to configure pipe for stdin communication with child "
-      "process: %s", format_win32_error(GetLastError()));
-    return status;
-  }
-
-  /* Create the child process */
-
-  /* Windows expects argv to be a whitespace delimited string, so join argv up
-   */
-  joined_argv = tor_join_win_cmdline(argv);
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-
-  ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION));
-  ZeroMemory(&siStartInfo, sizeof(STARTUPINFO));
-  siStartInfo.cb = sizeof(STARTUPINFO);
-  siStartInfo.hStdError = stderr_pipe_write;
-  siStartInfo.hStdOutput = stdout_pipe_write;
-  siStartInfo.hStdInput = stdin_pipe_read;
-  siStartInfo.dwFlags |= STARTF_USESTDHANDLES;
-
-  /* Create the child process */
-
-  retval = CreateProcessA(filename,      // module name
-                 joined_argv,   // command line
-  /* TODO: should we set explicit security attributes? (#2046, comment 5) */
-                 NULL,          // process security attributes
-                 NULL,          // primary thread security attributes
-                 TRUE,          // handles are inherited
-  /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess()
-   * work?) */
-                 CREATE_NO_WINDOW,             // creation flags
-                 (env==NULL) ? NULL : env->windows_environment_block,
-                 NULL,          // use parent's current directory
-                 &siStartInfo,  // STARTUPINFO pointer
-                 &(process_handle->pid));  // receives PROCESS_INFORMATION
-
-  tor_free(joined_argv);
-
-  if (!retval) {
-    log_warn(LD_GENERAL,
-      "Failed to create child process %s: %s", filename?filename:argv[0],
-      format_win32_error(GetLastError()));
-    tor_free(process_handle);
-  } else  {
-    /* TODO: Close hProcess and hThread in process_handle->pid? */
-    process_handle->stdout_pipe = stdout_pipe_read;
-    process_handle->stderr_pipe = stderr_pipe_read;
-    process_handle->stdin_pipe = stdin_pipe_write;
-    status = process_handle->status = PROCESS_STATUS_RUNNING;
-  }
-
-  /* TODO: Close pipes on exit */
-  *process_handle_out = process_handle;
-  return status;
-#else /* !(defined(_WIN32)) */
-  pid_t pid;
-  int stdout_pipe[2];
-  int stderr_pipe[2];
-  int stdin_pipe[2];
-  int fd, retval;
-  process_handle_t *process_handle;
-  int status;
-
-  const char *error_message = SPAWN_ERROR_MESSAGE;
-  size_t error_message_length;
-
-  /* Represents where in the process of spawning the program is;
-     this is used for printing out the error message */
-  unsigned char child_state = CHILD_STATE_INIT;
-
-  char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */
-
-  static int max_fd = -1;
-
-  status = PROCESS_STATUS_ERROR;
-
-  /* We do the strlen here because strlen() is not signal handler safe,
-     and we are not allowed to use unsafe functions between fork and exec */
-  error_message_length = strlen(error_message);
-
-  // child_state = CHILD_STATE_PIPE;
-
-  /* Set up pipe for redirecting stdout, stderr, and stdin of child */
-  retval = pipe(stdout_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdout communication with child process: %s",
-       strerror(errno));
-    return status;
-  }
-
-  retval = pipe(stderr_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stderr communication with child process: %s",
-      strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-
-    return status;
-  }
-
-  retval = pipe(stdin_pipe);
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-      "Failed to set up pipe for stdin communication with child process: %s",
-       strerror(errno));
-
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-
-    return status;
-  }
-
-  // child_state = CHILD_STATE_MAXFD;
-
-#ifdef _SC_OPEN_MAX
-  if (-1 == max_fd) {
-    max_fd = (int) sysconf(_SC_OPEN_MAX);
-    if (max_fd == -1) {
-      max_fd = DEFAULT_MAX_FD;
-      log_warn(LD_GENERAL,
-               "Cannot find maximum file descriptor, assuming %d", max_fd);
-    }
-  }
-#else /* !(defined(_SC_OPEN_MAX)) */
-  max_fd = DEFAULT_MAX_FD;
-#endif /* defined(_SC_OPEN_MAX) */
-
-  // child_state = CHILD_STATE_FORK;
-
-  pid = fork();
-  if (0 == pid) {
-    /* In child */
-
-#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__)
-    /* Attempt to have the kernel issue a SIGTERM if the parent
-     * goes away. Certain attributes of the binary being execve()ed
-     * will clear this during the execve() call, but it's better
-     * than nothing.
-     */
-    prctl(PR_SET_PDEATHSIG, SIGTERM);
-#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */
-
-    child_state = CHILD_STATE_DUPOUT;
-
-    /* Link child stdout to the write end of the pipe */
-    retval = dup2(stdout_pipe[1], STDOUT_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPERR;
-
-    /* Link child stderr to the write end of the pipe */
-    retval = dup2(stderr_pipe[1], STDERR_FILENO);
-    if (-1 == retval)
-        goto error;
-
-    child_state = CHILD_STATE_DUPIN;
-
-    /* Link child stdin to the read end of the pipe */
-    retval = dup2(stdin_pipe[0], STDIN_FILENO);
-    if (-1 == retval)
-      goto error;
-
-    // child_state = CHILD_STATE_CLOSEFD;
-
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-
-    /* Close all other fds, including the read end of the pipe */
-    /* XXX: We should now be doing enough FD_CLOEXEC setting to make
-     * this needless. */
-    for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) {
-      close(fd);
-    }
-
-    // child_state = CHILD_STATE_EXEC;
-
-    /* Call the requested program. We need the cast because
-       execvp doesn't define argv as const, even though it
-       does not modify the arguments */
-    if (env)
-      execve(filename, (char *const *) argv, env->unixoid_environment_block);
-    else {
-      static char *new_env[] = { NULL };
-      execve(filename, (char *const *) argv, new_env);
-    }
-
-    /* If we got here, the exec or open(/dev/null) failed */
-
-    child_state = CHILD_STATE_FAILEXEC;
-
-  error:
-    {
-      /* XXX: are we leaking fds from the pipe? */
-      int n, err=0;
-      ssize_t nbytes;
-
-      n = format_helper_exit_status(child_state, errno, hex_errno);
-
-      if (n >= 0) {
-        /* Write the error message. GCC requires that we check the return
-           value, but there is nothing we can do if it fails */
-        /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */
-        nbytes = write(STDOUT_FILENO, error_message, error_message_length);
-        err = (nbytes < 0);
-        nbytes = write(STDOUT_FILENO, hex_errno, n);
-        err += (nbytes < 0);
-      }
-
-      _exit(err?254:255); // exit ok: in child.
-    }
-
-    /* Never reached, but avoids compiler warning */
-    return status; // LCOV_EXCL_LINE
-  }
-
-  /* In parent */
-
-  if (-1 == pid) {
-    log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno));
-    close(stdin_pipe[0]);
-    close(stdin_pipe[1]);
-    close(stdout_pipe[0]);
-    close(stdout_pipe[1]);
-    close(stderr_pipe[0]);
-    close(stderr_pipe[1]);
-    return status;
-  }
-
-  process_handle = process_handle_new();
-  process_handle->status = status;
-  process_handle->pid = pid;
-
-  /* TODO: If the child process forked but failed to exec, waitpid it */
-
-  /* Return read end of the pipes to caller, and close write end */
-  process_handle->stdout_pipe = stdout_pipe[0];
-  retval = close(stdout_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stdout pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  process_handle->waitpid_cb = set_waitpid_callback(pid,
-                                                    process_handle_waitpid_cb,
-                                                    process_handle);
-
-  process_handle->stderr_pipe = stderr_pipe[0];
-  retval = close(stderr_pipe[1]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close write end of stderr pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  /* Return write end of the stdin pipe to caller, and close the read end */
-  process_handle->stdin_pipe = stdin_pipe[1];
-  retval = close(stdin_pipe[0]);
-
-  if (-1 == retval) {
-    log_warn(LD_GENERAL,
-            "Failed to close read end of stdin pipe in parent process: %s",
-            strerror(errno));
-  }
-
-  status = process_handle->status = PROCESS_STATUS_RUNNING;
-  /* Set stdin/stdout/stderr pipes to be non-blocking */
-  if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 ||
-      fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) {
-    log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes "
-             "nonblocking in parent process: %s", strerror(errno));
-  }
-
-  *process_handle_out = process_handle;
-  return status;
-#endif /* defined(_WIN32) */
-}
-
-/** Destroy all resources allocated by the process handle in
- *  <b>process_handle</b>.
- *  If <b>also_terminate_process</b> is true, also terminate the
- *  process of the process handle. */
-MOCK_IMPL(void,
-tor_process_handle_destroy,(process_handle_t *process_handle,
-                            int also_terminate_process))
-{
-  if (!process_handle)
-    return;
-
-  if (also_terminate_process) {
-    if (tor_terminate_process(process_handle) < 0) {
-      const char *errstr =
-#ifdef _WIN32
-        format_win32_error(GetLastError());
-#else
-        strerror(errno);
-#endif
-      log_notice(LD_GENERAL, "Failed to terminate process with "
-                 "PID '%d' ('%s').", tor_process_get_pid(process_handle),
-                 errstr);
-    } else {
-      log_info(LD_GENERAL, "Terminated process with PID '%d'.",
-               tor_process_get_pid(process_handle));
-    }
-  }
-
-  process_handle->status = PROCESS_STATUS_NOTRUNNING;
-
-#ifdef _WIN32
-  if (process_handle->stdout_pipe)
-    CloseHandle(process_handle->stdout_pipe);
-
-  if (process_handle->stderr_pipe)
-    CloseHandle(process_handle->stderr_pipe);
-
-  if (process_handle->stdin_pipe)
-    CloseHandle(process_handle->stdin_pipe);
-#else /* !(defined(_WIN32)) */
-  close(process_handle->stdout_pipe);
-  close(process_handle->stderr_pipe);
-  close(process_handle->stdin_pipe);
-
-  clear_waitpid_callback(process_handle->waitpid_cb);
-#endif /* defined(_WIN32) */
-
-  memset(process_handle, 0x0f, sizeof(process_handle_t));
-  tor_free(process_handle);
-}
-
-/** Get the exit code of a process specified by <b>process_handle</b> and store
- * it in <b>exit_code</b>, if set to a non-NULL value.  If <b>block</b> is set
- * to true, the call will block until the process has exited.  Otherwise if
- * the process is still running, the function will return
- * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns
- * PROCESS_EXIT_EXITED if the process did exit. If there is a failure,
- * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if
- * non-NULL) will be undefined. N.B. Under *nix operating systems, this will
- * probably not work in Tor, because waitpid() is called in main.c to reap any
- * terminated child processes.*/
-int
-tor_get_exit_code(process_handle_t *process_handle,
-                  int block, int *exit_code)
-{
-#ifdef _WIN32
-  DWORD retval;
-  BOOL success;
-
-  if (block) {
-    /* Wait for the process to exit */
-    retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE);
-    if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-              (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  } else {
-    retval = WaitForSingleObject(process_handle->pid.hProcess, 0);
-    if (WAIT_TIMEOUT == retval) {
-      /* Process has not exited */
-      return PROCESS_EXIT_RUNNING;
-    } else if (retval != WAIT_OBJECT_0) {
-      log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s",
-               (int)retval, format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-
-  if (exit_code != NULL) {
-    success = GetExitCodeProcess(process_handle->pid.hProcess,
-                                 (PDWORD)exit_code);
-    if (!success) {
-      log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s",
-               format_win32_error(GetLastError()));
-      return PROCESS_EXIT_ERROR;
-    }
-  }
-#else /* !(defined(_WIN32)) */
-  int stat_loc;
-  int retval;
-
-  if (process_handle->waitpid_cb) {
-    /* We haven't processed a SIGCHLD yet. */
-    retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG);
-    if (retval == process_handle->pid) {
-      clear_waitpid_callback(process_handle->waitpid_cb);
-      process_handle->waitpid_cb = NULL;
-      process_handle->waitpid_exit_status = stat_loc;
-    }
-  } else {
-    /* We already got a SIGCHLD for this process, and handled it. */
-    retval = process_handle->pid;
-    stat_loc = process_handle->waitpid_exit_status;
-  }
-
-  if (!block && 0 == retval) {
-    /* Process has not exited */
-    return PROCESS_EXIT_RUNNING;
-  } else if (retval != process_handle->pid) {
-    log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s",
-             (int)process_handle->pid, strerror(errno));
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (!WIFEXITED(stat_loc)) {
-    log_warn(LD_GENERAL, "Process %d did not exit normally",
-             (int)process_handle->pid);
-    return PROCESS_EXIT_ERROR;
-  }
-
-  if (exit_code != NULL)
-    *exit_code = WEXITSTATUS(stat_loc);
-#endif /* defined(_WIN32) */
-
-  return PROCESS_EXIT_EXITED;
-}
-
-#ifdef _WIN32
-/** Read from a handle <b>h</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>hProcess</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise <b>hProcess</b> should be set to the handle
- * to the process owning the <b>h</b>. In this case, the function will exit
- * only once the process has exited, or <b>count</b> bytes are read. Returns
- * the number of bytes read, or -1 on error. */
-ssize_t
-tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                    const process_handle_t *process)
-{
-  size_t numread = 0;
-  BOOL retval;
-  DWORD byte_count;
-  BOOL process_exited = FALSE;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    /* Check if there is anything to read */
-    retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL);
-    if (!retval) {
-      log_warn(LD_GENERAL,
-        "Failed to peek from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* Nothing available: process exited or it is busy */
-
-      /* Exit if we don't know whether the process is running */
-      if (NULL == process)
-        break;
-
-      /* The process exited and there's nothing left to read from it */
-      if (process_exited)
-        break;
-
-      /* If process is not running, check for output one more time in case
-         it wrote something after the peek was performed. Otherwise keep on
-         waiting for output */
-      tor_assert(process != NULL);
-      byte_count = WaitForSingleObject(process->pid.hProcess, 0);
-      if (WAIT_TIMEOUT != byte_count)
-        process_exited = TRUE;
-
-      continue;
-    }
-
-    /* There is data to read; read it */
-    retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL);
-    tor_assert(byte_count + numread <= count);
-    if (!retval) {
-      log_warn(LD_GENERAL, "Failed to read from handle: %s",
-        format_win32_error(GetLastError()));
-      return -1;
-    } else if (0 == byte_count) {
-      /* End of file */
-      break;
-    }
-    numread += byte_count;
-  }
-  return (ssize_t)numread;
-}
-#else /* !(defined(_WIN32)) */
-/** Read from a handle <b>fd</b> into <b>buf</b>, up to <b>count</b> bytes.  If
- * <b>process</b> is NULL, the function will return immediately if there is
- * nothing more to read. Otherwise data will be read until end of file, or
- * <b>count</b> bytes are read.  Returns the number of bytes read, or -1 on
- * error. Sets <b>eof</b> to true if <b>eof</b> is not NULL and the end of the
- * file has been reached. */
-ssize_t
-tor_read_all_handle(int fd, char *buf, size_t count,
-                    const process_handle_t *process,
-                    int *eof)
-{
-  size_t numread = 0;
-  ssize_t result;
-
-  if (eof)
-    *eof = 0;
-
-  if (count > SIZE_T_CEILING || count > SSIZE_MAX)
-    return -1;
-
-  while (numread < count) {
-    result = read(fd, buf+numread, count-numread);
-
-    if (result == 0) {
-      log_debug(LD_GENERAL, "read() reached end of file");
-      if (eof)
-        *eof = 1;
-      break;
-    } else if (result < 0 && errno == EAGAIN) {
-      if (process)
-        continue;
-      else
-        break;
-    } else if (result < 0) {
-      log_warn(LD_GENERAL, "read() failed: %s", strerror(errno));
-      return -1;
-    }
-
-    numread += result;
-  }
-
-  log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread);
-  return (ssize_t)numread;
-}
-#endif /* defined(_WIN32) */
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stdout(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stdout_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Read from stdout of a process until the process exits. */
-ssize_t
-tor_read_all_from_process_stderr(const process_handle_t *process_handle,
-                                 char *buf, size_t count)
-{
-#ifdef _WIN32
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle);
-#else
-  return tor_read_all_handle(process_handle->stderr_pipe, buf, count,
-                             process_handle, NULL);
-#endif /* defined(_WIN32) */
-}
-
-/** Return a string corresponding to <b>stream_status</b>. */
-const char *
-stream_status_to_string(enum stream_status stream_status)
-{
-  switch (stream_status) {
-    case IO_STREAM_OKAY:
-      return "okay";
-    case IO_STREAM_EAGAIN:
-      return "temporarily unavailable";
-    case IO_STREAM_TERM:
-      return "terminated";
-    case IO_STREAM_CLOSED:
-      return "closed";
-    default:
-      tor_fragile_assert();
-      return "unknown";
-  }
-}
-
-/** Split buf into lines, and add to smartlist. The buffer <b>buf</b> will be
- * modified. The resulting smartlist will consist of pointers to buf, so there
- * is no need to free the contents of sl. <b>buf</b> must be a NUL-terminated
- * string. <b>len</b> should be set to the length of the buffer excluding the
- * NUL. Non-printable characters (including NUL) will be replaced with "." */
-int
-tor_split_lines(smartlist_t *sl, char *buf, int len)
-{
-  /* Index in buf of the start of the current line */
-  int start = 0;
-  /* Index in buf of the current character being processed */
-  int cur = 0;
-  /* Are we currently in a line */
-  char in_line = 0;
-
-  /* Loop over string */
-  while (cur < len) {
-    /* Loop until end of line or end of string */
-    for (; cur < len; cur++) {
-      if (in_line) {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* End of line */
-          buf[cur] = '\0';
-          /* Point cur to the next line */
-          cur++;
-          /* Line starts at start and ends with a nul */
-          break;
-        } else {
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      } else {
-        if ('\r' == buf[cur] || '\n' == buf[cur]) {
-          /* Skip leading vertical space */
-          ;
-        } else {
-          in_line = 1;
-          start = cur;
-          if (!TOR_ISPRINT(buf[cur]))
-            buf[cur] = '.';
-        }
-      }
-    }
-    /* We are at the end of the line or end of string. If in_line is true there
-     * is a line which starts at buf+start and ends at a NUL. cur points to
-     * the character after the NUL. */
-    if (in_line)
-      smartlist_add(sl, (void *)(buf+start));
-    in_line = 0;
-  }
-  return smartlist_len(sl);
-}
-
-#ifdef _WIN32
-
-/** Return a smartlist containing lines outputted from
- *  <b>handle</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (HANDLE *handle,
-                            enum stream_status *stream_status_out))
-{
-  int pos;
-  char stdout_buf[600] = {0};
-  smartlist_t *lines = NULL;
-
-  tor_assert(stream_status_out);
-
-  *stream_status_out = IO_STREAM_TERM;
-
-  pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL);
-  if (pos < 0) {
-    *stream_status_out = IO_STREAM_TERM;
-    return NULL;
-  }
-  if (pos == 0) {
-    *stream_status_out = IO_STREAM_EAGAIN;
-    return NULL;
-  }
-
-  /* End with a null even if there isn't a \r\n at the end */
-  /* TODO: What if this is a partial line? */
-  stdout_buf[pos] = '\0';
-
-  /* Split up the buffer */
-  lines = smartlist_new();
-  tor_split_lines(lines, stdout_buf, pos);
-
-  /* Currently 'lines' is populated with strings residing on the
-     stack. Replace them with their exact copies on the heap: */
-  SMARTLIST_FOREACH(lines, char *, line,
-                    SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line)));
-
-  *stream_status_out = IO_STREAM_OKAY;
-
-  return lines;
-}
-
-#else /* !(defined(_WIN32)) */
-
-/** Return a smartlist containing lines outputted from
- *  <b>fd</b>. Return NULL on error, and set
- *  <b>stream_status_out</b> appropriately. */
-MOCK_IMPL(smartlist_t *,
-tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out))
-{
-  enum stream_status stream_status;
-  char stdout_buf[400];
-  smartlist_t *lines = NULL;
-
-  while (1) {
-    memset(stdout_buf, 0, sizeof(stdout_buf));
-
-    stream_status = get_string_from_pipe(fd,
-                                         stdout_buf, sizeof(stdout_buf) - 1);
-    if (stream_status != IO_STREAM_OKAY)
-      goto done;
-
-    if (!lines) lines = smartlist_new();
-    smartlist_split_string(lines, stdout_buf, "\n", 0, 0);
-  }
-
- done:
-  *stream_status_out = stream_status;
-  return lines;
-}
-
-#endif /* defined(_WIN32) */
-
-/** Reads from <b>fd</b> and stores input in <b>buf_out</b> making
- *  sure it's below <b>count</b> bytes.
- *  If the string has a trailing newline, we strip it off.
- *
- * This function is specifically created to handle input from managed
- * proxies, according to the pluggable transports spec. Make sure it
- * fits your needs before using it.
- *
- * Returns:
- * IO_STREAM_CLOSED: If the stream is closed.
- * IO_STREAM_EAGAIN: If there is nothing to read and we should check back
- *  later.
- * IO_STREAM_TERM: If something is wrong with the stream.
- * IO_STREAM_OKAY: If everything went okay and we got a string
- *  in <b>buf_out</b>. */
-enum stream_status
-get_string_from_pipe(int fd, char *buf_out, size_t count)
-{
-  ssize_t ret;
-
-  tor_assert(count <= INT_MAX);
-
-  ret = read(fd, buf_out, count);
-
-  if (ret == 0)
-    return IO_STREAM_CLOSED;
-  else if (ret < 0 && errno == EAGAIN)
-    return IO_STREAM_EAGAIN;
-  else if (ret < 0)
-    return IO_STREAM_TERM;
-
-  if (buf_out[ret - 1] == '\n') {
-    /* Remove the trailing newline */
-    buf_out[ret - 1] = '\0';
-  } else
-    buf_out[ret] = '\0';
-
-  return IO_STREAM_OKAY;
-}
diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h
deleted file mode 100644
index aa3127d62d..0000000000
--- a/src/lib/process/subprocess.h
+++ /dev/null
@@ -1,134 +0,0 @@
-/* Copyright (c) 2003-2004, Roger Dingledine
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-/**
- * \file subprocess.h
- * \brief Header for subprocess.c
- **/
-
-#ifndef TOR_SUBPROCESS_H
-#define TOR_SUBPROCESS_H
-
-#include "lib/cc/torint.h"
-#include "lib/testsupport/testsupport.h"
-#include <stddef.h>
-#ifdef _WIN32
-#include <windows.h>
-#endif
-
-struct smartlist_t;
-
-void tor_disable_spawning_background_processes(void);
-
-typedef struct process_handle_t process_handle_t;
-struct process_environment_t;
-int tor_spawn_background(const char *const filename, const char **argv,
-                         struct process_environment_t *env,
-                         process_handle_t **process_handle_out);
-
-#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code "
-
-/** Status of an I/O stream. */
-enum stream_status {
-  IO_STREAM_OKAY,
-  IO_STREAM_EAGAIN,
-  IO_STREAM_TERM,
-  IO_STREAM_CLOSED
-};
-
-const char *stream_status_to_string(enum stream_status stream_status);
-
-enum stream_status get_string_from_pipe(int fd, char *buf, size_t count);
-
-/* Values of process_handle_t.status. */
-#define PROCESS_STATUS_NOTRUNNING 0
-#define PROCESS_STATUS_RUNNING 1
-#define PROCESS_STATUS_ERROR -1
-
-#ifdef SUBPROCESS_PRIVATE
-struct waitpid_callback_t;
-
-/** Structure to represent the state of a process with which Tor is
- * communicating. The contents of this structure are private to util.c */
-struct process_handle_t {
-  /** One of the PROCESS_STATUS_* values */
-  int status;
-#ifdef _WIN32
-  HANDLE stdin_pipe;
-  HANDLE stdout_pipe;
-  HANDLE stderr_pipe;
-  PROCESS_INFORMATION pid;
-#else /* !(defined(_WIN32)) */
-  int stdin_pipe;
-  int stdout_pipe;
-  int stderr_pipe;
-  pid_t pid;
-  /** If the process has not given us a SIGCHLD yet, this has the
-   * waitpid_callback_t that gets invoked once it has. Otherwise this
-   * contains NULL. */
-  struct waitpid_callback_t *waitpid_cb;
-  /** The exit status reported by waitpid. */
-  int waitpid_exit_status;
-#endif /* defined(_WIN32) */
-};
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-/* Return values of tor_get_exit_code() */
-#define PROCESS_EXIT_RUNNING 1
-#define PROCESS_EXIT_EXITED 0
-#define PROCESS_EXIT_ERROR -1
-int tor_get_exit_code(process_handle_t *process_handle,
-                      int block, int *exit_code);
-int tor_split_lines(struct smartlist_t *sl, char *buf, int len);
-#ifdef _WIN32
-ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count,
-                            const process_handle_t *process);
-#else
-ssize_t tor_read_all_handle(int fd, char *buf, size_t count,
-                            const process_handle_t *process,
-                            int *eof);
-#endif /* defined(_WIN32) */
-ssize_t tor_read_all_from_process_stdout(
-    const process_handle_t *process_handle, char *buf, size_t count);
-ssize_t tor_read_all_from_process_stderr(
-    const process_handle_t *process_handle, char *buf, size_t count);
-char *tor_join_win_cmdline(const char *argv[]);
-
-int tor_process_get_pid(process_handle_t *process_handle);
-#ifdef _WIN32
-HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#else
-int tor_process_get_stdout_pipe(process_handle_t *process_handle);
-#endif
-
-#ifdef _WIN32
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle,
-                           enum stream_status *stream_status));
-#else
-MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd,
-                                       enum stream_status *stream_status));
-#endif /* defined(_WIN32) */
-
-int tor_terminate_process(process_handle_t *process_handle);
-
-MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle,
-                                            int also_terminate_process));
-
-#ifdef SUBPROCESS_PRIVATE
-/* Prototypes for private functions only used by util.c (and unit tests) */
-
-#ifndef _WIN32
-STATIC int format_helper_exit_status(unsigned char child_state,
-                              int saved_errno, char *hex_errno);
-
-/* Space for hex values of child state, a slash, saved_errno (with
-   leading minus) and newline (no null) */
-#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \
-                        1 + sizeof(int) * 2 + 1)
-#endif /* !defined(_WIN32) */
-
-#endif /* defined(SUBPROCESS_PRIVATE) */
-
-#endif
diff --git a/src/lib/process/waitpid.c b/src/lib/process/waitpid.c
index 9b626394d2..33798f65f0 100644
--- a/src/lib/process/waitpid.c
+++ b/src/lib/process/waitpid.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,7 +16,7 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
-#include "ht.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>
@@ -58,9 +58,9 @@ process_map_entries_eq_(const waitpid_callback_t *a,
 static HT_HEAD(process_map, waitpid_callback_t) process_map = HT_INITIALIZER();
 
 HT_PROTOTYPE(process_map, waitpid_callback_t, node, process_map_entry_hash_,
-             process_map_entries_eq_)
+             process_map_entries_eq_);
 HT_GENERATE2(process_map, waitpid_callback_t, node, process_map_entry_hash_,
-             process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_)
+             process_map_entries_eq_, 0.6, tor_reallocarray_, tor_free_);
 
 /**
  * Begin monitoring the child pid <b>pid</b> to see if we get a SIGCHLD for
diff --git a/src/lib/process/waitpid.h b/src/lib/process/waitpid.h
index 5faef468c1..171cf206fb 100644
--- a/src/lib/process/waitpid.h
+++ b/src/lib/process/waitpid.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/pubsub/.may_include b/src/lib/pubsub/.may_include
new file mode 100644
index 0000000000..5623492f00
--- /dev/null
+++ b/src/lib/pubsub/.may_include
@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/cc/*.h
+lib/container/*.h
+lib/dispatch/*.h
+lib/intmath/*.h
+lib/log/*.h
+lib/malloc/*.h
+lib/pubsub/*.h
+lib/string/*.h
diff --git a/src/lib/pubsub/include.am b/src/lib/pubsub/include.am
new file mode 100644
index 0000000000..e2abebcd40
--- /dev/null
+++ b/src/lib/pubsub/include.am
@@ -0,0 +1,28 @@
+
+noinst_LIBRARIES += src/lib/libtor-pubsub.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-pubsub-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_pubsub_a_SOURCES =			\
+	src/lib/pubsub/pubsub_build.c			\
+	src/lib/pubsub/pubsub_check.c			\
+	src/lib/pubsub/pubsub_publish.c
+
+src_lib_libtor_pubsub_testing_a_SOURCES = \
+	$(src_lib_libtor_pubsub_a_SOURCES)
+src_lib_libtor_pubsub_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_pubsub_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/pubsub/pub_binding_st.h			\
+	src/lib/pubsub/pubsub.h				\
+	src/lib/pubsub/pubsub_build.h			\
+	src/lib/pubsub/pubsub_builder_st.h		\
+	src/lib/pubsub/pubsub_connect.h			\
+	src/lib/pubsub/pubsub_flags.h			\
+	src/lib/pubsub/pubsub_macros.h			\
+	src/lib/pubsub/pubsub_publish.h
diff --git a/src/lib/pubsub/lib_pubsub.md b/src/lib/pubsub/lib_pubsub.md
new file mode 100644
index 0000000000..3f4c473436
--- /dev/null
+++ b/src/lib/pubsub/lib_pubsub.md
@@ -0,0 +1,14 @@
+@dir /lib/pubsub
+@brief lib/pubsub: Publish-subscribe message passing.
+
+This module wraps the \refdir{lib/dispatch} module, to provide a more
+ergonomic and type-safe approach to message passing.
+
+In general, we favor this mechanism for cases where higher-level modules
+need to be notified when something happens in lower-level modules. (The
+alternative would be calling up from the lower-level modules, which
+would be error-prone; or maintaining lists of function-pointers, which
+would be clumsy and tend to complicate the call graph.)
+
+See pubsub.c for more information.
+
diff --git a/src/lib/pubsub/pub_binding_st.h b/src/lib/pubsub/pub_binding_st.h
new file mode 100644
index 0000000000..d7c562fc35
--- /dev/null
+++ b/src/lib/pubsub/pub_binding_st.h
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pub_binding_st.h
+ * @brief Declaration of pub_binding_t.
+ *
+ * This is an internal type for the pubsub implementation.
+ */
+
+#ifndef TOR_PUB_BINDING_ST_H
+#define TOR_PUB_BINDING_ST_H
+
+#include "lib/dispatch/msgtypes.h"
+struct dispatch_t;
+
+/**
+ * A pub_binding_t is an opaque object that subsystems use to publish
+ * messages.  The DISPATCH_ADD_PUB*() macros set it up.
+ **/
+typedef struct pub_binding_t {
+  /**
+   * A pointer to a configured dispatch_t object.  This is filled in
+   * when the dispatch_t is finally constructed.
+   **/
+  struct dispatch_t *dispatch_ptr;
+  /**
+   * A template for the msg_t fields that are filled in for this message.
+   * This is copied into outgoing messages, ensuring that their fields are set
+   * corretly.
+   **/
+  msg_t msg_template;
+} pub_binding_t;
+
+#endif /* !defined(TOR_PUB_BINDING_ST_H) */
diff --git a/src/lib/pubsub/publish_subscribe.md b/src/lib/pubsub/publish_subscribe.md
new file mode 100644
index 0000000000..bb05b100b1
--- /dev/null
+++ b/src/lib/pubsub/publish_subscribe.md
@@ -0,0 +1,144 @@
+
+@page publish_subscribe Publish-subscribe message passing in Tor
+
+@tableofcontents
+
+## Introduction
+
+Tor has introduced a generic publish-subscribe mechanism for delivering
+messages internally.  It is meant to help us improve the modularity of
+our code, by avoiding direct coupling between modules that don't
+actually need to invoke one another.
+
+This publish-subscribe mechanism is *not* meant for handing
+multithreading or multiprocess issues, thought we hope that eventually
+it might be extended and adapted for that purpose.  Instead, we use
+publish-subscribe today to decouple modules that shouldn't be calling
+each other directly.
+
+For example, there are numerous parts of our code that might need to
+take action when a circuit is completed: a controller might need to be
+informed, an onion service negotiation might need to be attached, a
+guard might need to be marked as working, or a client connection might
+need to be attached.  But many of those actions occur at a higher layer
+than circuit completion: calling them directly is a layering violation,
+and makes our code harder to understand and analyze.
+
+But with message-passing, we can invert this layering violation: circuit
+completion can become a "message" that the circuit code publishes, and
+to which higher-level layers subscribe.  This means that circuit
+handling can be decoupled from higher-level modules, and stay nice and
+simple. (@ref pubsub_notyet "1")
+
+> @anchor pubsub_notyet 1. Unfortunately, like most of our code, circuit
+> handling is _not_ yet refactored to use publish-subscribe throughout.
+> Instead, layer violations of the type described here are pretty common
+> in Tor today.  To see a small part of what happens when a circuit is
+> completed today, have a look at circuit_build_no_more_hops() and its
+> associated code.
+
+## Channels and delivery policies
+
+To work with messages, especially when refactoring existing code, you'll
+need to understand "channels" and "delivery policies".
+
+Every message is delivered on a "message channel".  Each channel
+(conceptually) a queue-like structure that can support an arbitrarily
+number of message types.  Where channels vary is their delivery
+mechanisms, and their guarantees about when messages are processed.
+
+Currently, three delivery policies are possible:
+
+   - `DELIV_PROMPT` -- causes messages to be processed via a callback in
+      Tor's event loop.  This is generally the best choice, since it
+      avoids unexpected growth of the stack.
+
+   - `DELIV_IMMEDIATE` -- causes messages to be processed immediately
+      on the call stack when they are published.  This choice grows the
+      stack, and can lead to unexpected complexity in the call graph.
+      We should only use it when necessary.
+
+   - `DELIV_NEVER` -- causes messages not to be delivered by the message
+      dispatch system at all. Instead, some other part of the code must
+      call dispatch_flush() to get the messages delivered.
+
+See mainloop_pubsub.c and mainloop_pubsub.h for more information and
+implementation details.
+
+## Layers: Dispatch vs publish-subsubscribe vs mainloop.
+
+At the lowest level, messages are sent via the "dispatcher" module in
+@refdir{lib/dispatch}.  For performance, this dispatcher works with a
+untyped messages.  Publishers, subscribers, channels, and messages are
+distinguished by short integers.  Associated data is handled as
+dynamically-typed data pointers, and its types are also stored as short
+integers.
+
+Naturally, this results in a type-unsafe C API, so most other modules
+shouldn't invoke @refdir{lib/dispatch} directly.  At a higher level,
+@refdir{lib/pubsub} defines a set of functions and macros that make
+messages named and type-safe.  This is the one that other modules should
+use when they want to send or receive a message.
+
+The two modules above do not handle message delivery.  Instead, the
+dispatch module takes a callback that it can invoke when a channel
+becomes nonempty, and defines a dispatch_flush() function to deliver all
+the messages queued in a channel.  The work of actually making sure that
+dispatch_flush() is called when appropriate falls to the main loop,
+which needs to integrate the message dispatcher with the rest of our
+events and callbacks.  This work happens in mainloop_pubsub.c.
+
+
+## How to publish and subscribe
+
+This section gives an overview of how to make new messages and how to
+use them.  For full details, see pubsub_macros.h.
+
+Before anybody can publish or subscribe to a message, the message must
+be declared, typically in a header.  This uses DECLARE_MESSAGE() or
+DECLARE_MESSAGE_INT().
+
+Only subsystems can publish or subscribe messages.  For more information
+about the subsystems architecture, see @ref initialization.
+
+To publish a message, you must:
+   - Include the header that declares the message.
+   - Declare a set of helper functions via DECLARE_PUBLISH().  These
+     must be visible wherever you call PUBLISH().
+   - Call PUBLISH() to actually send a message.
+   - Connect your subsystem to the dispatcher by calling
+     DISPATCH_ADD_PUB() from your subsystem's subsys_fns_t.add_pubsub
+     callback.
+
+To subscribe to a message, you must:
+   - Include the header that declares the message.
+   - Declare a callback function to be invoked when the message is delivered.
+   - Use DISPATCH_SUBSCRIBE at file scope to define a set of wrapper
+     functions to call your callback function with the appropriate type.
+   - Connect your subsystem to the dispatcher by calling
+     DISPATCH_ADD_SUB() from your subsystem's subsys_fns_t.add_pubsub
+     callback.
+
+Again, the file-level documentation for pubsub_macros.h describes how to
+declare a message, how to publish it, and how to subscribe to it.
+
+## Designing good messages
+
+**Frequency**:
+The publish-subscribe system uses a few function calls
+and allocations for each message sent. This makes it unsuitable for
+very-high-bandwidth events, like "receiving a single data cell" or "a
+socket has become writable."  It's fine, however, for events that
+ordinarily happen a bit less frequently than that, like a circuit
+getting finished, a new connection getting opened, or so on.
+
+**Semantics**:
+A message should declare that something has happened or is happening,
+not that something in particular should be done.
+
+For example, suppose you want to set up a message so that onion services
+clean up their replay caches whenever we're low on memory.  The event
+should be something like `memory_low`, not `clean_up_replay_caches`.
+The latter name would imply that the publisher knew who was subscribing
+to the message and what they intended to do about it, which would be a
+layering violation.
diff --git a/src/lib/pubsub/pubsub.h b/src/lib/pubsub/pubsub.h
new file mode 100644
index 0000000000..d0a4d317f3
--- /dev/null
+++ b/src/lib/pubsub/pubsub.h
@@ -0,0 +1,89 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub.h
+ * @brief Header for OO publish-subscribe functionality.
+ *
+ * This module provides a wrapper around the "dispatch" module,
+ * ensuring type-safety and allowing us to do static analysis on
+ * publication and subscriptions.
+ *
+ * With this module, we enforce:
+ *  <ul>
+ *   <li>that every message has (potential) publishers and subscribers;
+ *   <li>that every message is published and subscribed from the correct
+ *       channels, with the correct type ID, every time it is published.
+ *   <li>that type IDs correspond to a single C type, and that the C types are
+ *       used correctly.
+ *   <li>that when a message is published or subscribed, it is done with
+ *       a correct subsystem identifier
+ * </ul>
+ *
+ * We do this by making "publication requests" and "subscription requests"
+ * into objects, and doing some computation on them before we create
+ * a dispatch_t with them.
+ *
+ * Rather than using the dispatch module directly, a publishing module
+ * receives a "binding" object that it uses to send messages with the right
+ * settings.
+ *
+ * Most users of this module will want to use this header, and the
+ * pubsub_macros.h header for convenience.
+ */
+
+/*
+ *
+ * Overview: Messages are sent over channels.  Before sending a message on a
+ * channel, or receiving a message on a channel, a subsystem needs to register
+ * that it publishes, or subscribes, to that message, on that channel.
+ *
+ * Messages, channels, and subsystems are represented internally as short
+ * integers, though they are associated with human-readable strings for
+ * initialization and debugging.
+ *
+ * When registering for a message, a subsystem must say whether it is an
+ * exclusive publisher/subscriber to that message type, or whether other
+ * subsystems may also publish/subscribe to it.
+ *
+ * All messages and their publishers/subscribers must be registered early in
+ * the initialization process.
+ *
+ * By default, it is an error for a message type to have publishers and no
+ * subscribers on a channel, or subscribers and no publishers on a channel.
+ *
+ * A subsystem may register for a message with a note that delivery or
+ * production is disabled -- for example, because the subsystem is
+ * disabled at compile-time. It is not an error for a message type to
+ * have all of its publishers or subscribers disabled.
+ *
+ * After a message is sent, it is delivered to every recipient.  This
+ * delivery happens from the top level of the event loop; it may be
+ * interleaved with network events, timers, etc.
+ *
+ * Messages may have associated data.  This data is typed, and is owned
+ * by the message.  Strings, byte-arrays, and integers have built-in
+ * support.  Other types may be added.  If objects are to be sent,
+ * they should be identified by handle.  If an object requires cleanup,
+ * it should be declared with an associated free function.
+ *
+ * Semantically, if two subsystems communicate only by this kind of
+ * message passing, neither is considered to depend on the other, though
+ * both are considered to have a dependency on the message and on any
+ * types it contains.
+ *
+ * (Or generational index?)
+ */
+#ifndef TOR_PUBSUB_PUBSUB_H
+#define TOR_PUBSUB_PUBSUB_H
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#endif /* !defined(TOR_PUBSUB_PUBSUB_H) */
diff --git a/src/lib/pubsub/pubsub_build.c b/src/lib/pubsub/pubsub_build.c
new file mode 100644
index 0000000000..3c134f015c
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.c
@@ -0,0 +1,307 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.c
+ * @brief Construct a dispatch_t in safer, more OO way.
+ **/
+
+#define PUBSUB_PRIVATE
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+
+ #include <string.h>
+
+/** Construct and return a new empty pubsub_items_t. */
+static pubsub_items_t *
+pubsub_items_new(void)
+{
+  pubsub_items_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->items = smartlist_new();
+  cfg->type_items = smartlist_new();
+  return cfg;
+}
+
+/** Release all storage held in a pubsub_items_t. */
+void
+pubsub_items_free_(pubsub_items_t *cfg)
+{
+  if (! cfg)
+    return;
+  SMARTLIST_FOREACH(cfg->items, pubsub_cfg_t *, item, tor_free(item));
+  SMARTLIST_FOREACH(cfg->type_items,
+                    pubsub_type_cfg_t *, item, tor_free(item));
+  smartlist_free(cfg->items);
+  smartlist_free(cfg->type_items);
+  tor_free(cfg);
+}
+
+/** Construct and return a new pubsub_builder_t. */
+pubsub_builder_t *
+pubsub_builder_new(void)
+{
+  dispatch_naming_init();
+
+  pubsub_builder_t *pb = tor_malloc_zero(sizeof(*pb));
+  pb->cfg = dcfg_new();
+  pb->items = pubsub_items_new();
+  return pb;
+}
+
+/**
+ * Release all storage held by a pubsub_builder_t.
+ *
+ * You'll (mostly) only want to call this function on an error case: if you're
+ * constructing a dispatch_t instead, you should call
+ * pubsub_builder_finalize() to consume the pubsub_builder_t.
+ */
+void
+pubsub_builder_free_(pubsub_builder_t *pb)
+{
+  if (pb == NULL)
+    return;
+  pubsub_items_free(pb->items);
+  dcfg_free(pb->cfg);
+  tor_free(pb);
+}
+
+/**
+ * Create and return a pubsub_connector_t for the subsystem with ID
+ * <b>subsys</b> to use in adding publications, subscriptions, and types to
+ * <b>builder</b>.
+ **/
+pubsub_connector_t *
+pubsub_connector_for_subsystem(pubsub_builder_t *builder,
+                               subsys_id_t subsys)
+{
+  tor_assert(builder);
+  ++builder->n_connectors;
+
+  pubsub_connector_t *con = tor_malloc_zero(sizeof(*con));
+
+  con->builder = builder;
+  con->subsys_id = subsys;
+
+  return con;
+}
+
+/**
+ * Release all storage held by a pubsub_connector_t.
+ **/
+void
+pubsub_connector_free_(pubsub_connector_t *con)
+{
+  if (!con)
+    return;
+
+  if (con->builder) {
+    --con->builder->n_connectors;
+    tor_assert(con->builder->n_connectors >= 0);
+  }
+  tor_free(con);
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>.
+ **/
+int
+pubsub_add_pub_(pubsub_connector_t *con,
+                pub_binding_t *out,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  memset(out, 0, sizeof(*out));
+  cfg->is_publish = true;
+
+  out->msg_template.sender = cfg->subsys = con->subsys_id;
+  out->msg_template.channel = cfg->channel = channel;
+  out->msg_template.msg = cfg->msg = msg;
+  out->msg_template.type = cfg->type = type;
+
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  /* We're grabbing a pointer to the pub_binding_t so we can tell it about
+   * the dispatcher later on.
+   */
+  cfg->pub_binding = out;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to add a request for being able to publish messages of type
+ * <b>msg</b> with auxiliary data of <b>type</b> on <b>channel</b>,
+ * passing them to the callback in <b>recv_fn</b>.
+ **/
+int
+pubsub_add_sub_(pubsub_connector_t *con,
+                recv_fn_t recv_fn,
+                channel_id_t channel,
+                message_id_t msg,
+                msg_type_id_t type,
+                unsigned flags,
+                const char *file,
+                unsigned line)
+{
+  pubsub_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+
+  cfg->is_publish = false;
+  cfg->subsys = con->subsys_id;
+  cfg->channel = channel;
+  cfg->msg = msg;
+  cfg->type = type;
+  cfg->flags = flags;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  cfg->recv_fn = recv_fn;
+
+  smartlist_add(con->builder->items->items, cfg);
+
+  if (dcfg_msg_set_type(con->builder->cfg, msg, type) < 0)
+    goto err;
+  if (dcfg_msg_set_chan(con->builder->cfg, msg, channel) < 0)
+    goto err;
+  if (! (flags & DISP_FLAG_STUB)) {
+    if (dcfg_add_recv(con->builder->cfg, msg, cfg->subsys, recv_fn) < 0)
+      goto err;
+  }
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Use <b>con</b> to define the functions to use for manipulating the type
+ * <b>type</b>.  Any function pointers left as NULL will be implemented as
+ * no-ops.
+ **/
+int
+pubsub_connector_register_type_(pubsub_connector_t *con,
+                                msg_type_id_t type,
+                                dispatch_typefns_t *fns,
+                                const char *file,
+                                unsigned line)
+{
+  pubsub_type_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  cfg->type = type;
+  memcpy(&cfg->fns, fns, sizeof(*fns));
+  cfg->subsys = con->subsys_id;
+  cfg->added_by_file = file;
+  cfg->added_by_line = line;
+
+  smartlist_add(con->builder->items->type_items, cfg);
+
+  if (dcfg_type_set_fns(con->builder->cfg, type, fns) < 0)
+    goto err;
+
+  return 0;
+ err:
+  ++con->builder->n_errors;
+  return -1;
+}
+
+/**
+ * Initialize the dispatch_ptr field in every relevant publish binding
+ * for <b>d</b>.
+ */
+static void
+pubsub_items_install_bindings(pubsub_items_t *items,
+                              dispatch_t *d)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      // XXXX we could skip this for STUB publishers, and for any publishers
+      // XXXX where all subscribers are STUB.
+      cfg->pub_binding->dispatch_ptr = d;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Remove the dispatch_ptr fields for all the relevant publish bindings
+ * in <b>items</b>.  The prevents subsequent dispatch_pub_() calls from
+ * sending messages to a dispatcher that has been freed.
+ **/
+void
+pubsub_items_clear_bindings(pubsub_items_t *items)
+{
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, cfg) {
+    if (cfg->pub_binding) {
+      cfg->pub_binding->dispatch_ptr = NULL;
+    }
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Create a new dispatcher as configured in a pubsub_builder_t.
+ *
+ * Consumes and frees its input.
+ **/
+dispatch_t *
+pubsub_builder_finalize(pubsub_builder_t *builder,
+                        pubsub_items_t **items_out)
+{
+  dispatch_t *dispatcher = NULL;
+  tor_assert_nonfatal(builder->n_connectors == 0);
+
+  if (pubsub_builder_check(builder) < 0)
+    goto err;
+
+  if (builder->n_errors) {
+    log_warn(LD_GENERAL, "At least one error occurred previously when "
+             "configuring the dispatcher.");
+    goto err;
+  }
+
+  dispatcher = dispatch_new(builder->cfg);
+
+  if (!dispatcher)
+    goto err;
+
+  pubsub_items_install_bindings(builder->items, dispatcher);
+  if (items_out) {
+    *items_out = builder->items;
+    builder->items = NULL; /* Prevent free */
+  }
+
+ err:
+  pubsub_builder_free(builder);
+  return dispatcher;
+}
diff --git a/src/lib/pubsub/pubsub_build.h b/src/lib/pubsub/pubsub_build.h
new file mode 100644
index 0000000000..2781b8251a
--- /dev/null
+++ b/src/lib/pubsub/pubsub_build.h
@@ -0,0 +1,97 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_build.h
+ * @brief Header used for constructing the OO publish-subscribe facility.
+ *
+ * (See pubsub.h for more general information on this API.)
+ **/
+
+#ifndef TOR_PUBSUB_BUILD_H
+#define TOR_PUBSUB_BUILD_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct dispatch_t;
+struct pubsub_connector_t;
+
+/**
+ * A "dispatch builder" is an incomplete dispatcher, used when
+ * registering messages.  It does not have the same integrity guarantees
+ * as a dispatcher.  It cannot actually handle messages itself: once all
+ * subsystems have registered, it is converted into a dispatch_t.
+ **/
+typedef struct pubsub_builder_t pubsub_builder_t;
+
+/**
+ * A "pubsub items" holds the configuration items used to configure a
+ * pubsub_builder.  After the builder is finalized, this field is extracted,
+ * and used later to tear down pointers that enable publishing.
+ **/
+typedef struct pubsub_items_t pubsub_items_t;
+
+/**
+ * Create a new pubsub_builder. This should only happen in the
+ * main-init code.
+ */
+pubsub_builder_t *pubsub_builder_new(void);
+
+/** DOCDOC */
+int pubsub_builder_check(pubsub_builder_t *);
+
+/**
+ * Free a pubsub builder.  This should only happen on error paths, where
+ * we have decided not to construct a dispatcher for some reason.
+ */
+#define pubsub_builder_free(db) \
+  FREE_AND_NULL(pubsub_builder_t, pubsub_builder_free_, (db))
+
+/** Internal implementation of pubsub_builder_free(). */
+void pubsub_builder_free_(pubsub_builder_t *);
+
+/**
+ * Create a pubsub connector that a single subsystem will use to
+ * register its messages.  The main-init code does this during susbsystem
+ * initialization.
+ */
+struct pubsub_connector_t *pubsub_connector_for_subsystem(pubsub_builder_t *,
+                                                          subsys_id_t);
+
+/**
+ * The main-init code does this after subsystem initialization.
+ */
+#define pubsub_connector_free(c) \
+  FREE_AND_NULL(struct pubsub_connector_t, pubsub_connector_free_, (c))
+
+void pubsub_connector_free_(struct pubsub_connector_t *);
+
+/**
+ * Constructs a dispatcher from a dispatch_builder, after checking that the
+ * invariances on the messages, channels, and connections have been
+ * respected.
+ *
+ * This should happen after every subsystem has initialized, and before
+ * entering the mainloop.
+ */
+struct dispatch_t *pubsub_builder_finalize(pubsub_builder_t *,
+                                           pubsub_items_t **items_out);
+
+/**
+ * Clear all pub_binding_t backpointers in <b>items</b>.
+ **/
+void pubsub_items_clear_bindings(pubsub_items_t *items);
+
+/**
+ * @copydoc pubsub_items_free_
+ *
+ * Additionally, set the pointer <b>cfg</b> to NULL.
+ **/
+#define pubsub_items_free(cfg) \
+  FREE_AND_NULL(pubsub_items_t, pubsub_items_free_, (cfg))
+void pubsub_items_free_(pubsub_items_t *cfg);
+
+#endif /* !defined(TOR_PUBSUB_BUILD_H) */
diff --git a/src/lib/pubsub/pubsub_builder_st.h b/src/lib/pubsub/pubsub_builder_st.h
new file mode 100644
index 0000000000..57de1240ee
--- /dev/null
+++ b/src/lib/pubsub/pubsub_builder_st.h
@@ -0,0 +1,161 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_builder_st.h
+ *
+ * @brief private structures used for configuring dispatchers and messages.
+ */
+
+#ifndef TOR_PUBSUB_BUILDER_ST_H
+#define TOR_PUBSUB_BUILDER_ST_H
+
+#ifdef PUBSUB_PRIVATE
+
+#include <stdbool.h>
+#include <stddef.h>
+
+struct dispatch_cfg_t;
+struct smartlist_t;
+struct pub_binding_t;
+
+/**
+ * Configuration for a single publication or subscription request.
+ *
+ * These can be stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ *
+ * There are various fields in this request describing the message; all of
+ * them must match other descriptions of the message, or a bug has occurred.
+ **/
+typedef struct pubsub_cfg_t {
+  /** True if this is a publishing request; false for a subscribing request. */
+  bool is_publish;
+  /** The system making this request. */
+  subsys_id_t subsys;
+  /** The channel on which the message is to be sent. */
+  channel_id_t channel;
+  /** The message ID to be sent or received. */
+  message_id_t msg;
+  /** The C type associated with the message. */
+  msg_type_id_t type;
+  /** One or more DISP_FLAGS_* items, combined with bitwise OR. */
+  unsigned flags;
+
+  /**
+   * Publishing only: a pub_binding object that will receive the binding for
+   * this request.  We will finish filling this in when the dispatcher is
+   * constructed, so that the subsystem can publish then and not before.
+   */
+  struct pub_binding_t *pub_binding;
+
+  /**
+   * Subscribing only: a function to receive message objects for this request.
+   */
+  recv_fn_t recv_fn;
+
+  /** The file from which this message was configured */
+  const char *added_by_file;
+  /** The line at which this message was configured */
+  unsigned added_by_line;
+} pubsub_cfg_t;
+
+/**
+ * Configuration request for a single C type.
+ *
+ * These are stored while the dispatcher is in use, but are only used for
+ * setup, teardown, and debugging.
+ **/
+typedef struct pubsub_type_cfg_t {
+  /**
+   * The identifier for this type.
+   */
+  msg_type_id_t type;
+  /**
+   * Functions to use when manipulating the type.
+   */
+  dispatch_typefns_t fns;
+
+  /** The subsystem that configured this type. */
+  subsys_id_t subsys;
+  /** The file from which this type was configured */
+  const char *added_by_file;
+  /** The line at which this type was configured */
+  unsigned added_by_line;
+} pubsub_type_cfg_t;
+
+/**
+ * The set of configuration requests for a dispatcher, as made by various
+ * subsystems.
+ **/
+struct pubsub_items_t {
+  /** List of pubsub_cfg_t. */
+  struct smartlist_t *items;
+  /** List of pubsub_type_cfg_t. */
+  struct smartlist_t *type_items;
+};
+
+/**
+ * Type used to construct a dispatcher.  We use this type to build up the
+ * configuration for a dispatcher, and then pass ownership of that
+ * configuration to the newly constructed dispatcher.
+ **/
+struct pubsub_builder_t {
+  /** Number of outstanding pubsub_connector_t objects pointing to this
+   * pubsub_builder_t. */
+  int n_connectors;
+  /** Number of errors encountered while constructing this object so far. */
+  int n_errors;
+  /** In-progress configuration that we're constructing, as a list of the
+   * requests that have been made. */
+  struct pubsub_items_t *items;
+  /** In-progress configuration that we're constructing, in a form that can
+   * be converted to a dispatch_t. */
+  struct dispatch_cfg_t *cfg;
+};
+
+/**
+ * Type given to a subsystem when adding connections to a pubsub_builder_t.
+ * We use this type to force each subsystem to get blamed for the
+ * publications, subscriptions, and types that it adds.
+ **/
+struct pubsub_connector_t {
+  /** The pubsub_builder that this connector refers to. */
+  struct pubsub_builder_t *builder;
+  /** The subsystem that has been given this connector. */
+  subsys_id_t subsys_id;
+};
+
+/**
+ * Helper structure used when constructing a dispatcher that sorts the
+ * pubsub_cfg_t objects in various ways.
+ **/
+typedef struct pubsub_adjmap_t {
+  /* XXXX The next three fields are currently constructed but not yet
+   * XXXX used. I believe we'll want them in the future, though. -nickm
+   */
+  /** Number of subsystems; length of the *_by_subsys arrays. */
+  size_t n_subsystems;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **pub_by_subsys;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * subsystem. */
+  struct smartlist_t **sub_by_subsys;
+
+  /** Number of message IDs; length of the *_by_msg arrays. */
+  size_t n_msgs;
+  /** Array of lists of publisher pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **pub_by_msg;
+  /** Array of lists of subscriber pubsub_cfg_t objects, indexed by
+   * message ID. */
+  struct smartlist_t **sub_by_msg;
+} pubsub_adjmap_t;
+
+#endif /* defined(PUBSUB_PRIVATE) */
+
+#endif /* !defined(TOR_PUBSUB_BUILDER_ST_H) */
diff --git a/src/lib/pubsub/pubsub_check.c b/src/lib/pubsub/pubsub_check.c
new file mode 100644
index 0000000000..dbcbb14746
--- /dev/null
+++ b/src/lib/pubsub/pubsub_check.c
@@ -0,0 +1,414 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_check.c
+ * @brief Enforce various requirements on a pubsub_builder.
+ **/
+
+/** @{ */
+#define PUBSUB_PRIVATE
+/** @} */
+
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_build.h"
+
+#include "lib/container/bitarray.h"
+#include "lib/container/smartlist.h"
+#include "lib/log/util_bug.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/compat_string.h"
+
+#include <string.h>
+
+static void pubsub_adjmap_add(pubsub_adjmap_t *map,
+                                const pubsub_cfg_t *item);
+
+/**
+ * Helper: contruct and return a new pubsub_adjacency_map from <b>cfg</b>.
+ * Return NULL on error.
+ **/
+static pubsub_adjmap_t *
+pubsub_build_adjacency_map(const pubsub_items_t *cfg)
+{
+  pubsub_adjmap_t *map = tor_malloc_zero(sizeof(*map));
+  const size_t n_subsystems = get_num_subsys_ids();
+  const size_t n_msgs = get_num_message_ids();
+
+  map->n_subsystems = n_subsystems;
+  map->n_msgs = n_msgs;
+
+  map->pub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->sub_by_subsys = tor_calloc(n_subsystems, sizeof(smartlist_t*));
+  map->pub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+  map->sub_by_msg = tor_calloc(n_msgs, sizeof(smartlist_t*));
+
+  SMARTLIST_FOREACH_BEGIN(cfg->items, const pubsub_cfg_t *, item) {
+    pubsub_adjmap_add(map, item);
+  } SMARTLIST_FOREACH_END(item);
+
+  return map;
+}
+
+/**
+ * Helper: add a single pubsub_cfg_t to an adjacency map.
+ **/
+static void
+pubsub_adjmap_add(pubsub_adjmap_t *map,
+                  const pubsub_cfg_t *item)
+{
+  smartlist_t **by_subsys;
+  smartlist_t **by_msg;
+
+  tor_assert(item->subsys < map->n_subsystems);
+  tor_assert(item->msg < map->n_msgs);
+
+  if (item->is_publish) {
+    by_subsys = &map->pub_by_subsys[item->subsys];
+    by_msg = &map->pub_by_msg[item->msg];
+  } else {
+    by_subsys = &map->sub_by_subsys[item->subsys];
+    by_msg = &map->sub_by_msg[item->msg];
+  }
+
+  if (! *by_subsys)
+    *by_subsys = smartlist_new();
+  if (! *by_msg)
+    *by_msg = smartlist_new();
+  smartlist_add(*by_subsys, (void*) item);
+  smartlist_add(*by_msg, (void *) item);
+}
+
+/**
+ * Release all storage held by m and set m to NULL.
+ **/
+#define pubsub_adjmap_free(m) \
+  FREE_AND_NULL(pubsub_adjmap_t, pubsub_adjmap_free_, m)
+
+/**
+ * Free every element of an <b>n</b>-element array of smartlists, then
+ * free the array itself.
+ **/
+static void
+pubsub_adjmap_free_helper(smartlist_t **lsts, size_t n)
+{
+  if (!lsts)
+    return;
+
+  for (unsigned i = 0; i < n; ++i) {
+    smartlist_free(lsts[i]);
+  }
+  tor_free(lsts);
+}
+
+/**
+ * Release all storage held by <b>map</b>.
+ **/
+static void
+pubsub_adjmap_free_(pubsub_adjmap_t *map)
+{
+  if (!map)
+    return;
+  pubsub_adjmap_free_helper(map->pub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->sub_by_subsys, map->n_subsystems);
+  pubsub_adjmap_free_helper(map->pub_by_msg, map->n_msgs);
+  pubsub_adjmap_free_helper(map->sub_by_msg, map->n_msgs);
+  tor_free(map);
+}
+
+/**
+ * Helper: return the length of <b>sl</b>, or 0 if sl is NULL.
+ **/
+static int
+smartlist_len_opt(const smartlist_t *sl)
+{
+  if (sl)
+    return smartlist_len(sl);
+  else
+    return 0;
+}
+
+/** Return a pointer to a statically allocated string encoding the
+ * dispatcher flags in <b>flags</b>. */
+static const char *
+format_flags(unsigned flags)
+{
+  static char buf[32];
+  buf[0] = 0;
+  if (flags & DISP_FLAG_EXCL) {
+    strlcat(buf, " EXCL", sizeof(buf));
+  }
+  if (flags & DISP_FLAG_STUB) {
+    strlcat(buf, " STUB", sizeof(buf));
+  }
+  return buf[0] ? buf+1 : buf;
+}
+
+/**
+ * Log a message containing a description of <b>cfg</b> at severity, prefixed
+ * by the string <b>prefix</b>.
+ */
+static void
+pubsub_cfg_dump(const pubsub_cfg_t *cfg, int severity, const char *prefix)
+{
+  tor_assert(prefix);
+
+  tor_log(severity, LD_MESG,
+          "%s%s %s: %s{%s} on %s (%s) <%u %u %u %u %x> [%s:%d]",
+          prefix,
+          get_subsys_id_name(cfg->subsys),
+          cfg->is_publish ? "PUB" : "SUB",
+          get_message_id_name(cfg->msg),
+          get_msg_type_id_name(cfg->type),
+          get_channel_id_name(cfg->channel),
+          format_flags(cfg->flags),
+          cfg->subsys, cfg->msg, cfg->type, cfg->channel, cfg->flags,
+          cfg->added_by_file, cfg->added_by_line);
+}
+
+/**
+ * Helper: fill a bitarray <b>out</b> with entries corresponding to the
+ * subsystems listed in <b>items</b>.
+ **/
+static void
+get_message_bitarray(const pubsub_adjmap_t *map,
+                     const smartlist_t *items,
+                     bitarray_t **out)
+{
+  *out = bitarray_init_zero((unsigned)map->n_subsystems);
+  if (! items)
+    return;
+
+  SMARTLIST_FOREACH_BEGIN(items, const pubsub_cfg_t *, cfg) {
+    bitarray_set(*out, cfg->subsys);
+  } SMARTLIST_FOREACH_END(cfg);
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists obey our local graph topology rules.
+ *
+ * (Right now this is just a matter of "each subsystem only
+ * publishes/subscribes once; no subsystem is a publisher and subscriber for
+ * the same message.")
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+static int
+lint_message_graph(const pubsub_adjmap_t *map,
+                   message_id_t msg,
+                   const smartlist_t *pub,
+                   const smartlist_t *sub)
+{
+  bitarray_t *published_by = NULL;
+  bitarray_t *subscribed_by = NULL;
+  bool ok = true;
+
+  get_message_bitarray(map, pub, &published_by);
+  get_message_bitarray(map, sub, &subscribed_by);
+
+  /* Check whether any subsystem is publishing and subscribing the same
+   * message. [??]
+   */
+  for (unsigned i = 0; i < map->n_subsystems; ++i) {
+    if (bitarray_is_set(published_by, i) &&
+        bitarray_is_set(subscribed_by, i)) {
+      log_warn(LD_MESG|LD_BUG,
+               "Message \"%s\" is published and subscribed by the same "
+               "subsystem \"%s\".",
+               get_message_id_name(msg),
+               get_subsys_id_name(i));
+      ok = false;
+    }
+  }
+
+  bitarray_free(published_by);
+  bitarray_free(subscribed_by);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Helper for lint_message: check that all the pubsub_cfg_t items in the two
+ * respective smartlists have compatible flags, channels, and types.
+ **/
+static int
+lint_message_consistency(message_id_t msg,
+                         const smartlist_t *pub,
+                         const smartlist_t *sub)
+{
+  if (!smartlist_len_opt(pub) && !smartlist_len_opt(sub))
+    return 0; // LCOV_EXCL_LINE -- this was already checked.
+
+  /* The 'all' list has the publishers and the subscribers. */
+  smartlist_t *all = smartlist_new();
+  if (pub)
+    smartlist_add_all(all, pub);
+  if (sub)
+    smartlist_add_all(all, sub);
+
+  const pubsub_cfg_t *item0 = smartlist_get(all, 0);
+
+  /* Indicates which subsystems we've found publishing/subscribing here. */
+  bool pub_excl = false, sub_excl = false, chan_same = true, type_same = true;
+
+  /* Simple message consistency properties across messages.
+   */
+  SMARTLIST_FOREACH_BEGIN(all, const pubsub_cfg_t *, cfg) {
+    chan_same &= (cfg->channel == item0->channel);
+    type_same &= (cfg->type == item0->type);
+    if (cfg->is_publish)
+      pub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+    else
+      sub_excl |= (cfg->flags & DISP_FLAG_EXCL) != 0;
+  } SMARTLIST_FOREACH_END(cfg);
+
+  bool ok = true;
+
+  if (! chan_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "channels.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (! type_same) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" is associated with multiple inconsistent "
+             "message types.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Enforce exclusive-ness for publishers and subscribers that have asked for
+   * it.
+   */
+  if (pub_excl && smartlist_len_opt(pub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple publishers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+  if (sub_excl && smartlist_len_opt(sub) > 1) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has multiple subscribers, but at least one is "
+             "marked as exclusive.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  smartlist_free(all);
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check whether there are any errors or inconsistencies for the message
+ * described by <b>msg</b> in <b>map</b>.  If there are problems, log about
+ * them, and return -1.  Otherwise return 0.
+ **/
+static int
+lint_message(const pubsub_adjmap_t *map, message_id_t msg)
+{
+  /* NOTE: Some of the checks in this function are maybe over-zealous, and we
+   * might not want to have them forever.  I've marked them with [?] below.
+   */
+  if (BUG(msg >= map->n_msgs))
+    return 0; // LCOV_EXCL_LINE
+
+  const smartlist_t *pub = map->pub_by_msg[msg];
+  const smartlist_t *sub = map->sub_by_msg[msg];
+
+  const size_t n_pub = smartlist_len_opt(pub);
+  const size_t n_sub = smartlist_len_opt(sub);
+
+  if (n_pub == 0 && n_sub == 0) {
+    log_info(LD_MESG, "Nobody is publishing or subscribing to message "
+             "\"%s\".",
+             get_message_id_name(msg));
+    return 0; // No publishers or subscribers: nothing to do.
+  }
+  /* We'll set this to false if there are any problems. */
+  bool ok = true;
+
+  /* First make sure that if there are publishers, there are subscribers. */
+  if (n_pub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has subscribers, but no publishers.",
+             get_message_id_name(msg));
+    ok = false;
+  } else if (n_sub == 0) {
+    log_warn(LD_MESG|LD_BUG,
+             "Message \"%s\" has publishers, but no subscribers.",
+             get_message_id_name(msg));
+    ok = false;
+  }
+
+  /* Check the message graph topology. */
+  if (lint_message_graph(map, msg, pub, sub) < 0)
+    ok = false;
+
+  /* Check whether the messages have the same fields set on them. */
+  if (lint_message_consistency(msg, pub, sub) < 0)
+    ok = false;
+
+  if (!ok) {
+    /* There was a problem -- let's log all the publishers and subscribers on
+     * this message */
+    if (pub) {
+      SMARTLIST_FOREACH(pub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+    if (sub) {
+      SMARTLIST_FOREACH(sub, pubsub_cfg_t *, cfg,
+                        pubsub_cfg_dump(cfg, LOG_WARN, "   "));
+    }
+  }
+
+  return ok ? 0 : -1;
+}
+
+/**
+ * Check all the messages in <b>map</b> for consistency.  Return 0 on success,
+ * -1 on problems.
+ **/
+static int
+pubsub_adjmap_check(const pubsub_adjmap_t *map)
+{
+  bool all_ok = true;
+  for (unsigned i = 0; i < map->n_msgs; ++i) {
+    if (lint_message(map, i) < 0) {
+      all_ok = false;
+    }
+  }
+  return all_ok ? 0 : -1;
+}
+
+/**
+ * Check builder for consistency and various constraints. Return 0 on success,
+ * -1 on failure.
+ **/
+int
+pubsub_builder_check(pubsub_builder_t *builder)
+{
+  pubsub_adjmap_t *map = pubsub_build_adjacency_map(builder->items);
+  int rv = -1;
+
+  if (!map)
+    goto err; // should be impossible
+
+  if (pubsub_adjmap_check(map) < 0)
+    goto err;
+
+  rv = 0;
+ err:
+  pubsub_adjmap_free(map);
+  return rv;
+}
diff --git a/src/lib/pubsub/pubsub_connect.h b/src/lib/pubsub/pubsub_connect.h
new file mode 100644
index 0000000000..b0d6ae7e92
--- /dev/null
+++ b/src/lib/pubsub/pubsub_connect.h
@@ -0,0 +1,54 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_connect.h
+ * @brief Header for functions that add relationships to a pubsub builder.
+ *
+ * These functions are used by modules that need to add publication and
+ * subscription requests.  Most users will want to call these functions
+ * indirectly, via the macros in pubsub_macros.h.
+ **/
+
+#ifndef TOR_PUBSUB_CONNECT_H
+#define TOR_PUBSUB_CONNECT_H
+
+#include "lib/dispatch/msgtypes.h"
+
+struct pub_binding_t;
+/**
+ * A "dispatch connector" is a view of the dispatcher that a subsystem
+ * uses while initializing itself.  It is specific to the subsystem, and
+ * ensures that each subsystem doesn't need to identify itself
+ * repeatedly while registering its messages.
+ **/
+typedef struct pubsub_connector_t pubsub_connector_t;
+
+int pubsub_add_pub_(struct pubsub_connector_t *con,
+                    struct pub_binding_t *out,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_add_sub_(struct pubsub_connector_t *con,
+                    recv_fn_t recv_fn,
+                    channel_id_t channel,
+                    message_id_t msg,
+                    msg_type_id_t type,
+                    unsigned flags,
+                    const char *file,
+                    unsigned line);
+
+int pubsub_connector_register_type_(struct pubsub_connector_t *,
+                                    msg_type_id_t,
+                                    dispatch_typefns_t *,
+                                    const char *file,
+                                    unsigned line);
+
+#endif /* !defined(TOR_PUBSUB_CONNECT_H) */
diff --git a/src/lib/pubsub/pubsub_flags.h b/src/lib/pubsub/pubsub_flags.h
new file mode 100644
index 0000000000..9912c1ae89
--- /dev/null
+++ b/src/lib/pubsub/pubsub_flags.h
@@ -0,0 +1,32 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_flags.h
+ * @brief Flags that can be set on publish/subscribe messages.
+ **/
+
+#ifndef TOR_PUBSUB_FLAGS_H
+#define TOR_PUBSUB_FLAGS_H
+
+/**
+ * Flag for registering a message: declare that no other module is allowed to
+ * publish this message if we are publishing it, or subscribe to it if we are
+ * subscribing to it.
+ */
+#define DISP_FLAG_EXCL (1u<<0)
+
+/**
+ * Flag for registering a message: declare that this message is a stub, and we
+ * will not actually publish/subscribe it, but that the dispatcher should
+ * treat us as if we did when typechecking.
+ *
+ * We use this so that messages aren't treated as "dangling" if they are
+ * potentially used by some other build of Tor.
+ */
+#define DISP_FLAG_STUB (1u<<1)
+
+#endif /* !defined(TOR_PUBSUB_FLAGS_H) */
diff --git a/src/lib/pubsub/pubsub_macros.h b/src/lib/pubsub/pubsub_macros.h
new file mode 100644
index 0000000000..e5ffbe501a
--- /dev/null
+++ b/src/lib/pubsub/pubsub_macros.h
@@ -0,0 +1,373 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file pubsub_macros.h
+ * \brief Macros to help with the publish/subscribe dispatch API.
+ *
+ * The dispatch API allows different subsystems of Tor to communicate with
+ * another asynchronously via a shared "message" system.  Some subsystems
+ * declare that they publish a given message, and others declare that they
+ * subscribe to it.  Both subsystems depend on the message, but not upon one
+ * another.
+ *
+ * To declare a message, use DECLARE_MESSAGE() (for messages that take their
+ * data as a pointer) or DECLARE_MESSAGE_INT() (for messages that take their
+ * data as an integer.  For example, you might say
+ *
+ *     DECLARE_MESSAGE(new_circuit, circ, circuit_handle_t *);
+ * or
+ *     DECLARE_MESSAGE_INT(shutdown_requested, boolean, bool);
+ *
+ * Every message has a unique name, a "type name" that the dispatch system
+ * uses to manage associated data, and a C type name.  You'll want to put
+ * these declarations in a header, to be included by all publishers and all
+ * subscribers.
+ *
+ * When a subsystem wants to publish a message, it uses DECLARE_PUBLISH() at
+ * file scope to create necessary static functions.  Then, in its subsystem
+ * initialization (in the "bind to dispatcher" callback) (TODO: name this
+ * properly!), it calls DISPATCH_ADD_PUB() to tell the dispatcher about its
+ * intent to publish.  When it actually wants to publish, it uses the
+ * PUBLISH() macro.  For example:
+ *
+ *     // At file scope
+ *     DECLARE_PUBLISH(shutdown_requested);
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_PUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ *     // somewhere in a function
+ *        {
+ *            PUBLISH(shutdown_requested, true);
+ *        }
+ *
+ * When a subsystem wants to subscribe to a message, it uses
+ * DECLARE_SUBSCRIBE() at file scope to declare static functions.  It must
+ * declare a hook function that receives the message type.  Then, in its "bind
+ * to dispatcher" function, it calls DISPATCHER_ADD_SUB() to tell the
+ * dispatcher about its intent to subscribe.  When another module publishes
+ * the message, the dispatcher will call the provided hook function.
+ *
+ *     // At file scope.  The first argument is the message that you're
+ *     // subscribing to; the second argument is the hook function to declare.
+ *     DECLARE_SUBSCRIBE(shutdown_requested, on_shutdown_req_cb);
+ *
+ *     // You need to declare this function.
+ *     static void on_shutdown_req_cb(const msg_t *msg,
+ *                                    bool value)
+ *     {
+ *         // (do something here.)
+ *     }
+ *
+ *     static void bind_to_dispatcher(pubsub_connector_t *con)
+ *     {
+ *         DISPATCH_ADD_SUB(con, mainchannel, shutdown_requested);
+ *     }
+ *
+ * Where did these types come from?  Somewhere in the code, you need to call
+ * DISPATCH_REGISTER_TYPE() to make sure that the dispatcher can manage the
+ * message auxiliary data.  It associates a vtbl-like structure with the
+ * type name, so that the dispatcher knows how to manipulate the type you're
+ * giving it.
+ *
+ * For example, the "boolean" type we're using above could be defined as:
+ *
+ *    static char *boolean_fmt(msg_aux_data_t d)
+ *    {
+ *        // This is used for debugging and dumping messages.
+ *        if (d.u64)
+ *            return tor_strdup("true");
+ *        else
+ *            return tor_strdup("false");
+ *    }
+ *
+ *    static void boolean_free(msg_aux_data_t d)
+ *    {
+ *        // We don't actually need to do anything to free a boolean.
+ *        // We could use "NULL" instead of this function, but I'm including
+ *        // it as an example.
+ *    }
+ *
+ *    static void bind_to_dispatcher(pubsub_connector_t *con)
+ *    {
+ *        dispatch_typefns_t boolean_fns = {
+ *            .fmt_fn = boolean_fmt,
+ *            .free_fn = boolean_free,
+ *        };
+ *        DISPATCH_REGISTER_TYPE(con, boolean, &boolean_fns);
+ *    }
+ *
+ *
+ *
+ * So, how does this all work?  (You can stop reading here, unless you're
+ * debugging something.)
+ *
+ * When you declare a message in a header with DECLARE_MESSAGE() or
+ * DECLARE_MESSAGE_INT(), it creates five things:
+ *
+ *    * two typedefs for the message argument (constant and non-constant
+ *      variants).
+ *    * a constant string to hold the declared message type name
+ *    * two inline functions, to coerce the message argument type to and from
+ *      a "msg_aux_data_t" union.
+ *
+ * All of these declarations have names based on the message name.
+ *
+ * Later, when you say DECLARE_PUBLISH() or DECLARE_SUBSCRIBE(), we use the
+ * elements defined by DECLARE_MESSAGE() to make sure that the publish
+ * function takes the correct argument type, and that the subscription hook is
+ * declared with the right argument type.
+ **/
+
+#ifndef TOR_DISPATCH_MSG_H
+#define TOR_DISPATCH_MSG_H
+
+#include "lib/cc/compat_compiler.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+/* Implemenation notes:
+ *
+ * For a messagename "foo", the DECLARE_MESSAGE*() macros must declare:
+ *
+ * msg_arg_type__foo -- a typedef for the argument type of the foo message.
+ * msg_arg_consttype__foo -- a typedef for the const argument type of the
+ *     foo message.
+ * msg_arg_name__foo[] -- a static string constant holding the unique
+ *      identifier for the type of the foo message.
+ * msg_arg_get__foo() -- an inline function taking a msg_aux_data_t and
+ *      returning the C data type.
+ * msg_arg_set__foo() -- an inline function taking a msg_aux_data_t and
+ *      the C type, setting the msg_aux_data_t to hold the C type.
+ *
+ * For a messagename "foo", the DECLARE_PUBLISH() macro must declare:
+ *
+ * pub_binding__foo -- A static pub_binding_t object used to send messages
+ *      from this module.
+ * publish_fn__foo -- A function taking an argument of the appropriate
+ *      C type, to be invoked by PUBLISH().
+ *
+ * For a messagename "foo", the DECLARE_SUBSCRIBE() macro must declare:
+ *
+ * hookfn -- A user-provided function name, with the correct signature.
+ * recv_fn__foo -- A wrapper callback that takes a msg_t *, and calls
+ *      hookfn with the appropriate arguments.
+ */
+
+/** Macro to declare common elements shared by DECLARE_MESSAGE and
+ * DECLARE_MESSAGE_INT.  Don't call this directly.
+ *
+ * Note that the "msg_arg_name" string constant is defined in each
+ * translation unit.  This might be undesirable; we can tweak it in the
+ * future if need be.
+ */
+#define DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)         \
+  typedef c_type msg_arg_type__ ##messagename;                          \
+  typedef const c_type msg_arg_consttype__ ##messagename;               \
+  ATTR_UNUSED static const char msg_arg_name__ ##messagename[] = # typename;
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking a pointer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.
+ * It needs to be defined somewhere in Tor, using
+ * "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_ptr_type" is a C pointer type (like "char *" or "struct foo *").
+ * The "*" needs to be included.
+ */
+#define DECLARE_MESSAGE(messagename, typename, c_ptr_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_ptr_type)           \
+  ATTR_UNUSED static inline c_ptr_type                                  \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return m.ptr;                                                       \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_ptr_type v)          \
+  {                                                                     \
+    m->ptr = v;                                                         \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro in a header to declare the existence of a given message,
+ * taking an integer as auxiliary data.
+ *
+ * "messagename" is a unique identifier for the message; it must be a valid
+ * C identifier.
+ *
+ * "typename" is a unique identifier for the type of the auxiliary data.  It
+ * needs to be defined somewhere in Tor, using "DISPATCH_REGISTER_TYPE."
+ *
+ * "c_type" is a C integer type, like "int" or "bool".  It needs to fit inside
+ * a uint64_t.
+ */
+#define DECLARE_MESSAGE_INT(messagename, typename, c_type)              \
+  DECLARE_MESSAGE_COMMON__(messagename, typename, c_type)               \
+  ATTR_UNUSED static inline c_type                                      \
+  msg_arg_get__ ##messagename(msg_aux_data_t m)                         \
+  {                                                                     \
+    return (c_type)m.u64;                                               \
+  }                                                                     \
+  ATTR_UNUSED static inline void                                        \
+  msg_arg_set__ ##messagename(msg_aux_data_t *m, c_type v)              \
+  {                                                                     \
+    m->u64 = (uint64_t)v;                                               \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C module declare that we'll be publishing a given
+ * message type from within this module.
+ *
+ * It creates necessary functions and wrappers to publish a message whose
+ * unique identifier is "messagename".
+ *
+ * Before you use this, you need to include the header where DECLARE_MESSAGE*()
+ * was used for this message.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_PUBLISH(messagename)                                    \
+  static pub_binding_t pub_binding__ ##messagename;                     \
+  static void                                                           \
+  publish_fn__ ##messagename(msg_arg_type__ ##messagename arg)          \
+  {                                                                     \
+    msg_aux_data_t data;                                                \
+    msg_arg_set__ ##messagename(&data, arg);                            \
+    pubsub_pub_(&pub_binding__ ##messagename, data);                    \
+  }                                                                     \
+  EAT_SEMICOLON
+
+/**
+ * Use this macro inside a C file to declare that we're subscribing to a
+ * given message and associating it with a given "hook function".  It
+ * declares the hook function static, and helps with strong typing.
+ *
+ * Before you use this, you need to include the header where
+ * DECLARE_MESSAGE*() was used for the message whose unique identifier is
+ * "messagename".
+ *
+ * You will need to define a function with the name that you provide for
+ * "hookfn".  The type of this function will be:
+ *     static void hookfn(const msg_t *, const c_type)
+ * where c_type is the c type that you declared in the header.
+ *
+ * You can only use this once per message in each subsystem.
+ */
+#define DECLARE_SUBSCRIBE(messagename, hookfn) \
+  static void hookfn(const msg_t *,                             \
+                     const msg_arg_consttype__ ##messagename);  \
+  static void recv_fn__ ## messagename(const msg_t *m)          \
+  {                                                             \
+    msg_arg_type__ ## messagename arg;                          \
+    arg = msg_arg_get__ ##messagename(m->aux_data__);           \
+    hookfn(m, arg);                                             \
+  }                                                             \
+  EAT_SEMICOLON
+
+/**
+ * Add a fake use of the publish function for 'messagename', so that
+ * the compiler does not call it unused.
+ */
+#define DISPATCH__FAKE_USE_OF_PUBFN_(messagename)                       \
+  ( 0 ? (publish_fn__ ##messagename((msg_arg_type__##messagename)0), 1) \
+    : 1)
+
+/**
+ * This macro is for internal use.  It backs DISPATCH_ADD_PUB*()
+ */
+#define DISPATCH_ADD_PUB_(connector, channel, messagename, flags)       \
+  (                                                                     \
+    DISPATCH__FAKE_USE_OF_PUBFN_(messagename),                          \
+    pubsub_add_pub_((connector),                                        \
+                      &pub_binding__ ##messagename,                     \
+                      get_channel_id(# channel),                        \
+                    get_message_id(# messagename),                      \
+                      get_msg_type_id(msg_arg_name__ ## messagename),   \
+                      (flags),                                          \
+                      __FILE__,                                         \
+                      __LINE__)                                         \
+    )
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB(connector, channel, messagename)       \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, 0)
+
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * publish a given message type, and that no other subsystem is allowed to.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_PUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_PUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/**
+ * This macro is for internal use. It backs DISPATCH_ADD_SUB*()
+ */
+#define DISPATCH_ADD_SUB_(connector, channel, messagename, flags)       \
+  pubsub_add_sub_((connector),                                          \
+                    recv_fn__ ##messagename,                            \
+                    get_channel_id(#channel),                           \
+                    get_message_id(# messagename),                      \
+                    get_msg_type_id(msg_arg_name__ ##messagename),      \
+                    (flags),                                            \
+                    __FILE__,                                           \
+                    __LINE__)
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB(connector, channel, messagename)       \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, 0)
+/**
+ * Use a given connector and channel name to declare that this subsystem will
+ * receive a given message type, and that no other subsystem is allowed to do
+ * so.
+ *
+ * Call this macro from within the add_subscriptions() function of a module.
+ */
+#define DISPATCH_ADD_SUB_EXCL(connector, channel, messagename)  \
+    DISPATCH_ADD_SUB_(connector, channel, messagename, DISP_FLAG_EXCL)
+
+/**
+ * Publish a given message with a given argument.  (Takes ownership of the
+ * argument if it is a pointer.)
+ */
+#define PUBLISH(messagename, arg)               \
+  publish_fn__ ##messagename(arg)
+
+/**
+ * Use a given connector to declare that the functions to be used to manipuate
+ * a certain C type.
+ **/
+#define DISPATCH_REGISTER_TYPE(con, type, fns)                  \
+  pubsub_connector_register_type_((con),                        \
+                                  get_msg_type_id(#type),       \
+                                  (fns),                        \
+                                  __FILE__,                     \
+                                  __LINE__)
+
+#endif /* !defined(TOR_DISPATCH_MSG_H) */
diff --git a/src/lib/pubsub/pubsub_publish.c b/src/lib/pubsub/pubsub_publish.c
new file mode 100644
index 0000000000..84c7dae02c
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.c
@@ -0,0 +1,72 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_publish.c
+ * @brief Header for functions to publish using a pub_binding_t.
+ **/
+
+#define PUBSUB_PRIVATE
+#define DISPATCH_PRIVATE
+#include "orconfig.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_st.h"
+
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/malloc/malloc.h"
+#include "lib/log/util_bug.h"
+
+#include <string.h>
+
+/**
+ * Publish a message from the publication binding <b>pub</b> using the
+ * auxiliary data <b>auxdata</b>.
+ *
+ * Return 0 on success, -1 on failure.
+ **/
+int
+pubsub_pub_(const pub_binding_t *pub, msg_aux_data_t auxdata)
+{
+  dispatch_t *d = pub->dispatch_ptr;
+  if (BUG(! d)) {
+    /* Tried to publish a message before the dispatcher was configured. */
+    /* (Without a dispatcher, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.type >= d->n_types)) {
+    /* The type associated with this message is not known to the dispatcher. */
+    /* (Without a correct type, we don't know how to free auxdata.) */
+    return -1;
+  }
+
+  if (BUG(pub->msg_template.msg >= d->n_msgs) ||
+      BUG(pub->msg_template.channel >= d->n_queues)) {
+    /* The message ID or channel ID was out of bounds. */
+    // LCOV_EXCL_START
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return -1;
+    // LCOV_EXCL_STOP
+  }
+
+  if (! d->table[pub->msg_template.msg]) {
+    /* Fast path: nobody wants this data. */
+
+    // XXXX Faster path: we could store this in the pub_binding_t.
+    d->typefns[pub->msg_template.type].free_fn(auxdata);
+    return 0;
+  }
+
+  /* Construct the message object */
+  msg_t *m = tor_malloc(sizeof(msg_t));
+  memcpy(m, &pub->msg_template, sizeof(msg_t));
+  m->aux_data__ = auxdata;
+
+  return dispatch_send_msg_unchecked(d, m);
+}
diff --git a/src/lib/pubsub/pubsub_publish.h b/src/lib/pubsub/pubsub_publish.h
new file mode 100644
index 0000000000..d9d6fa9ba5
--- /dev/null
+++ b/src/lib/pubsub/pubsub_publish.h
@@ -0,0 +1,20 @@
+/* Copyright (c) 2001, Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file pubsub_publish.h
+ * @brief Header for pubsub_publish.c
+ **/
+
+#ifndef TOR_PUBSUB_PUBLISH_H
+#define TOR_PUBSUB_PUBLISH_H
+
+#include "lib/dispatch/msgtypes.h"
+struct pub_binding_t;
+
+int pubsub_pub_(const struct pub_binding_t *pub, msg_aux_data_t auxdata);
+
+#endif /* !defined(TOR_PUBSUB_PUBLISH_H) */
diff --git a/src/lib/sandbox/.may_include b/src/lib/sandbox/.may_include
index 84906dfb3d..853dae7880 100644
--- a/src/lib/sandbox/.may_include
+++ b/src/lib/sandbox/.may_include
@@ -5,11 +5,10 @@ lib/container/*.h
 lib/err/*.h
 lib/log/*.h
 lib/malloc/*.h
-lib/net/*.h
 lib/sandbox/*.h
 lib/sandbox/*.inc
 lib/string/*.h
 
-ht.h
-siphash.h
-tor_queue.h
+ext/ht.h
+ext/siphash.h
+ext/tor_queue.h
diff --git a/src/lib/sandbox/include.am b/src/lib/sandbox/include.am
index adfda6bde5..e81f14b55f 100644
--- a/src/lib/sandbox/include.am
+++ b/src/lib/sandbox/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-sandbox-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_sandbox_a_SOURCES =			\
 	src/lib/sandbox/sandbox.c
 
@@ -13,6 +14,7 @@ src_lib_libtor_sandbox_testing_a_SOURCES = \
 src_lib_libtor_sandbox_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_sandbox_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/sandbox/linux_syscalls.inc		\
 	src/lib/sandbox/sandbox.h
diff --git a/src/lib/sandbox/lib_sandbox.md b/src/lib/sandbox/lib_sandbox.md
new file mode 100644
index 0000000000..dd168c9b13
--- /dev/null
+++ b/src/lib/sandbox/lib_sandbox.md
@@ -0,0 +1,15 @@
+@dir /lib/sandbox
+@brief lib/sandbox: Linux seccomp2-based sandbox.
+
+This module uses Linux's seccomp2 facility via the
+[`libseccomp` library](https://github.com/seccomp/libseccomp), to restrict
+the set of system calls that Tor is allowed to invoke while it is running.
+
+Because there are many libc versions that invoke different system calls, and
+because handling strings is quite complex, this module is more complex and
+less portable than it needs to be.
+
+A better architecture would put the responsibility for invoking tricky system
+calls (like open()) in another, less restricted process, and give that
+process responsibility for enforcing our sandbox rules.
+
diff --git a/src/lib/sandbox/sandbox.c b/src/lib/sandbox/sandbox.c
index 8f577b0660..d4f0da8397 100644
--- a/src/lib/sandbox/sandbox.c
+++ b/src/lib/sandbox/sandbox.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -38,13 +38,12 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "lib/cc/torint.h"
-#include "lib/net/resolve.h"
 #include "lib/malloc/malloc.h"
 #include "lib/string/scanf.h"
 
-#include "tor_queue.h"
-#include "ht.h"
-#include "siphash.h"
+#include "ext/tor_queue.h"
+#include "ext/ht.h"
+#include "ext/siphash.h"
 
 #define DEBUGGING_CLOSE
 
@@ -83,7 +82,7 @@
 #if defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
   defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION)
 #define USE_BACKTRACE
-#define EXPOSE_CLEAN_BACKTRACE
+#define BACKTRACE_PRIVATE
 #include "lib/err/backtrace.h"
 #endif /* defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && ... */
 
@@ -118,6 +117,10 @@
 
 #endif /* defined(__i386__) || ... */
 
+#ifdef M_SYSCALL
+#define SYSCALL_NAME_DEBUGGING
+#endif
+
 /**Determines if at least one sandbox is active.*/
 static int sandbox_active = 0;
 /** Holds the parameter list configuration for the sandbox.*/
@@ -134,6 +137,10 @@ static sandbox_cfg_t *filter_dynamic = NULL;
  * the high bits of the value might get masked out improperly. */
 #define SCMP_CMP_MASKED(a,b,c) \
   SCMP_CMP4((a), SCMP_CMP_MASKED_EQ, ~(scmp_datum_t)(b), (c))
+/* For negative constants, the rule to add depends on the glibc version. */
+#define SCMP_CMP_NEG(a,op,b) (libc_negative_constant_needs_cast() ? \
+                              (SCMP_CMP((a), (op), (unsigned int)(b))) : \
+                              (SCMP_CMP_STR((a), (op), (b))))
 
 /** Variable used for storing all syscall numbers that will be allowed with the
  * stage 1 general Tor sandbox.
@@ -144,6 +151,7 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(clock_gettime),
     SCMP_SYS(close),
     SCMP_SYS(clone),
+    SCMP_SYS(dup),
     SCMP_SYS(epoll_create),
     SCMP_SYS(epoll_wait),
 #ifdef __NR_epoll_pwait
@@ -166,6 +174,7 @@ static int filter_nopar_gen[] = {
 #ifdef __NR_fstat64
     SCMP_SYS(fstat64),
 #endif
+    SCMP_SYS(fsync),
     SCMP_SYS(futex),
     SCMP_SYS(getdents),
     SCMP_SYS(getdents64),
@@ -265,13 +274,27 @@ static int filter_nopar_gen[] = {
     SCMP_SYS(listen),
     SCMP_SYS(connect),
     SCMP_SYS(getsockname),
+#ifdef ENABLE_NSS
+#ifdef __NR_getpeername
+    SCMP_SYS(getpeername),
+#endif
+#endif
     SCMP_SYS(recvmsg),
     SCMP_SYS(recvfrom),
     SCMP_SYS(sendto),
     SCMP_SYS(unlink),
+#ifdef __NR_unlinkat
+    SCMP_SYS(unlinkat),
+#endif
     SCMP_SYS(poll)
 };
 
+/* opendir is not a syscall but it will use either open or openat. We do not
+ * want the decision to allow open/openat to be the callers reponsability, so
+ * we create a phony syscall number for opendir and sb_opendir will choose the
+ * correct syscall. */
+#define PHONY_OPENDIR_SYSCALL -2
+
 /* These macros help avoid the error where the number of filters we add on a
  * single rule don't match the arg_cnt param. */
 #define seccomp_rule_add_0(ctx,act,call) \
@@ -295,6 +318,7 @@ sb_rt_sigaction(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   unsigned i;
   int rc;
   int param[] = { SIGINT, SIGTERM, SIGPIPE, SIGUSR1, SIGUSR2, SIGHUP, SIGCHLD,
+                  SIGSEGV, SIGILL, SIGFPE, SIGBUS, SIGSYS, SIGIO,
 #ifdef SIGXFSZ
       SIGXFSZ
 #endif
@@ -424,31 +448,59 @@ sb_mmap2(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
 #endif
 #endif
 
-/* Return true if we think we're running with a libc that always uses
- * openat on linux. */
+/* Return true the libc version is greater or equal than
+ * <b>major</b>.<b>minor</b>. Returns false otherwise. */
 static int
-libc_uses_openat_for_everything(void)
+is_libc_at_least(int major, int minor)
 {
 #ifdef CHECK_LIBC_VERSION
   const char *version = gnu_get_libc_version();
   if (version == NULL)
     return 0;
 
-  int major = -1;
-  int minor = -1;
+  int libc_major = -1;
+  int libc_minor = -1;
 
-  tor_sscanf(version, "%d.%d", &major, &minor);
-  if (major >= 3)
+  tor_sscanf(version, "%d.%d", &libc_major, &libc_minor);
+  if (libc_major > major)
     return 1;
-  else if (major == 2 && minor >= 26)
+  else if (libc_major == major && libc_minor >= minor)
     return 1;
   else
     return 0;
-#else /* !(defined(CHECK_LIBC_VERSION)) */
+#else /* !defined(CHECK_LIBC_VERSION) */
+  (void)major;
+  (void)minor;
   return 0;
 #endif /* defined(CHECK_LIBC_VERSION) */
 }
 
+/* Return true if we think we're running with a libc that uses openat for the
+ * open function on linux. */
+static int
+libc_uses_openat_for_open(void)
+{
+  return is_libc_at_least(2, 26);
+}
+
+/* Return true if we think we're running with a libc that uses openat for the
+ * opendir function on linux. */
+static int
+libc_uses_openat_for_opendir(void)
+{
+  // libc 2.27 and above or between 2.15 (inclusive) and 2.22 (exclusive)
+  return is_libc_at_least(2, 27) ||
+         (is_libc_at_least(2, 15) && !is_libc_at_least(2, 22));
+}
+
+/* Return true if we think we're running with a libc that needs to cast
+ * negative arguments like AT_FDCWD for seccomp rules. */
+static int
+libc_negative_constant_needs_cast(void)
+{
+  return is_libc_at_least(2, 27);
+}
+
 /** Allow a single file to be opened.  If <b>use_openat</b> is true,
  * we're using a libc that remaps all the opens into openats. */
 static int
@@ -456,7 +508,7 @@ allow_file_open(scmp_filter_ctx ctx, int use_openat, const char *file)
 {
   if (use_openat) {
     return seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat),
-                              SCMP_CMP(0, SCMP_CMP_EQ, (unsigned int)AT_FDCWD),
+                              SCMP_CMP_NEG(0, SCMP_CMP_EQ, AT_FDCWD),
                               SCMP_CMP_STR(1, SCMP_CMP_EQ, file));
   } else {
     return seccomp_rule_add_1(ctx, SCMP_ACT_ALLOW, SCMP_SYS(open),
@@ -474,7 +526,7 @@ sb_open(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   int rc;
   sandbox_cfg_t *elem = NULL;
 
-  int use_openat = libc_uses_openat_for_everything();
+  int use_openat = libc_uses_openat_for_open();
 
   // for each dynamic parameter filters
   for (elem = filter; elem != NULL; elem = elem->next) {
@@ -592,7 +644,7 @@ sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
     if (param != NULL && param->prot == 1 && param->syscall
         == SCMP_SYS(openat)) {
       rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(openat),
-          SCMP_CMP(0, SCMP_CMP_EQ, AT_FDCWD),
+          SCMP_CMP_NEG(0, SCMP_CMP_EQ, AT_FDCWD),
           SCMP_CMP_STR(1, SCMP_CMP_EQ, param->value),
           SCMP_CMP(2, SCMP_CMP_EQ, O_RDONLY|O_NONBLOCK|O_LARGEFILE|O_DIRECTORY|
               O_CLOEXEC));
@@ -607,6 +659,30 @@ sb_openat(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   return 0;
 }
 
+static int
+sb_opendir(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
+{
+  int rc;
+  sandbox_cfg_t *elem = NULL;
+
+  // for each dynamic parameter filters
+  for (elem = filter; elem != NULL; elem = elem->next) {
+    smp_param_t *param = elem->param;
+
+    if (param != NULL && param->prot == 1 && param->syscall
+        == PHONY_OPENDIR_SYSCALL) {
+      rc = allow_file_open(ctx, libc_uses_openat_for_opendir(), param->value);
+      if (rc != 0) {
+        log_err(LD_BUG,"(Sandbox) failed to add openat syscall, received "
+            "libseccomp error %d", rc);
+        return rc;
+      }
+    }
+  }
+
+  return 0;
+}
+
 /**
  * Function responsible for setting up the socket syscall for
  * the seccomp filter sandbox.
@@ -647,6 +723,15 @@ sb_socket(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
     }
   }
 
+#ifdef ENABLE_NSS
+  rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket),
+    SCMP_CMP(0, SCMP_CMP_EQ, PF_INET),
+    SCMP_CMP(1, SCMP_CMP_EQ, SOCK_STREAM),
+    SCMP_CMP(2, SCMP_CMP_EQ, IPPROTO_IP));
+  if (rc)
+    return rc;
+#endif
+
   rc = seccomp_rule_add_3(ctx, SCMP_ACT_ALLOW, SCMP_SYS(socket),
       SCMP_CMP(0, SCMP_CMP_EQ, PF_UNIX),
       SCMP_CMP_MASKED(1, SOCK_CLOEXEC|SOCK_NONBLOCK, SOCK_STREAM),
@@ -798,6 +883,12 @@ sb_getsockopt(scmp_filter_ctx ctx, sandbox_cfg_t *filter)
   if (rc)
     return rc;
 
+  rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
+      SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
+      SCMP_CMP(2, SCMP_CMP_EQ, SO_ACCEPTCONN));
+  if (rc)
+    return rc;
+
 #ifdef HAVE_SYSTEMD
   rc = seccomp_rule_add_2(ctx, SCMP_ACT_ALLOW, SCMP_SYS(getsockopt),
       SCMP_CMP(1, SCMP_CMP_EQ, SOL_SOCKET),
@@ -1106,6 +1197,7 @@ static sandbox_filter_func_t filter_func[] = {
     sb_chmod,
     sb_open,
     sb_openat,
+    sb_opendir,
     sb_rename,
 #ifdef __NR_fcntl64
     sb_fcntl64,
@@ -1425,6 +1517,19 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
   return 0;
 }
 
+int
+sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir)
+{
+  sandbox_cfg_t *elem = NULL;
+
+  elem = new_element(PHONY_OPENDIR_SYSCALL, dir);
+
+  elem->next = *cfg;
+  *cfg = elem;
+
+  return 0;
+}
+
 /**
  * Function responsible for going through the parameter syscall filters and
  * call each function pointer in the list.
@@ -1517,15 +1622,16 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // marking the sandbox as active
   sandbox_active = 1;
-  tor_make_getaddrinfo_cache_active();
 
  end:
   seccomp_release(ctx);
   return (rc < 0 ? -rc : rc);
 }
 
+#ifdef SYSCALL_NAME_DEBUGGING
 #include "lib/sandbox/linux_syscalls.inc"
 
+/** Return a string containing the name of a given syscall (if we know it) */
 static const char *
 get_syscall_name(int syscall_num)
 {
@@ -1543,6 +1649,28 @@ get_syscall_name(int syscall_num)
   }
 }
 
+/** Return the syscall number from a ucontext_t that we got in a signal
+ * handler (if we know how to do that). */
+static int
+get_syscall_from_ucontext(const ucontext_t *ctx)
+{
+  return (int) ctx->uc_mcontext.M_SYSCALL;
+}
+#else
+static const char *
+get_syscall_name(int syscall_num)
+{
+  (void) syscall_num;
+  return "unknown";
+}
+static int
+get_syscall_from_ucontext(const ucontext_t *ctx)
+{
+  (void) ctx;
+  return -1;
+}
+#endif
+
 #ifdef USE_BACKTRACE
 #define MAX_DEPTH 256
 static void *syscall_cb_buf[MAX_DEPTH];
@@ -1558,7 +1686,6 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
 {
   ucontext_t *ctx = (ucontext_t *) (void_context);
   const char *syscall_name;
-  int syscall;
 #ifdef USE_BACKTRACE
   size_t depth;
   int n_fds, i;
@@ -1573,7 +1700,7 @@ sigsys_debugging(int nr, siginfo_t *info, void *void_context)
   if (!ctx)
     return;
 
-  syscall = (int) ctx->uc_mcontext.M_SYSCALL;
+  int syscall = get_syscall_from_ucontext(ctx);
 
 #ifdef USE_BACKTRACE
   depth = backtrace(syscall_cb_buf, MAX_DEPTH);
@@ -1731,6 +1858,13 @@ sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file)
 }
 
 int
+sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir)
+{
+  (void)cfg; (void)dir;
+  return 0;
+}
+
+int
 sandbox_cfg_allow_stat_filename(sandbox_cfg_t **cfg, char *file)
 {
   (void)cfg; (void)file;
@@ -1764,9 +1898,4 @@ sandbox_is_active(void)
   return 0;
 }
 
-void
-sandbox_disable_getaddrinfo_cache(void)
-{
-}
-
 #endif /* !defined(USE_LIBSECCOMP) */
diff --git a/src/lib/sandbox/sandbox.h b/src/lib/sandbox/sandbox.h
index 5bec09a36a..a2b3227b90 100644
--- a/src/lib/sandbox/sandbox.h
+++ b/src/lib/sandbox/sandbox.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,10 +29,10 @@
 #define USE_LIBSECCOMP
 #endif
 
-struct sandbox_cfg_elem;
+struct sandbox_cfg_elem_t;
 
 /** Typedef to structure used to manage a sandbox configuration. */
-typedef struct sandbox_cfg_elem sandbox_cfg_t;
+typedef struct sandbox_cfg_elem_t sandbox_cfg_t;
 
 /**
  * Linux definitions
@@ -58,7 +58,7 @@ typedef enum {
  *  Configuration parameter structure associated with the LIBSECCOMP2
  *  implementation.
  */
-typedef struct smp_param {
+typedef struct smp_param_t {
   /** syscall associated with parameter. */
   int syscall;
 
@@ -77,7 +77,7 @@ typedef struct smp_param {
  * It is implemented as a linked list of parameters. Currently only controls
  * parameters for open, openat, execve, stat64.
  */
-struct sandbox_cfg_elem {
+struct sandbox_cfg_elem_t {
   /** Sandbox implementation which dictates the parameter type. */
   SB_IMPL implem;
 
@@ -85,7 +85,7 @@ struct sandbox_cfg_elem {
   smp_param_t *param;
 
   /** Next element of the configuration*/
-  struct sandbox_cfg_elem *next;
+  struct sandbox_cfg_elem_t *next;
 };
 
 /** Function pointer defining the prototype of a filter function.*/
@@ -108,7 +108,7 @@ typedef struct {
  * it matches the parameter.
  */
 const char* sandbox_intern_string(const char *param);
-#else /* !(defined(USE_LIBSECCOMP)) */
+#else /* !defined(USE_LIBSECCOMP) */
 #define sandbox_intern_string(s) (s)
 #endif /* defined(USE_LIBSECCOMP) */
 
@@ -136,6 +136,13 @@ int sandbox_cfg_allow_rename(sandbox_cfg_t **cfg, char *file1, char *file2);
 int sandbox_cfg_allow_openat_filename(sandbox_cfg_t **cfg, char *file);
 
 /**
+ * Function used to add a opendir allowed filename to a supplied configuration.
+ * The (char*) specifies the path to the allowed dir; we steal the pointer to
+ * that dir.
+ */
+int sandbox_cfg_allow_opendir_dirname(sandbox_cfg_t **cfg, char *dir);
+
+/**
  * Function used to add a stat/stat64 allowed filename to a configuration.
  * The (char*) specifies the path to the allowed file; that pointer is stolen.
  */
diff --git a/src/lib/smartlist_core/.may_include b/src/lib/smartlist_core/.may_include
index a8507761a4..2f0c8d341e 100644
--- a/src/lib/smartlist_core/.may_include
+++ b/src/lib/smartlist_core/.may_include
@@ -4,4 +4,4 @@ lib/malloc/*.h
 lib/err/*.h
 lib/string/*.h
 lib/smartlist_core/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
diff --git a/src/lib/smartlist_core/include.am b/src/lib/smartlist_core/include.am
index 99d65f0b23..548179bc4f 100644
--- a/src/lib/smartlist_core/include.am
+++ b/src/lib/smartlist_core/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-smartlist-core-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_smartlist_core_a_SOURCES =			\
 	src/lib/smartlist_core/smartlist_core.c                 \
 	src/lib/smartlist_core/smartlist_split.c
@@ -15,6 +16,7 @@ src_lib_libtor_smartlist_core_testing_a_CPPFLAGS = \
 	$(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_smartlist_core_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/smartlist_core/smartlist_core.h         \
 	src/lib/smartlist_core/smartlist_foreach.h      \
diff --git a/src/lib/smartlist_core/lib_smartlist_core.md b/src/lib/smartlist_core/lib_smartlist_core.md
new file mode 100644
index 0000000000..c031dd6f24
--- /dev/null
+++ b/src/lib/smartlist_core/lib_smartlist_core.md
@@ -0,0 +1,10 @@
+@dir /lib/smartlist_core
+@brief lib/smartlist_core: Minimal dynamic array implementation
+
+A `smartlist_t` is a dynamic array type for holding `void *`.  We use it
+throughout the rest of the codebase.
+
+There are higher-level pieces in \refdir{lib/container} but
+the ones in lib/smartlist_core are used by the logging code, and therefore
+cannot use the logging code.
+
diff --git a/src/lib/smartlist_core/smartlist_core.c b/src/lib/smartlist_core/smartlist_core.c
index ac85a6cc84..571d17aa5d 100644
--- a/src/lib/smartlist_core/smartlist_core.c
+++ b/src/lib/smartlist_core/smartlist_core.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -88,6 +88,30 @@ smartlist_ensure_capacity(smartlist_t *sl, size_t size)
 #undef MAX_CAPACITY
 }
 
+/** Expand <b>sl</b> so that its length is at least <b>new_size</b>,
+ * filling in previously unused entries with NULL>
+ *
+ * Do nothing if <b>sl</b> already had at least <b>new_size</b> elements.
+ */
+void
+smartlist_grow(smartlist_t *sl, size_t new_size)
+{
+  smartlist_ensure_capacity(sl, new_size);
+
+  if (new_size > (size_t)sl->num_used) {
+    /* This memset() should be a no-op: everything else in the smartlist code
+     * tries to make sure that unused entries are always NULL.  Still, that is
+     * meant as a safety mechanism, so let's clear the memory here.
+     */
+    memset(sl->list + sl->num_used, 0,
+           sizeof(void *) * (new_size - sl->num_used));
+
+    /* This cast is safe, since we already asserted that we were below
+     * MAX_CAPACITY in smartlist_ensure_capacity(). */
+    sl->num_used = (int)new_size;
+  }
+}
+
 /** Append element to the end of the list. */
 void
 smartlist_add(smartlist_t *sl, void *element)
@@ -153,6 +177,8 @@ smartlist_remove_keeporder(smartlist_t *sl, const void *element)
       sl->list[i++] = sl->list[j];
     }
   }
+  memset(sl->list + sl->num_used, 0,
+         sizeof(void *) * (num_used_orig - sl->num_used));
 }
 
 /** If <b>sl</b> is nonempty, remove and return the final element.  Otherwise,
diff --git a/src/lib/smartlist_core/smartlist_core.h b/src/lib/smartlist_core/smartlist_core.h
index a7fbaa099b..de6fe69d3a 100644
--- a/src/lib/smartlist_core/smartlist_core.h
+++ b/src/lib/smartlist_core/smartlist_core.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -43,6 +43,7 @@ void smartlist_clear(smartlist_t *sl);
 void smartlist_add(smartlist_t *sl, void *element);
 void smartlist_add_all(smartlist_t *sl, const smartlist_t *s2);
 void smartlist_add_strdup(struct smartlist_t *sl, const char *string);
+void smartlist_grow(smartlist_t *sl, size_t new_size);
 
 void smartlist_remove(smartlist_t *sl, const void *element);
 void smartlist_remove_keeporder(smartlist_t *sl, const void *element);
@@ -76,7 +77,7 @@ static inline void smartlist_set(smartlist_t *sl, int idx, void *val) {
   raw_assert(sl->num_used > idx);
   sl->list[idx] = val;
 }
-#else /* !(defined(DEBUG_SMARTLIST)) */
+#else /* !defined(DEBUG_SMARTLIST) */
 #define smartlist_len(sl) ((sl)->num_used)
 #define smartlist_get(sl, idx) ((sl)->list[idx])
 #define smartlist_set(sl, idx, val) ((sl)->list[idx] = (val))
@@ -97,4 +98,4 @@ void smartlist_del(smartlist_t *sl, int idx);
 void smartlist_del_keeporder(smartlist_t *sl, int idx);
 void smartlist_insert(smartlist_t *sl, int idx, void *val);
 
-#endif /* !defined(TOR_CONTAINER_H) */
+#endif /* !defined(TOR_SMARTLIST_CORE_H) */
diff --git a/src/lib/smartlist_core/smartlist_foreach.h b/src/lib/smartlist_core/smartlist_foreach.h
index 0f6fe30074..03edb80f05 100644
--- a/src/lib/smartlist_core/smartlist_foreach.h
+++ b/src/lib/smartlist_core/smartlist_foreach.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -83,6 +83,19 @@
          ++var ## _sl_idx) {                                    \
       var = (sl)->list[var ## _sl_idx];
 
+/** Iterates over the items in smartlist <b>sl</b> in reverse order, similar to
+ *  SMARTLIST_FOREACH_BEGIN
+ *
+ * NOTE: This macro is incompatible with SMARTLIST_DEL_CURRENT.
+ */
+#define SMARTLIST_FOREACH_REVERSE_BEGIN(sl, type, var)  \
+  STMT_BEGIN                                                       \
+    int var ## _sl_idx, var ## _sl_len=(sl)->num_used;             \
+    type var;                                                      \
+    for (var ## _sl_idx = var ## _sl_len-1; var ## _sl_idx >= 0;   \
+         --var ## _sl_idx) {                                       \
+      var = (sl)->list[var ## _sl_idx];
+
 #define SMARTLIST_FOREACH_END(var)              \
     var = NULL;                                 \
     (void) var ## _sl_idx;                      \
diff --git a/src/lib/smartlist_core/smartlist_split.c b/src/lib/smartlist_core/smartlist_split.c
index c9cf59851f..b76b87406d 100644
--- a/src/lib/smartlist_core/smartlist_split.c
+++ b/src/lib/smartlist_core/smartlist_split.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/smartlist_core/smartlist_split.h b/src/lib/smartlist_core/smartlist_split.h
index 4f72376125..fc964201e9 100644
--- a/src/lib/smartlist_core/smartlist_split.h
+++ b/src/lib/smartlist_core/smartlist_split.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@
 int smartlist_split_string(smartlist_t *sl, const char *str, const char *sep,
                            int flags, int max);
 
-#endif
+#endif /* !defined(TOR_SMARTLIST_SPLIT_H) */
diff --git a/src/lib/string/.may_include b/src/lib/string/.may_include
index ec5c769831..1fb9127f19 100644
--- a/src/lib/string/.may_include
+++ b/src/lib/string/.may_include
@@ -6,5 +6,5 @@ lib/malloc/*.h
 lib/ctime/*.h
 lib/string/*.h
 
-strlcat.c
-strlcpy.c
+ext/strlcat.c
+ext/strlcpy.c
diff --git a/src/lib/string/compat_ctype.c b/src/lib/string/compat_ctype.c
index f5d82be3ae..a7668bfbfb 100644
--- a/src/lib/string/compat_ctype.c
+++ b/src/lib/string/compat_ctype.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -29,6 +29,7 @@ const uint32_t TOR_ISPRINT_TABLE[8] =
   { 0, 0xffffffff, 0xffffffff, 0x7fffffff, 0, 0, 0, 0x0 };
 const uint32_t TOR_ISUPPER_TABLE[8] = { 0, 0, 0x7fffffe, 0, 0, 0, 0, 0 };
 const uint32_t TOR_ISLOWER_TABLE[8] = { 0, 0, 0, 0x7fffffe, 0, 0, 0, 0 };
+/**@}*/
 
 /** Upper-casing and lowercasing tables to map characters to upper/lowercase
  * equivalents.  Used by tor_toupper() and tor_tolower(). */
diff --git a/src/lib/string/compat_ctype.h b/src/lib/string/compat_ctype.h
index dbddd356c1..53ee6066f8 100644
--- a/src/lib/string/compat_ctype.h
+++ b/src/lib/string/compat_ctype.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/string/compat_string.c b/src/lib/string/compat_string.c
index e125c921a4..2bd3c2f2b4 100644
--- a/src/lib/string/compat_string.c
+++ b/src/lib/string/compat_string.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,10 +14,10 @@
 
 /* Inline the strl functions if the platform doesn't have them. */
 #ifndef HAVE_STRLCPY
-#include "strlcpy.c"
+#include "ext/strlcpy.c"
 #endif
 #ifndef HAVE_STRLCAT
-#include "strlcat.c"
+#include "ext/strlcat.c"
 #endif
 
 #include <stdlib.h>
diff --git a/src/lib/string/compat_string.h b/src/lib/string/compat_string.h
index a0e37bb6dc..f05265bdcc 100644
--- a/src/lib/string/compat_string.h
+++ b/src/lib/string/compat_string.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,20 +25,23 @@ static inline int strncasecmp(const char *a, const char *b, size_t n);
 static inline int strncasecmp(const char *a, const char *b, size_t n) {
   return _strnicmp(a,b,n);
 }
-#endif
+#endif /* !defined(HAVE_STRNCASECMP) */
 #ifndef HAVE_STRCASECMP
 static inline int strcasecmp(const char *a, const char *b);
 static inline int strcasecmp(const char *a, const char *b) {
   return _stricmp(a,b);
 }
-#endif
-#endif
+#endif /* !defined(HAVE_STRCASECMP) */
+#endif /* defined(_WIN32) */
 
 #if defined __APPLE__
 /* On OSX 10.9 and later, the overlap-checking code for strlcat would
  * appear to have a severe bug that can sometimes cause aborts in Tor.
  * Instead, use the non-checking variants.  This is sad.
  *
+ * (If --enable-fragile-hardening is passed to configure, we use the hardened
+ * variants, which do not suffer from this issue.)
+ *
  * See https://trac.torproject.org/projects/tor/ticket/15205
  */
 #undef strlcat
@@ -59,4 +62,4 @@ char *tor_strtok_r_impl(char *str, const char *sep, char **lasts);
 #define tor_strtok_r(str, sep, lasts) tor_strtok_r_impl(str, sep, lasts)
 #endif
 
-#endif
+#endif /* !defined(TOR_COMPAT_STRING_H) */
diff --git a/src/lib/string/include.am b/src/lib/string/include.am
index edd74b8a3e..82d35cc5af 100644
--- a/src/lib/string/include.am
+++ b/src/lib/string/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-string-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_string_a_SOURCES =			\
 	src/lib/string/compat_ctype.c			\
 	src/lib/string/compat_string.c			\
@@ -18,6 +19,7 @@ src_lib_libtor_string_testing_a_SOURCES = \
 src_lib_libtor_string_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_string_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/string/compat_ctype.h			\
 	src/lib/string/compat_string.h			\
diff --git a/src/lib/string/lib_string.md b/src/lib/string/lib_string.md
new file mode 100644
index 0000000000..98e3e652ed
--- /dev/null
+++ b/src/lib/string/lib_string.md
@@ -0,0 +1,13 @@
+@dir /lib/string
+@brief lib/string: Low-level string manipulation.
+
+We have a number of compatibility functions here: some are for handling
+functionality that is not implemented (or not implemented the same) on every
+platform; some are for providing locale-independent versions of libc
+functions that would otherwise be defined differently for different users.
+
+Other functions here are for common string-manipulation operations that we do
+in the rest of the codebase.
+
+Any string function high-level enough to need logging belongs in a
+higher-level module.
diff --git a/src/lib/string/parse_int.c b/src/lib/string/parse_int.c
index fbdd554a47..11ce0fa415 100644
--- a/src/lib/string/parse_int.c
+++ b/src/lib/string/parse_int.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,6 +9,7 @@
  **/
 
 #include "lib/string/parse_int.h"
+#include "lib/cc/compat_compiler.h"
 
 #include <errno.h>
 #include <stdlib.h>
@@ -17,6 +18,7 @@
 /* Helper: common code to check whether the result of a strtol or strtoul or
  * strtoll is correct. */
 #define CHECK_STRTOX_RESULT()                           \
+  STMT_BEGIN                                            \
   /* Did an overflow occur? */                          \
   if (errno == ERANGE)                                  \
     goto err;                                           \
@@ -38,7 +40,8 @@
  err:                                                   \
   if (ok) *ok = 0;                                      \
   if (next) *next = endptr;                             \
-  return 0
+  return 0;                                             \
+  STMT_END
 
 /** Extract a long from the start of <b>s</b>, in the given numeric
  * <b>base</b>.  If <b>base</b> is 0, <b>s</b> is parsed as a decimal,
diff --git a/src/lib/string/parse_int.h b/src/lib/string/parse_int.h
index 925547942e..27939ade61 100644
--- a/src/lib/string/parse_int.h
+++ b/src/lib/string/parse_int.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,4 @@ double tor_parse_double(const char *s, double min, double max, int *ok,
 uint64_t tor_parse_uint64(const char *s, int base, uint64_t min,
                          uint64_t max, int *ok, char **next);
 
-#endif
+#endif /* !defined(TOR_PARSE_INT_H) */
diff --git a/src/lib/string/printf.c b/src/lib/string/printf.c
index a5cb71ce09..86d860935e 100644
--- a/src/lib/string/printf.c
+++ b/src/lib/string/printf.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -117,8 +117,8 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = NULL;
     return -1;
   }
-  strp_tmp = tor_malloc(len + 1);
-  r = _vsnprintf(strp_tmp, len+1, fmt, args);
+  strp_tmp = tor_malloc((size_t)len + 1);
+  r = _vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
@@ -153,9 +153,9 @@ tor_vasprintf(char **strp, const char *fmt, va_list args)
     *strp = tor_strdup(buf);
     return len;
   }
-  strp_tmp = tor_malloc(len+1);
+  strp_tmp = tor_malloc((size_t)len+1);
   /* use of tor_vsnprintf() will ensure string is null terminated */
-  r = tor_vsnprintf(strp_tmp, len+1, fmt, args);
+  r = tor_vsnprintf(strp_tmp, (size_t)len+1, fmt, args);
   if (r != len) {
     tor_free(strp_tmp);
     *strp = NULL;
diff --git a/src/lib/string/printf.h b/src/lib/string/printf.h
index 2cc13d6bee..5ab751b338 100644
--- a/src/lib/string/printf.h
+++ b/src/lib/string/printf.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,4 +27,4 @@ int tor_asprintf(char **strp, const char *fmt, ...)
 int tor_vasprintf(char **strp, const char *fmt, va_list args)
   CHECK_PRINTF(2,0);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_PRINTF_H) */
diff --git a/src/lib/string/scanf.c b/src/lib/string/scanf.c
index 1bc39b5182..89d1683204 100644
--- a/src/lib/string/scanf.c
+++ b/src/lib/string/scanf.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/string/scanf.h b/src/lib/string/scanf.h
index 6673173de5..67e9c5eb78 100644
--- a/src/lib/string/scanf.h
+++ b/src/lib/string/scanf.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -21,4 +21,4 @@ int tor_vsscanf(const char *buf, const char *pattern, va_list ap) \
 int tor_sscanf(const char *buf, const char *pattern, ...)
   CHECK_SCANF(2, 3);
 
-#endif /* !defined(TOR_UTIL_STRING_H) */
+#endif /* !defined(TOR_UTIL_SCANF_H) */
diff --git a/src/lib/string/strings.md b/src/lib/string/strings.md
new file mode 100644
index 0000000000..b22574a05a
--- /dev/null
+++ b/src/lib/string/strings.md
@@ -0,0 +1,102 @@
+
+@page strings String processing in Tor
+
+Since you're reading about a C program, you probably expected this
+section: it's full of functions for manipulating the (notoriously
+dubious) C string abstraction.  I'll describe some often-missed
+highlights here.
+
+### Comparing strings and memory chunks ###
+
+We provide strcmpstart() and strcmpend() to perform a strcmp with the start
+or end of a string.
+
+	tor_assert(!strcmpstart("Hello world","Hello"));
+	tor_assert(!strcmpend("Hello world","world"));
+
+	tor_assert(!strcasecmpstart("HELLO WORLD","Hello"));
+	tor_assert(!strcasecmpend("HELLO WORLD","world"));
+
+To compare two string pointers, either of which might be NULL, use
+strcmp_opt().
+
+To search for a string or a chunk of memory within a non-null
+terminated memory block, use tor_memstr or tor_memmem respectively.
+
+We avoid using memcmp() directly, since it tends to be used in cases
+when having a constant-time operation would be better.  Instead, we
+recommend tor_memeq() and tor_memneq() for when you need a
+constant-time operation.  In cases when you need a fast comparison,
+and timing leaks are not a danger, you can use fast_memeq() and
+fast_memneq().
+
+It's a common pattern to take a string representing one or more lines
+of text, and search within it for some other string, at the start of a
+line.  You could search for "\\ntarget", but that would miss the first
+line.  Instead, use find_str_at_start_of_line.
+
+### Parsing text ###
+
+Over the years, we have accumulated lots of ways to parse text --
+probably too many. Refactoring them to be safer and saner could be a
+good project!  The one that seems most error-resistant is tokenizing
+text with smartlist_split_strings().  This function takes a smartlist,
+a string, and a separator, and splits the string along occurrences of
+the separator, adding new strings for the sub-elements to the given
+smartlist.
+
+To handle time, you can use one of the functions mentioned above in
+"Parsing and encoding time values".
+
+For numbers in general, use the tor_parse_{long,ulong,double,uint64}
+family of functions.  Each of these can be called in a few ways.  The
+most general is as follows:
+
+      const int BASE = 10;
+      const int MINVAL = 10, MAXVAL = 10000;
+      const char *next;
+      int ok;
+      long lng = tor_parse_long("100", BASE, MINVAL, MAXVAL, &ok, &next);
+
+The return value should be ignored if "ok" is set to false.  The input
+string needs to contain an entire number, or it's considered
+invalid... unless the "next" pointer is available, in which case extra
+characters at the end are allowed, and "next" is set to point to the
+first such character.
+
+### Generating blocks of text ###
+
+For not-too-large blocks of text, we provide tor_asprintf(), which
+behaves like other members of the sprintf() family, except that it
+always allocates enough memory on the heap for its output.
+
+For larger blocks: Rather than using strlcat and strlcpy to build
+text, or keeping pointers to the interior of a memory block, we
+recommend that you use the smartlist_* functions to build a smartlist
+full of substrings in order.  Then you can concatenate them into a
+single string with smartlist_join_strings(), which also takes optional
+separator and terminator arguments.
+
+Alternatively, you might find it more convenient (and more
+allocation-efficient) to use the buffer API in buffers.c: Construct a buf_t
+object, add your data to it with buf_add_string(), buf_add_printf(), and so
+on, then call buf_extract() to get the resulting output.
+
+As a convenience, we provide smartlist_add_asprintf(), which combines
+the two methods above together.  Many of the cryptographic digest
+functions also accept a not-yet-concatenated smartlist of strings.
+
+### Logging helpers ###
+
+Often we'd like to log a value that comes from an untrusted source.
+To do this, use escaped() to escape the nonprintable characters and
+other confusing elements in a string, and surround it by quotes.  (Use
+esc_for_log() if you need to allocate a new string.)
+
+It's also handy to put memory chunks into hexadecimal before logging;
+you can use hex_str(memory, length) for that.
+
+The escaped() and hex_str() functions both provide outputs that are
+only valid till they are next invoked; they are not threadsafe.
+
+*/
diff --git a/src/lib/string/util_string.c b/src/lib/string/util_string.c
index f934f66f02..c8f12d780e 100644
--- a/src/lib/string/util_string.c
+++ b/src/lib/string/util_string.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -71,7 +71,7 @@ tor_memstr(const void *haystack, size_t hlen, const char *needle)
 
 /** Return true iff the 'len' bytes at 'mem' are all zero. */
 int
-tor_mem_is_zero(const char *mem, size_t len)
+fast_mem_is_zero(const char *mem, size_t len)
 {
   static const char ZERO[] = {
     0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
@@ -95,17 +95,14 @@ tor_mem_is_zero(const char *mem, size_t len)
 int
 tor_digest_is_zero(const char *digest)
 {
-  static const uint8_t ZERO_DIGEST[] = {
-    0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
-  };
-  return tor_memeq(digest, ZERO_DIGEST, DIGEST_LEN);
+  return safe_mem_is_zero(digest, DIGEST_LEN);
 }
 
 /** Return true iff the DIGEST256_LEN bytes in digest are all zero. */
 int
 tor_digest256_is_zero(const char *digest)
 {
-  return tor_mem_is_zero(digest, DIGEST256_LEN);
+  return safe_mem_is_zero(digest, DIGEST256_LEN);
 }
 
 /** Remove from the string <b>s</b> every character which appears in
@@ -212,21 +209,6 @@ strcmpstart(const char *s1, const char *s2)
   return strncmp(s1, s2, n);
 }
 
-/** Compare the s1_len-byte string <b>s1</b> with <b>s2</b>,
- * without depending on a terminating nul in s1.  Sorting order is first by
- * length, then lexically; return values are as for strcmp.
- */
-int
-strcmp_len(const char *s1, const char *s2, size_t s1_len)
-{
-  size_t s2_len = strlen(s2);
-  if (s1_len < s2_len)
-    return -1;
-  if (s1_len > s2_len)
-    return 1;
-  return fast_memcmp(s1, s2, s2_len);
-}
-
 /** Compares the first strlen(s2) characters of s1 with s2.  Returns as for
  * strcasecmp.
  */
@@ -524,6 +506,23 @@ validate_char(const uint8_t *c, uint8_t len)
 int
 string_is_utf8(const char *str, size_t len)
 {
+  // If str is NULL, don't try to read it
+  if (!str) {
+    // We could test for this case, but the low-level logs would produce
+    // confusing test output.
+    // LCOV_EXCL_START
+    if (len) {
+      // Use the low-level logging function, so that the log module can
+      // validate UTF-8 (if needed in future code)
+      tor_log_err_sigsafe(
+        "BUG: string_is_utf8() called with NULL str but non-zero len.");
+      // Since it's a bug, we should probably reject this string
+      return false;
+    }
+    // LCOV_EXCL_STOP
+    return true;
+  }
+
   for (size_t i = 0; i < len;) {
     uint8_t num_bytes = bytes_in_char(str[i]);
     if (num_bytes == 0) // Invalid leading byte found.
@@ -541,3 +540,16 @@ string_is_utf8(const char *str, size_t len)
   }
   return true;
 }
+
+/** As string_is_utf8(), but returns false if the string begins with a UTF-8
+ * byte order mark (BOM).
+ */
+int
+string_is_utf8_no_bom(const char *str, size_t len)
+{
+  if (str && len >= 3 && (!strcmpstart(str, "\uFEFF") ||
+                          !strcmpstart(str, "\uFFFE"))) {
+    return false;
+  }
+  return string_is_utf8(str, len);
+}
diff --git a/src/lib/string/util_string.h b/src/lib/string/util_string.h
index d9fbf8c61e..e89233df88 100644
--- a/src/lib/string/util_string.h
+++ b/src/lib/string/util_string.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,10 @@ const void *tor_memmem(const void *haystack, size_t hlen, const void *needle,
                        size_t nlen);
 const void *tor_memstr(const void *haystack, size_t hlen,
                        const char *needle);
-int tor_mem_is_zero(const char *mem, size_t len);
+int fast_mem_is_zero(const char *mem, size_t len);
+#define fast_digest_is_zero(d) fast_mem_is_zero((d), DIGEST_LEN)
+#define fast_digetst256_is_zero(d) fast_mem_is_zero((d), DIGEST256_LEN)
+
 int tor_digest_is_zero(const char *digest);
 int tor_digest256_is_zero(const char *digest);
 
@@ -33,7 +36,6 @@ int tor_strisnonupper(const char *s);
 int tor_strisspace(const char *s);
 int strcmp_opt(const char *s1, const char *s2);
 int strcmpstart(const char *s1, const char *s2);
-int strcmp_len(const char *s1, const char *s2, size_t len);
 int strcasecmpstart(const char *s1, const char *s2);
 int strcmpend(const char *s1, const char *s2);
 int strcasecmpend(const char *s1, const char *s2);
@@ -53,5 +55,6 @@ const char *find_str_at_start_of_line(const char *haystack,
 int string_is_C_identifier(const char *string);
 
 int string_is_utf8(const char *str, size_t len);
+int string_is_utf8_no_bom(const char *str, size_t len);
 
 #endif /* !defined(TOR_UTIL_STRING_H) */
diff --git a/src/lib/subsys/.may_include b/src/lib/subsys/.may_include
new file mode 100644
index 0000000000..2b06e8519c
--- /dev/null
+++ b/src/lib/subsys/.may_include
@@ -0,0 +1 @@
+orconfig.h
diff --git a/src/lib/subsys/include.am b/src/lib/subsys/include.am
new file mode 100644
index 0000000000..c9ab54ca73
--- /dev/null
+++ b/src/lib/subsys/include.am
@@ -0,0 +1,4 @@
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS += \
+	src/lib/subsys/subsys.h
diff --git a/src/lib/subsys/initialization.md b/src/lib/subsys/initialization.md
new file mode 100644
index 0000000000..012ab7000d
--- /dev/null
+++ b/src/lib/subsys/initialization.md
@@ -0,0 +1,75 @@
+
+@page initialization Initialization and shutdown
+
+@tableofcontents
+
+@section overview Overview
+
+Tor has a single entry point: tor_run_main() in main.c.  All the ways of
+starting a Tor process (ntmain.c, tor_main.c, and tor_api.c) work by invoking tor_run_main().
+
+The tor_run_main() function normally exits (@ref init_exceptwhen "1") by
+returning: not by calling abort() or exit().  Before it returns, it calls
+tor_cleanup() in shutdown.c.
+
+Conceptually, there are several stages in running Tor.
+
+1. First, we initialize those modules that do not depend on the
+   configuration.  This happens in the first half of tor_run_main(), and the
+   first half of tor_init().  (@ref init_pending_refactor "2")
+
+2. Second, we parse the command line and our configuration, and configure
+   systems that depend on our configuration or state.  This configuration
+   happens midway through tor_init(), which invokes
+   options_init_from_torrc().  We then initialize more systems from the
+   second half of tor_init().
+
+3. At this point we may exit early if we have been asked to do something
+   requiring no further initialization, like printing our version number or
+   creating a new signing key.  Otherwise, we proceed to run_tor_main_loop(),
+   which initializes some network-specific parts of Tor, grabs some
+   daemon-only resources (like the data directory lock) and starts Tor itself
+   running.
+
+
+> @anchor init_exceptwhen 1. tor_run_main() _can_ terminate with a call to
+> abort() or exit(), but only when crashing due to a bug, or when forking to
+> run as a daemon.
+
+> @anchor init_pending_refactor 2. The pieces of code that I'm describing as
+> "the first part of tor_init()" and so on deserve to be functions with their
+> own name.  I'd like to refactor them, but before I do so, there is some
+> slight reorganization that needs to happen.  Notably, the
+> nt_service_parse_options() call ought logically to be later in our
+> initialization sequence.  See @ticket{32447} for our refactoring progress.
+
+
+@section subsys Subsystems and initialization
+
+Our current convention is to use the subsystem mechanism to initialize and
+clean up pieces of Tor.  The more recently updated pieces of Tor will use
+this mechanism.  For examples, see e.g. time_sys.c or log_sys.c.
+
+In simplest terms, a **subsytem** is a logically separate part of Tor that
+can be initialized, shut down, managed, and configured somewhat independently
+of the rest of the program.
+
+The subsys_fns_t type describes a subsystem and a set of functions that
+initialize it, desconstruct it, and so on. To define a subsystem, we declare
+a `const` instance of subsys_fns_t.  See the documentation for subsys_fns_t
+for a full list of these functions.
+
+After defining a subsytem, it must be inserted in subsystem_list.c.  At that
+point, table-driven mechanisms in subsysmgr.c will invoke its functions when
+appropriate.
+
+@subsection vsconfig Initialization versus configuration
+
+We note that the initialization phase of Tor occurs before any configuration
+is read from disk -- and therefore before any other files are read from
+disk.  Therefore, any behavior that depends on Tor's configuration or state
+must occur _after_ the initialization process, during configuration.
+
+
+
+
diff --git a/src/lib/subsys/lib_subsys.md b/src/lib/subsys/lib_subsys.md
new file mode 100644
index 0000000000..764d25d1b6
--- /dev/null
+++ b/src/lib/subsys/lib_subsys.md
@@ -0,0 +1,32 @@
+@dir /lib/subsys
+@brief lib/subsys: Types for declaring a "subsystem".
+
+## Subsystems in Tor
+
+A subsystem is a module with support for initialization, shutdown,
+configuration, and so on.
+
+Many parts of Tor can be initialized, cleaned up, and configured somewhat
+independently through a table-driven mechanism.  Each such part is called a
+"subsystem".
+
+To declare a subsystem, make a global `const` instance of the `subsys_fns_t`
+type, filling in the function pointer fields that you require with ones
+corresponding to your subsystem.  Any function pointers left as "NULL" will
+be a no-op.  Each system must have a name and a "level", which corresponds to
+the order in which it is initialized.  (See `app/main/subsystem_list.c` for a
+list of current subsystems and their levels.)
+
+Then, insert your subsystem in the list in `app/main/subsystem_list.c`.  It
+will need to occupy a position corresponding to its level.
+
+At this point, your subsystem will be handled like the others: it will get
+initialized at startup, torn down at exit, and so on.
+
+Historical note: Not all of Tor's code is currently handled as
+subsystems. As you work with older code, you may see some parts of the code
+that are initialized from `tor_init()` or `run_tor_main_loop()` or
+`tor_run_main()`; and torn down from `tor_cleanup()`.  We aim to migrate
+these to subsystems over time; please don't add any new code that follows
+this pattern.
+
diff --git a/src/lib/subsys/subsys.h b/src/lib/subsys/subsys.h
new file mode 100644
index 0000000000..62c0de026d
--- /dev/null
+++ b/src/lib/subsys/subsys.h
@@ -0,0 +1,218 @@
+/* Copyright (c) 2003-2004, Roger Dingledine
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file subsys.h
+ * @brief Types used to declare a subsystem.
+ **/
+
+#ifndef TOR_SUBSYS_T
+#define TOR_SUBSYS_T
+
+#include <stdbool.h>
+
+struct pubsub_connector_t;
+struct config_format_t;
+
+/**
+ * A subsystem is a part of Tor that is initialized, shut down, configured,
+ * and connected to other parts of Tor.
+ *
+ * All callbacks are optional -- if a callback is set to NULL, the subsystem
+ * manager will treat it as a no-op.
+ *
+ * You should use c99 named-field initializers with this structure, for
+ * readability and safety. (There are a lot of functions here, all of them
+ * optional, and many of them with similar signatures.)
+ *
+ * See @ref initialization for more information about initialization and
+ * shutdown in Tor.
+ *
+ * To make a new subsystem, you declare a const instance of this type, and
+ * include it on the list in subsystem_list.c.  The code that manages these
+ * subsystems is in subsysmgr.c.
+ **/
+typedef struct subsys_fns_t {
+  /**
+   * The name of this subsystem.  It should be a programmer-readable
+   * identifier.
+   **/
+  const char *name;
+
+  /**
+   * The file in which the subsystem object is declared. Used for debugging.
+   **/
+  const char *location;
+
+  /**
+   * Whether this subsystem is supported -- that is, whether it is compiled
+   * into Tor.  For most subsystems, this should be true.
+   **/
+  bool supported;
+
+  /**
+   * The 'initialization level' for the subsystem.  It should run from -100
+   * through +100.  The subsystems are initialized from lowest level to
+   * highest, and shut down from highest level to lowest.
+   **/
+  int level;
+
+  /**
+   * Initialize any global components of this subsystem.
+   *
+   * This function MAY rely on any lower-level subsystem being initialized.
+   *
+   * This function MUST NOT rely on any runtime configuration information;
+   * it is only for global state or pre-configuration state.
+   *
+   * (If you need to do any setup that depends on configuration, you'll need
+   * to declare a configuration callback instead. (Not yet designed))
+   *
+   * This function MUST NOT have any parts that can fail.
+   **/
+  int (*initialize)(void);
+
+  /**
+   * Connect a subsystem to the message dispatch system.
+   *
+   * This function should use the macros in @refdir{lib/pubsub} to register a
+   * set of messages that this subsystem may publish, and may subscribe to.
+   *
+   * See pubsub_macros.h for more information, and for examples.
+   **/
+  int (*add_pubsub)(struct pubsub_connector_t *);
+
+  /**
+   * Perform any necessary pre-fork cleanup.  This function may not fail.
+   *
+   * On Windows (and any other platforms without fork()), this function will
+   * never be invoked.  Otherwise it is used when we are about to start
+   * running as a background daemon, or when we are about to run a unit test
+   * in a subprocess.  Unlike the subsys_fns_t.postfork callback, it is run
+   * from the parent process.
+   *
+   * Note that we do not invoke this function when the child process's only
+   * purpose is to call exec() and run another program.
+   */
+  void (*prefork)(void);
+
+  /**
+   * Perform any necessary post-fork setup. This function may not fail.
+   *
+   * On Windows (and any other platforms without fork()), this function will
+   * never be invoked.  Otherwise it is used when we are about to start
+   * running as a background daemon, or when we are about to run a unit test
+   * in a subprocess.  Unlike the subsys_fns_t.prefork callback, it is run
+   * from the child process.
+   *
+   * Note that we do not invoke this function when the child process's only
+   * purpose is to call exec() and run another program.
+   */
+  void (*postfork)(void);
+
+  /**
+   * Free any thread-local resources held by this subsystem. Called before
+   * the thread exits.
+   *
+   * This function is not allowed to fail.
+   *
+   * \bug Note that this callback is currently buggy: See \ticket{32103}.
+   */
+  void (*thread_cleanup)(void);
+
+  /**
+   * Free all resources held by this subsystem.
+   *
+   * This function is not allowed to fail.
+   *
+   * Subsystems are shut down when Tor is about to exit or return control to
+   * an embedding program. This callback must return the process to a state
+   * such that subsys_fns_t.init will succeed if invoked again.
+   **/
+  void (*shutdown)(void);
+
+  /**
+   * A config_format_t describing all of the torrc fields owned by this
+   * subsystem.
+   *
+   * This object, if present, is registered in a confmgr_t for Tor's options,
+   * and used to parse option fields from the command line and torrc file.
+   **/
+  const struct config_format_t *options_format;
+
+  /**
+   * A config_format_t describing all of the DataDir/state fields owned by
+   * this subsystem.
+   *
+   * This object, if present, is registered in a confmgr_t for Tor's state,
+   * and used to parse state fields from the DataDir/state file.
+   **/
+  const struct config_format_t *state_format;
+
+  /**
+   * Receive an options object as defined by options_format. Return 0
+   * on success, -1 on failure.
+   *
+   * It is safe to store the pointer to the object until set_options()
+   * is called again.
+   *
+   * This function is only called after all the validation code defined
+   * by subsys_fns_t.options_format has passed.
+   **/
+  int (*set_options)(void *);
+
+  /* XXXX Add an implementation for options_act_reversible() later in this
+   * branch. */
+
+  /**
+   * Receive a state object as defined by state_format. Return 0 on success,
+   * -1 on failure.
+   *
+   * It is safe to store the pointer to the object; set_state() is only
+   * called on startup.
+   *
+   * This function is only called after all the validation code defined
+   * by subsys_fns_t.state_format has passed.
+   *
+   * This function will only be called once per invocation of Tor, since
+   * Tor does not reload its state while it is running.
+   **/
+  int (*set_state)(void *);
+
+  /**
+   * Update any information that needs to be stored in the provided state
+   * object (as defined by state_format).  Return 0 on success, -1 on failure.
+   *
+   * The object provided here will be the same one as provided earlier to
+   * set_state().  This method is called when we are about to save the state
+   * to disk.
+   **/
+  int (*flush_state)(void *);
+} subsys_fns_t;
+
+#ifndef COCCI
+/**
+ * Macro to declare a subsystem's location.
+ **/
+#define SUBSYS_DECLARE_LOCATION() \
+  .location = __FILE__
+#endif /* !defined(COCCI) */
+
+/**
+ * Lowest allowed subsystem level.
+ **/
+#define MIN_SUBSYS_LEVEL -100
+/**
+ * Highest allowed subsystem level.
+ **/
+#define MAX_SUBSYS_LEVEL 100
+
+/**
+ * All tor "libraries" (in src/libs) should have a subsystem level equal to or
+ * less than this value.
+ */
+#define SUBSYS_LEVEL_LIBS -10
+
+#endif /* !defined(TOR_SUBSYS_T) */
diff --git a/src/lib/term/.may_include b/src/lib/term/.may_include
index c93a06e59e..306fa57b7a 100644
--- a/src/lib/term/.may_include
+++ b/src/lib/term/.may_include
@@ -5,5 +5,4 @@ lib/log/*.h
 lib/term/*.h
 lib/malloc/*.h
 
-# From src/ext
-tor_readpassphrase.h
+ext/tor_readpassphrase.h
diff --git a/src/lib/term/getpass.c b/src/lib/term/getpass.c
index c6459f250f..d2d6cb2b7b 100644
--- a/src/lib/term/getpass.c
+++ b/src/lib/term/getpass.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #elif defined(HAVE_READPASSPHRASE_H)
 #include <readpassphrase.h>
 #else
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #endif /* defined(_WIN32) || ... */
 
 #include <stdlib.h>
diff --git a/src/lib/term/getpass.h b/src/lib/term/getpass.h
index a9c146ea8f..b080ad2473 100644
--- a/src/lib/term/getpass.h
+++ b/src/lib/term/getpass.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,4 +15,4 @@
 
 ssize_t tor_getpass(const char *prompt, char *output, size_t buflen);
 
-#endif
+#endif /* !defined(TOR_GETPASS_H) */
diff --git a/src/lib/term/include.am b/src/lib/term/include.am
index 55fe548ebc..a120bba0cb 100644
--- a/src/lib/term/include.am
+++ b/src/lib/term/include.am
@@ -11,6 +11,7 @@ else
 readpassphrase_source=
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_term_a_SOURCES =			\
 	src/lib/term/getpass.c			\
 	$(readpassphrase_source)
@@ -20,5 +21,6 @@ src_lib_libtor_term_testing_a_SOURCES = \
 src_lib_libtor_term_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_term_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/term/getpass.h
diff --git a/src/lib/term/lib_term.md b/src/lib/term/lib_term.md
new file mode 100644
index 0000000000..f96d25ffe8
--- /dev/null
+++ b/src/lib/term/lib_term.md
@@ -0,0 +1,2 @@
+@dir /lib/term
+@brief lib/term: Terminal operations (password input).
diff --git a/src/lib/testsupport/include.am b/src/lib/testsupport/include.am
index b2aa620985..a5ed46eb67 100644
--- a/src/lib/testsupport/include.am
+++ b/src/lib/testsupport/include.am
@@ -1,3 +1,4 @@
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS += \
 	src/lib/testsupport/testsupport.h
diff --git a/src/lib/testsupport/lib_testsupport.md b/src/lib/testsupport/lib_testsupport.md
new file mode 100644
index 0000000000..7358e6a80f
--- /dev/null
+++ b/src/lib/testsupport/lib_testsupport.md
@@ -0,0 +1,2 @@
+@dir /lib/testsupport
+@brief lib/testsupport: Helpers for test-only code and for function mocking.
diff --git a/src/lib/testsupport/testsupport.h b/src/lib/testsupport/testsupport.h
index 9363a9ba66..165c497f71 100644
--- a/src/lib/testsupport/testsupport.h
+++ b/src/lib/testsupport/testsupport.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,16 +15,41 @@
 #ifndef TOR_TESTSUPPORT_H
 #define TOR_TESTSUPPORT_H
 
-#ifdef TOR_UNIT_TESTS
 /** The "STATIC" macro marks a function or variable that is static when
  * building Tor for production, but non-static when building the unit
- * tests. */
+ * tests.
+ *
+ * For example, a function declared as:
+ *
+ *     STATIC int internal_function(void);
+ *
+ * should be only visible for the file on which it is declared, and in the
+ * unit tests.
+ */
+#ifdef TOR_UNIT_TESTS
 #define STATIC
+#else /* !defined(TOR_UNIT_TESTS) */
+#define STATIC static
+#endif /* defined(TOR_UNIT_TESTS) */
+
+/** The "EXTERN" macro is used along with "STATIC" for variables declarations:
+ * it expands to an extern declaration when Tor building unit tests, and to
+ * nothing otherwise.
+ *
+ * For example, to declare a variable as visible only visible in one
+ * file and in the unit tests, you would put this in the header:
+ *
+ *     EXTERN(int, local_variable)
+ *
+ * and this in the source:
+ *
+ *     STATIC int local_variable;
+ */
+#ifdef TOR_UNIT_TESTS
 #define EXTERN(type, name) extern type name;
 #else
-#define STATIC static
 #define EXTERN(type, name)
-#endif /* defined(TOR_UNIT_TESTS) */
+#endif
 
 /** Quick and dirty macros to implement test mocking.
  *
@@ -70,32 +95,42 @@
  *
  * @{ */
 #ifdef TOR_UNIT_TESTS
+/** Declare a mocked function. For use in headers. */
 #define MOCK_DECL(rv, funcname, arglist)     \
   rv funcname ##__real arglist;              \
   extern rv(*funcname) arglist
+/** Define the implementation of a mocked function. */
 #define MOCK_IMPL(rv, funcname, arglist)     \
   rv(*funcname) arglist = funcname ##__real; \
   rv funcname ##__real arglist
+/** As MOCK_DECL(), but allow attributes. */
 #define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
   rv funcname ##__real arglist attr;                \
   extern rv(*funcname) arglist
-#define MOCK_IMPL(rv, funcname, arglist)     \
-  rv(*funcname) arglist = funcname ##__real; \
-  rv funcname ##__real arglist
+/**
+ * Replace <b>func</b> (a mockable function) with a replacement function.
+ *
+ * Only usable when Tor has been built for unit tests. */
 #define MOCK(func, replacement)                 \
   do {                                          \
     (func) = (replacement);                     \
   } while (0)
+/** Replace <b>func</b> (a mockable function) with its original value.
+ *
+ * Only usable when Tor has been built for unit tests. */
 #define UNMOCK(func)                            \
   do {                                          \
     func = func ##__real;                       \
   } while (0)
-#else /* !(defined(TOR_UNIT_TESTS)) */
+#else /* !defined(TOR_UNIT_TESTS) */
+/** Declare a mocked function. For use in headers. */
 #define MOCK_DECL(rv, funcname, arglist) \
   rv funcname arglist
-#define MOCK_DECL_ATTR(rv, funcname, arglist, attr) \
+/** As MOCK_DECL(), but allow  */
+#define MOCK_DECL_ATTR(rv, funcname, arglist, attr)     \
   rv funcname arglist attr
-#define MOCK_IMPL(rv, funcname, arglist) \
+/** Define the implementation of a mocked function. */
+#define MOCK_IMPL(rv, funcname, arglist)        \
   rv funcname arglist
 #endif /* defined(TOR_UNIT_TESTS) */
 /** @} */
diff --git a/src/lib/thread/.may_include b/src/lib/thread/.may_include
index fc56f46836..02711348c5 100644
--- a/src/lib/thread/.may_include
+++ b/src/lib/thread/.may_include
@@ -2,6 +2,7 @@ orconfig.h
 lib/cc/*.h
 lib/lock/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 lib/wallclock/*.h
diff --git a/src/lib/thread/compat_pthreads.c b/src/lib/thread/compat_pthreads.c
index 6f7ecd17da..d143b80252 100644
--- a/src/lib/thread/compat_pthreads.c
+++ b/src/lib/thread/compat_pthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/thread/compat_threads.c b/src/lib/thread/compat_threads.c
index 94ab021c52..75ade9c9f2 100644
--- a/src/lib/thread/compat_threads.c
+++ b/src/lib/thread/compat_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,9 +14,11 @@
 #include "orconfig.h"
 #include <stdlib.h>
 #include "lib/thread/threads.h"
+#include "lib/thread/thread_sys.h"
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
+#include "lib/subsys/subsys.h"
 
 #include <string.h>
 
@@ -65,7 +67,15 @@ atomic_counter_init(atomic_counter_t *counter)
   memset(counter, 0, sizeof(*counter));
   tor_mutex_init_nonrecursive(&counter->mutex);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -109,3 +119,18 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return oldval;
 }
 #endif /* !defined(HAVE_WORKING_STDATOMIC) */
+
+static int
+subsys_threads_initialize(void)
+{
+  tor_threads_init();
+  return 0;
+}
+
+const subsys_fns_t sys_threads = {
+  .name = "threads",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  .level = -89,
+  .initialize = subsys_threads_initialize,
+};
diff --git a/src/lib/thread/compat_winthreads.c b/src/lib/thread/compat_winthreads.c
index f0b1430e84..2ca5620d23 100644
--- a/src/lib/thread/compat_winthreads.c
+++ b/src/lib/thread/compat_winthreads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/thread/include.am b/src/lib/thread/include.am
index 9ec23d166e..cd8016b5df 100644
--- a/src/lib/thread/include.am
+++ b/src/lib/thread/include.am
@@ -12,6 +12,7 @@ if THREADS_WIN32
 threads_impl_source=src/lib/thread/compat_winthreads.c
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_thread_a_SOURCES =			\
 	src/lib/thread/compat_threads.c			\
 	src/lib/thread/numcpus.c			\
@@ -22,6 +23,8 @@ src_lib_libtor_thread_testing_a_SOURCES = \
 src_lib_libtor_thread_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_thread_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
-	src/lib/thread/threads.h			\
-	src/lib/thread/numcpus.h
+	src/lib/thread/numcpus.h			\
+	src/lib/thread/thread_sys.h			\
+	src/lib/thread/threads.h
diff --git a/src/lib/thread/lib_thread.md b/src/lib/thread/lib_thread.md
new file mode 100644
index 0000000000..5870ad790f
--- /dev/null
+++ b/src/lib/thread/lib_thread.md
@@ -0,0 +1,7 @@
+@dir /lib/thread
+@brief lib/thread: Mid-level threading.
+
+This module contains compatibility and convenience code for multithreading,
+except for low-level locks (which are in \refdir{lib/lock} and
+workqueue/threadpool code (which belongs in \refdir{lib/evloop}.)
+
diff --git a/src/lib/thread/numcpus.c b/src/lib/thread/numcpus.c
index b293d965d2..18454ce3ad 100644
--- a/src/lib/thread/numcpus.c
+++ b/src/lib/thread/numcpus.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/thread/numcpus.h b/src/lib/thread/numcpus.h
index 3f0a29ce7c..65e6c430cf 100644
--- a/src/lib/thread/numcpus.h
+++ b/src/lib/thread/numcpus.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -13,4 +13,4 @@
 
 int compute_num_cpus(void);
 
-#endif
+#endif /* !defined(TOR_NUMCPUS_H) */
diff --git a/src/lib/thread/thread_sys.h b/src/lib/thread/thread_sys.h
new file mode 100644
index 0000000000..6206fac9d6
--- /dev/null
+++ b/src/lib/thread/thread_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file thread_sys.h
+ * \brief Declare subsystem object for threads library
+ **/
+
+#ifndef TOR_THREADS_SYS_H
+#define TOR_THREADS_SYS_H
+
+extern const struct subsys_fns_t sys_threads;
+
+#endif /* !defined(TOR_THREADS_SYS_H) */
diff --git a/src/lib/thread/threading.md b/src/lib/thread/threading.md
new file mode 100644
index 0000000000..a1058c97de
--- /dev/null
+++ b/src/lib/thread/threading.md
@@ -0,0 +1,26 @@
+
+@page threading Threading in Tor
+
+Tor is based around a single main thread and one or more worker
+threads.  We aim (with middling success) to use worker threads for
+CPU-intensive activities and the main thread for our networking.
+Fortunately (?) we have enough cryptography that moving what we can
+of the cryptographic processes to the workers should achieve good
+parallelism under most loads.  Unfortunately, we only have a small
+fraction of our cryptography done in our worker threads right now.
+
+Our threads-and-workers abstraction is defined in workqueue.c, which
+combines a work queue with a thread pool, and integrates the
+signalling with libevent.  Tor's main instance of a work queue is
+instantiated in cpuworker.c.  It will probably need some refactoring
+as more types of work are added.
+
+On a lower level, we provide locks with tor_mutex_t in \refdir{lib/lock}, and
+higher-level locking/threading tools in \refdir{lib/thread}, including
+conditions (tor_cond_t), thread-local storage (tor_threadlocal_t), and more.
+
+
+Try to minimize sharing between threads: it is usually best to simply
+make the worker "own" all the data it needs while the work is in
+progress, and to give up ownership when it's complete.
+
diff --git a/src/lib/thread/threads.h b/src/lib/thread/threads.h
index ecf60641b5..fcc0c23a87 100644
--- a/src/lib/thread/threads.h
+++ b/src/lib/thread/threads.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -63,7 +63,7 @@ int tor_cond_wait(tor_cond_t *cond, tor_mutex_t *mutex,
 void tor_cond_signal_one(tor_cond_t *cond);
 void tor_cond_signal_all(tor_cond_t *cond);
 
-typedef struct tor_threadlocal_s {
+typedef struct tor_threadlocal_t {
 #ifdef _WIN32
   DWORD index;
 #else
@@ -106,8 +106,10 @@ void tor_threadlocal_set(tor_threadlocal_t *threadlocal, void *value);
 typedef struct atomic_counter_t {
   atomic_size_t val;
 } atomic_counter_t;
+#ifndef COCCI
 #define ATOMIC_LINKAGE static
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#endif
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 typedef struct atomic_counter_t {
   tor_mutex_t mutex;
   size_t val;
@@ -131,7 +133,17 @@ atomic_counter_init(atomic_counter_t *counter)
 {
   atomic_init(&counter->val, 0);
 }
-/** Clean up all resources held by an atomic counter. */
+/** Clean up all resources held by an atomic counter.
+ *
+ * This usage note applies to the compat_threads implementation of
+ * atomic_counter_destroy():
+ * Destroying a locked mutex is undefined behaviour. Global mutexes may be
+ * locked when they are passed to this function, because multiple threads can
+ * still access them. So we can either:
+ *  - destroy on shutdown, and re-initialise when tor re-initialises, or
+ *  - skip destroying and re-initialisation, using a sentinel variable.
+ * See #31735 for details.
+ */
 static inline void
 atomic_counter_destroy(atomic_counter_t *counter)
 {
@@ -162,7 +174,7 @@ atomic_counter_exchange(atomic_counter_t *counter, size_t newval)
   return atomic_exchange(&counter->val, newval);
 }
 
-#else /* !(defined(HAVE_WORKING_STDATOMIC)) */
+#else /* !defined(HAVE_WORKING_STDATOMIC) */
 #endif /* defined(HAVE_WORKING_STDATOMIC) */
 
 #endif /* !defined(TOR_COMPAT_THREADS_H) */
diff --git a/src/lib/time/.may_include b/src/lib/time/.may_include
index 2c7e37a836..ae01431b60 100644
--- a/src/lib/time/.may_include
+++ b/src/lib/time/.may_include
@@ -4,8 +4,10 @@ lib/cc/*.h
 lib/err/*.h
 lib/intmath/*.h
 lib/log/*.h
+lib/subsys/*.h
 lib/time/*.h
 lib/wallclock/*.h
+lib/defs/time.h
 
 # For load_windows_system_lib.
 lib/fs/winlib.h
\ No newline at end of file
diff --git a/src/lib/time/compat_time.c b/src/lib/time/compat_time.c
index 98854bad2c..6bbad4f98a 100644
--- a/src/lib/time/compat_time.c
+++ b/src/lib/time/compat_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -164,6 +164,8 @@ static int64_t last_tick_count = 0;
  * to be monotonic; increments them as appropriate so that they actually
  * _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC int64_t
 ratchet_performance_counter(int64_t count_raw)
@@ -202,6 +204,8 @@ static struct timeval timeofday_offset = { 0, 0 };
  * supposed to be monotonic; increments them as appropriate so that they
  * actually _are_ monotonic.
  *
+ * The returned time may be the same as the previous returned time.
+ *
  * Caller must hold lock. */
 STATIC void
 ratchet_timeval(const struct timeval *timeval_raw, struct timeval *out)
@@ -270,7 +274,9 @@ monotime_init_internal(void)
 }
 
 /**
- * Set "out" to the most recent monotonic time value
+ * Set "out" to the most recent monotonic time value.
+ *
+ * The returned time may be the same as the previous returned time.
  */
 void
 monotime_get(monotime_t *out)
@@ -298,10 +304,12 @@ monotime_coarse_get(monotime_coarse_t *out)
 #endif /* defined(TOR_UNIT_TESTS) */
   out->abstime_ = mach_approximate_time();
 }
-#endif
+#endif /* defined(HAVE_MACH_APPROXIMATE_TIME) */
 
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ *
+ * The returned value may be equal to zero.
  */
 int64_t
 monotime_diff_nsec(const monotime_t *start,
@@ -522,7 +530,9 @@ monotime_init_internal(void)
     GetTickCount64_fn = (GetTickCount64_fn_t) (void(*)(void))
       GetProcAddress(h, "GetTickCount64");
   }
-  // FreeLibrary(h) ?
+  // We can't call FreeLibrary(h) here, because freeing the handle may
+  // unload the library, and cause future calls to GetTickCount64_fn()
+  // to fail. See 29642 for details.
 }
 
 void
@@ -757,7 +767,7 @@ monotime_coarse_zero(monotime_coarse_t *out)
 {
   memset(out, 0, sizeof(*out));
 }
-#endif
+#endif /* defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 
 int64_t
 monotime_diff_usec(const monotime_t *start,
@@ -787,8 +797,8 @@ monotime_absolute_nsec(void)
   return monotime_diff_nsec(&initialized_at, &now);
 }
 
-uint64_t
-monotime_absolute_usec(void)
+MOCK_IMPL(uint64_t,
+monotime_absolute_usec,(void))
 {
   return monotime_absolute_nsec() / 1000;
 }
@@ -823,7 +833,7 @@ monotime_coarse_absolute_msec(void)
 {
   return monotime_coarse_absolute_nsec() / ONE_MILLION;
 }
-#else
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define initialized_at_coarse initialized_at
 #endif /* defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 
@@ -855,7 +865,7 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
     mach_time_info.numer;
   return abstime_val >> monotime_shift;
 }
-#else
+#else /* !defined(__APPLE__) */
 uint64_t
 monotime_coarse_stamp_units_to_approx_msec(uint64_t units)
 {
@@ -866,4 +876,4 @@ monotime_msec_to_approx_coarse_stamp_units(uint64_t msec)
 {
   return (msec * STAMP_TICKS_PER_SECOND) / 1000;
 }
-#endif
+#endif /* defined(__APPLE__) */
diff --git a/src/lib/time/compat_time.h b/src/lib/time/compat_time.h
index 480d426ac7..5089e16ca5 100644
--- a/src/lib/time/compat_time.h
+++ b/src/lib/time/compat_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -15,6 +15,120 @@
  * of tens of milliseconds.
  */
 
+/* Q: When should I use monotonic time?
+ *
+ * A: If you need a time that never decreases, use monotonic time. If you need
+ * to send a time to a user or another process, or store a time, use the
+ * wall-clock time.
+ *
+ * Q: Should you use monotime or monotime_coarse as your source?
+ *
+ * A: Generally, you get better precision with monotime, but better
+ * performance with monotime_coarse.
+ *
+ * Q: What is a "monotonic" time, exactly?
+ *
+ * A: Monotonic times are strictly non-decreasing. The difference between any
+ * previous monotonic time, and the current monotonic time, is always greater
+ * than *or equal to* zero.
+ * Zero deltas happen more often:
+ *  - on Windows (due to an OS bug),
+ *  - when using monotime_coarse, or on systems with low-resolution timers,
+ *  - on platforms where we emulate monotonic time using wall-clock time, and
+ *  - when using time units that are larger than nanoseconds (due to
+ *    truncation on division).
+ *
+ * Q: Should you use monotime_t or monotime_coarse_t directly? Should you use
+ *    usec? msec? "stamp units?"
+ *
+ * A: Using monotime_t and monotime_coarse_t directly is most time-efficient,
+ * since no conversion needs to happen.  But they can potentially use more
+ * memory than you would need for a usec/msec/"stamp unit" count.
+ *
+ * Converting to usec or msec on some platforms, and working with them in
+ * general, creates a risk of doing a 64-bit division.  64-bit division is
+ * expensive on 32-bit platforms, which still do exist.
+ *
+ * The "stamp unit" type is designed to give a type that is cheap to convert
+ * from monotime_coarse, has resolution of about 1-2ms, and fits nicely in a
+ * 32-bit integer.  Its downside is that it does not correspond directly
+ * to a natural unit of time.
+ *
+ * There is not much point in using "coarse usec" or "coarse nsec", since the
+ * current coarse monotime implementations give you on the order of
+ * milliseconds of precision.
+ *
+ * Q: So, what backends is monotime_coarse using?
+ *
+ * A: Generally speaking, it uses "whatever monotonic-ish time implemenation
+ * does not require a context switch."  The various implementations provide
+ * this by having a view of the current time in a read-only memory page that
+ * is updated with a frequency corresponding to the kernel's tick count.
+ *
+ * On Windows, monotime_coarse uses GetCount64() [or GetTickCount() on
+ * obsolete systems].  MSDN claims that the resolution is "typically in the
+ * range of 10-16 msec", but it has said that for years.  Storing
+ * monotime_coarse_t uses 8 bytes.
+ *
+ * On OSX/iOS, monotime_coarse uses uses mach_approximate_time() where
+ * available, and falls back to regular monotime. The precision is not
+ * documented, but the implementation is open-source: it reads from a page
+ * that the kernel updates. Storing monotime_coarse_t uses 8 bytes.
+ *
+ * On unixy systems, monotime_coarse uses clock_gettime() with
+ * CLOCK_MONOTONIC_COARSE where available, and falls back to CLOCK_MONOTONIC.
+ * It typically uses vdso tricks to read from a page that the kernel updates.
+ * Its precision fixed, but you can get it with clock_getres(): on my Linux
+ * desktop, it claims to be 1 msec, but it will depend on the system HZ
+ * setting. Storing monotime_coarse_t uses 16 bytes.
+ *
+ * [TODO: Try CLOCK_MONOTONIC_FAST on foobsd.]
+ *
+ * Q: What backends is regular monotonic time using?
+ *
+ * A: In general, regular monotime uses something that requires a system call.
+ * On platforms where system calls are cheap, you win!  Otherwise, you lose.
+ *
+ * On Windows, monotonic time uses QuereyPerformanceCounter.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On OSX/Apple, monotonic time uses mach_absolute_time.  Storing
+ * monotime_t costs 8 bytes.
+ *
+ * On unixy systems, monotonic time uses CLOCK_MONOTONIC.  Storing
+ * monotime_t costs 16 bytes.
+ *
+ * Q: Tell me about the costs of converting to a 64-bit nsec, usec, or msec
+ *    count.
+ *
+ * A: Windows, coarse: Cheap, since it's all multiplication.
+ *
+ * Windows, precise: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Apple, all: Expensive on 32-bit: it needs 64-bit division.
+ *
+ * Unixy, all: Fairly cheap, since the only division required is dividing
+ * tv_nsec 1000, and nanoseconds-per-second fits in a 32-bit value.
+ *
+ * All, "timestamp units": Cheap everywhere: it never divides.
+ *
+ * Q: This is only somewhat related, but how much precision could I hope for
+ *    from a libevent time?
+ *
+ * A: Actually, it's _very_ related if you're timing in order to have a
+ * timeout happen.
+ *
+ * On Windows, it uses select: you could in theory have a microsecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On OSX, iOS, and BSD, you have kqueue: You could in theory have a nanosecond
+ * resolution, but it usually isn't that accurate.
+ *
+ * On Linux, you have epoll: It has a millisecond resolution.  Some recent
+ * Libevents can also use timerfd for higher resolution if
+ * EVENT_BASE_FLAG_PRECISE_TIMER is set: Tor doesn't set that flag.
+ */
+
 #ifndef TOR_COMPAT_TIME_H
 #define TOR_COMPAT_TIME_H
 
@@ -86,26 +200,36 @@ void monotime_init(void);
 void monotime_get(monotime_t *out);
 /**
  * Return the number of nanoseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
  */
 int64_t monotime_diff_nsec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of microseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_usec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of milliseconds between <b>start</b> and <b>end</b>.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int64_t monotime_diff_msec(const monotime_t *start, const monotime_t *end);
 /**
  * Return the number of nanoseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
  */
 uint64_t monotime_absolute_nsec(void);
 /**
  * Return the number of microseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
-uint64_t monotime_absolute_usec(void);
+MOCK_DECL(uint64_t, monotime_absolute_usec,(void));
 /**
  * Return the number of milliseconds since the timer system was initialized.
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_absolute_msec(void);
 
@@ -129,10 +253,13 @@ void monotime_add_msec(monotime_t *out, const monotime_t *val, uint32_t msec);
  * Set <b>out</b> to the current coarse time.
  */
 void monotime_coarse_get(monotime_coarse_t *out);
+/**
+ * Like monotime_absolute_*(), but faster on some platforms.
+ */
 uint64_t monotime_coarse_absolute_nsec(void);
 uint64_t monotime_coarse_absolute_usec(void);
 uint64_t monotime_coarse_absolute_msec(void);
-#else /* !(defined(MONOTIME_COARSE_FN_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_FN_IS_DIFFERENT) */
 #define monotime_coarse_get monotime_get
 #define monotime_coarse_absolute_nsec monotime_absolute_nsec
 #define monotime_coarse_absolute_usec monotime_absolute_usec
@@ -152,23 +279,32 @@ uint32_t monotime_coarse_to_stamp(const monotime_coarse_t *t);
 /**
  * Convert a difference, expressed in the units of monotime_coarse_to_stamp,
  * into an approximate number of milliseconds.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 uint64_t monotime_coarse_stamp_units_to_approx_msec(uint64_t units);
 uint64_t monotime_msec_to_approx_coarse_stamp_units(uint64_t msec);
 uint32_t monotime_coarse_get_stamp(void);
 
 #if defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)
+/**
+ * Like monotime_diff_*(), but faster on some platforms.
+ */
 int64_t monotime_coarse_diff_nsec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_usec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
 int64_t monotime_coarse_diff_msec(const monotime_coarse_t *start,
     const monotime_coarse_t *end);
+/**
+ * Like monotime_*(), but faster on some platforms.
+ */
 void monotime_coarse_zero(monotime_coarse_t *out);
 int monotime_coarse_is_zero(const monotime_coarse_t *val);
 void monotime_coarse_add_msec(monotime_coarse_t *out,
                               const monotime_coarse_t *val, uint32_t msec);
-#else /* !(defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT)) */
+#else /* !defined(MONOTIME_COARSE_TYPE_IS_DIFFERENT) */
 #define monotime_coarse_diff_nsec monotime_diff_nsec
 #define monotime_coarse_diff_usec monotime_diff_usec
 #define monotime_coarse_diff_msec monotime_diff_msec
@@ -182,6 +318,9 @@ void monotime_coarse_add_msec(monotime_coarse_t *out,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
                                      const monotime_coarse_t *end);
@@ -191,6 +330,9 @@ int32_t monotime_coarse_diff_msec32_(const monotime_coarse_t *start,
  *
  * Requires that the difference fit into an int32_t; not for use with
  * large time differences.
+ *
+ * The returned value may be equal to zero.
+ * Fractional units are truncated, not rounded.
  */
 static inline int32_t
 monotime_coarse_diff_msec32(const monotime_coarse_t *start,
@@ -202,7 +344,7 @@ monotime_coarse_diff_msec32(const monotime_coarse_t *start,
 #else
 #define USING_32BIT_MSEC_HACK
   return monotime_coarse_diff_msec32_(start, end);
-#endif
+#endif /* SIZEOF_VOID_P == 8 */
 }
 
 #ifdef TOR_UNIT_TESTS
diff --git a/src/lib/time/include.am b/src/lib/time/include.am
index a3f93a3744..dcb199b142 100644
--- a/src/lib/time/include.am
+++ b/src/lib/time/include.am
@@ -5,8 +5,10 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-time-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_time_a_SOURCES =	\
 		src/lib/time/compat_time.c	\
+		src/lib/time/time_sys.c		\
 		src/lib/time/tvdiff.c
 
 src_lib_libtor_time_testing_a_SOURCES = \
@@ -14,6 +16,8 @@ src_lib_libtor_time_testing_a_SOURCES = \
 src_lib_libtor_time_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_time_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 		src/lib/time/compat_time.h	\
+		src/lib/time/time_sys.h		\
 		src/lib/time/tvdiff.h
diff --git a/src/lib/time/lib_time.md b/src/lib/time/lib_time.md
new file mode 100644
index 0000000000..8e58aafcd8
--- /dev/null
+++ b/src/lib/time/lib_time.md
@@ -0,0 +1,9 @@
+@dir /lib/time
+@brief lib/time: Higher-level time functions
+
+This includes both fine-grained timers and monotonic timers, along with
+wrappers for them to try to improve efficiency.
+
+For "what time is it" in UTC, see \refdir{lib/wallclock}.  For parsing and
+encoding times and dates, see \refdir{lib/encoding}.
+
diff --git a/src/lib/time/time_sys.c b/src/lib/time/time_sys.c
new file mode 100644
index 0000000000..1c1bc4cd18
--- /dev/null
+++ b/src/lib/time/time_sys.c
@@ -0,0 +1,29 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.c
+ * \brief Subsystem object for monotime setup.
+ **/
+
+#include "orconfig.h"
+#include "lib/subsys/subsys.h"
+#include "lib/time/time_sys.h"
+#include "lib/time/compat_time.h"
+
+static int
+subsys_time_initialize(void)
+{
+  monotime_init();
+  return 0;
+}
+
+const subsys_fns_t sys_time = {
+  .name = "time",
+  SUBSYS_DECLARE_LOCATION(),
+  /* Monotonic time depends on logging, and a lot of other modules depend on
+   * monotonic time. */
+  .level = -80,
+  .supported = true,
+  .initialize = subsys_time_initialize,
+};
diff --git a/src/lib/time/time_sys.h b/src/lib/time/time_sys.h
new file mode 100644
index 0000000000..f2401e1911
--- /dev/null
+++ b/src/lib/time/time_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file time_sys.h
+ * \brief Declare subsystem object for the time module.
+ **/
+
+#ifndef TOR_TIME_SYS_H
+#define TOR_TIME_SYS_H
+
+extern const struct subsys_fns_t sys_time;
+
+#endif /* !defined(TOR_TIME_SYS_H) */
diff --git a/src/lib/time/tvdiff.c b/src/lib/time/tvdiff.c
index a87d0d96dc..cbad5a48b8 100644
--- a/src/lib/time/tvdiff.c
+++ b/src/lib/time/tvdiff.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,6 +11,7 @@
 #include "lib/time/tvdiff.h"
 
 #include "lib/cc/compat_compiler.h"
+#include "lib/defs/time.h"
 #include "lib/log/log.h"
 
 #ifdef _WIN32
@@ -20,8 +21,6 @@
 #include <sys/time.h>
 #endif
 
-#define TOR_USEC_PER_SEC 1000000
-
 /** Return the difference between start->tv_sec and end->tv_sec.
  * Returns INT64_MAX on overflow and underflow.
  */
diff --git a/src/lib/time/tvdiff.h b/src/lib/time/tvdiff.h
index 724af1528a..e779e758f1 100644
--- a/src/lib/time/tvdiff.h
+++ b/src/lib/time/tvdiff.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,4 +20,4 @@ int64_t tv_to_msec(const struct timeval *tv);
 
 time_t time_diff(const time_t from, const time_t to);
 
-#endif
+#endif /* !defined(TOR_TVDIFF_H) */
diff --git a/src/lib/tls/.may_include b/src/lib/tls/.may_include
index 2840e590b8..c550bde024 100644
--- a/src/lib/tls/.may_include
+++ b/src/lib/tls/.may_include
@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
@@ -11,7 +12,7 @@ lib/log/*.h
 lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/subsys/*.h
+lib/testsupport/*.h
 lib/tls/*.h
-
-ciphers.inc
+lib/tls/*.inc
diff --git a/src/lib/tls/buffers_tls.c b/src/lib/tls/buffers_tls.c
index b570216df0..b92a14d6a1 100644
--- a/src/lib/tls/buffers_tls.c
+++ b/src/lib/tls/buffers_tls.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,7 +12,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
@@ -68,10 +68,10 @@ buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
 
   check_no_tls_errors();
 
-  IF_BUG_ONCE(buf->datalen >= INT_MAX)
-    return -1;
-  IF_BUG_ONCE(buf->datalen >= INT_MAX - at_most)
-    return -1;
+  IF_BUG_ONCE(buf->datalen > BUF_MAX_LEN)
+    return TOR_TLS_ERROR_MISC;
+  IF_BUG_ONCE(buf->datalen > BUF_MAX_LEN - at_most)
+    return TOR_TLS_ERROR_MISC;
 
   while (at_most > total_read) {
     size_t readlen = at_most - total_read;
@@ -90,7 +90,7 @@ buf_read_from_tls(buf_t *buf, tor_tls_t *tls, size_t at_most)
     r = read_to_chunk_tls(buf, chunk, tls, readlen);
     if (r < 0)
       return r; /* Error */
-    tor_assert(total_read+r < INT_MAX);
+    tor_assert(total_read+r <= BUF_MAX_LEN);
     total_read += r;
     if ((size_t)r < readlen) /* eof, block, or no more to read. */
       break;
@@ -177,6 +177,6 @@ buf_flush_to_tls(buf_t *buf, tor_tls_t *tls, size_t flushlen,
     if (r == 0) /* Can't flush any more now. */
       break;
   } while (sz > 0);
-  tor_assert(flushed < INT_MAX);
+  tor_assert(flushed <= BUF_MAX_LEN);
   return (int)flushed;
 }
diff --git a/src/lib/tls/buffers_tls.h b/src/lib/tls/buffers_tls.h
index 65788c3f34..587426801d 100644
--- a/src/lib/tls/buffers_tls.h
+++ b/src/lib/tls/buffers_tls.h
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/tls/include.am b/src/lib/tls/include.am
index a664b29fb2..7e05ef4f8c 100644
--- a/src/lib/tls/include.am
+++ b/src/lib/tls/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-tls-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_tls_a_SOURCES =			\
 	src/lib/tls/buffers_tls.c		\
 	src/lib/tls/tortls.c			\
@@ -29,6 +30,7 @@ src_lib_libtor_tls_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_tls_testing_a_CFLAGS = \
 	$(AM_CFLAGS) $(TOR_CFLAGS_CRYPTLIB) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=				\
 	src/lib/tls/ciphers.inc			\
 	src/lib/tls/buffers_tls.h		\
@@ -36,5 +38,6 @@ noinst_HEADERS +=				\
 	src/lib/tls/tortls.h			\
 	src/lib/tls/tortls_internal.h		\
 	src/lib/tls/tortls_st.h			\
+	src/lib/tls/tortls_sys.h		\
 	src/lib/tls/x509.h			\
 	src/lib/tls/x509_internal.h
diff --git a/src/lib/tls/lib_tls.md b/src/lib/tls/lib_tls.md
new file mode 100644
index 0000000000..26fea723f9
--- /dev/null
+++ b/src/lib/tls/lib_tls.md
@@ -0,0 +1,11 @@
+@dir /lib/tls
+@brief lib/tls: TLS library wrappers
+
+This module has compatibility wrappers around the library (NSS or OpenSSL,
+depending on configuration) that Tor uses to implement the TLS link security
+protocol.
+
+It also implements the logic for some legacy TLS protocol usage we used to
+support in old versions of Tor, involving conditional delivery of certificate
+chains (v1 link protocol) and conditional renegotiation (v2 link protocol).
+
diff --git a/src/lib/tls/nss_countbytes.c b/src/lib/tls/nss_countbytes.c
index 7761727acd..4b98df80ec 100644
--- a/src/lib/tls/nss_countbytes.c
+++ b/src/lib/tls/nss_countbytes.c
@@ -1,4 +1,4 @@
-/* Copyright 2018-2019, The Tor Project Inc. */
+/* Copyright 2018-2020, The Tor Project Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/tls/nss_countbytes.h b/src/lib/tls/nss_countbytes.h
index 8b31603923..36ed55e10d 100644
--- a/src/lib/tls/nss_countbytes.h
+++ b/src/lib/tls/nss_countbytes.h
@@ -1,4 +1,4 @@
-/* Copyright 2018-2019, The Tor Project, Inc. */
+/* Copyright 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,4 @@ int tor_get_prfiledesc_byte_counts(struct PRFileDesc *fd,
                                    uint64_t *n_read_out,
                                    uint64_t *n_written_out);
 
-#endif
+#endif /* !defined(TOR_NSS_COUNTBYTES_H) */
diff --git a/src/lib/tls/tortls.c b/src/lib/tls/tortls.c
index 4ca7c7d9d3..9e70e54725 100644
--- a/src/lib/tls/tortls.c
+++ b/src/lib/tls/tortls.c
@@ -1,12 +1,18 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tortls.c
+ * @brief Shared functionality for our TLS backends.
+ **/
+
 #define TORTLS_PRIVATE
 #define TOR_X509_PRIVATE
 #include "lib/tls/x509.h"
 #include "lib/tls/x509_internal.h"
+#include "lib/tls/tortls_sys.h"
 #include "lib/tls/tortls.h"
 #include "lib/tls/tortls_st.h"
 #include "lib/tls/tortls_internal.h"
@@ -15,6 +21,7 @@
 #include "lib/crypt_ops/crypto_rsa.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/net/socket.h"
+#include "lib/subsys/subsys.h"
 
 #ifdef _WIN32
   #include <winsock2.h>
@@ -440,3 +447,16 @@ tor_tls_verify(int severity, tor_tls_t *tls, crypto_pk_t **identity)
 
   return rv;
 }
+
+static void
+subsys_tortls_shutdown(void)
+{
+  tor_tls_free_all();
+}
+
+const subsys_fns_t sys_tortls = {
+  .name = "tortls",
+  SUBSYS_DECLARE_LOCATION(),
+  .level = -50,
+  .shutdown = subsys_tortls_shutdown
+};
diff --git a/src/lib/tls/tortls.h b/src/lib/tls/tortls.h
index 9398318035..3703ecad56 100644
--- a/src/lib/tls/tortls.h
+++ b/src/lib/tls/tortls.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TORTLS_H
@@ -25,12 +25,12 @@ struct ssl_ctx_st;
 struct ssl_session_st;
 typedef struct ssl_ctx_st tor_tls_context_impl_t;
 typedef struct ssl_st tor_tls_impl_t;
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 struct PRFileDesc;
 typedef struct PRFileDesc tor_tls_context_impl_t;
 typedef struct PRFileDesc tor_tls_impl_t;
-#endif
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+#endif /* defined(TORTLS_PRIVATE) */
 
 struct tor_x509_cert_t;
 
@@ -81,6 +81,7 @@ void tor_tls_free_all(void);
 void tor_tls_init(void);
 void tls_log_errors(tor_tls_t *tls, int severity, int domain,
                     const char *doing);
+const char *tor_tls_get_last_error_msg(const tor_tls_t *tls);
 int tor_tls_context_init(unsigned flags,
                          crypto_pk_t *client_identity,
                          crypto_pk_t *server_identity,
@@ -143,9 +144,9 @@ void check_no_tls_errors_(const char *fname, int line);
 
 void tor_tls_log_one_error(tor_tls_t *tls, unsigned long err,
                            int severity, int domain, const char *doing);
-#else
+#else /* !defined(ENABLE_OPENSSL) */
 #define check_no_tls_errors() STMT_NIL
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 int tor_tls_get_my_certs(int server,
                          const struct tor_x509_cert_t **link_cert_out,
diff --git a/src/lib/tls/tortls_internal.h b/src/lib/tls/tortls_internal.h
index 071c506561..3f56f181ee 100644
--- a/src/lib/tls/tortls_internal.h
+++ b/src/lib/tls/tortls_internal.h
@@ -1,11 +1,18 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+/**
+ * @file tortls_internal.h
+ * @brief Declare internal functions for lib/tls
+ **/
+
 #ifndef TORTLS_INTERNAL_H
 #define TORTLS_INTERNAL_H
 
+#include "lib/tls/x509.h"
+
 int tor_errno_to_tls_error(int e);
 #ifdef ENABLE_OPENSSL
 int tor_tls_get_error(tor_tls_t *tls, int r, int extra,
@@ -61,8 +68,8 @@ STATIC int tor_tls_session_secret_cb(struct ssl_st *ssl, void *secret,
                             void *arg);
 STATIC int find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m,
                              uint16_t cipher);
-#endif
-#endif
+#endif /* defined(TORTLS_OPENSSL_PRIVATE) */
+#endif /* defined(ENABLE_OPENSSL) */
 
 #ifdef TOR_UNIT_TESTS
 extern int tor_tls_object_ex_data_index;
@@ -73,4 +80,4 @@ extern uint64_t total_bytes_written_over_tls;
 extern uint64_t total_bytes_written_by_tls;
 #endif /* defined(TOR_UNIT_TESTS) */
 
-#endif /* defined(TORTLS_INTERNAL_H) */
+#endif /* !defined(TORTLS_INTERNAL_H) */
diff --git a/src/lib/tls/tortls_nss.c b/src/lib/tls/tortls_nss.c
index b7f3513150..25446a81af 100644
--- a/src/lib/tls/tortls_nss.c
+++ b/src/lib/tls/tortls_nss.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,7 +34,7 @@
 #include "lib/tls/nss_countbytes.h"
 #include "lib/log/util_bug.h"
 
-DISABLE_GCC_WARNING(strict-prototypes)
+DISABLE_GCC_WARNING("-Wstrict-prototypes")
 #include <prio.h>
 // For access to rar sockets.
 #include <private/pprio.h>
@@ -42,7 +42,7 @@ DISABLE_GCC_WARNING(strict-prototypes)
 #include <sslt.h>
 #include <sslproto.h>
 #include <certt.h>
-ENABLE_GCC_WARNING(strict-prototypes)
+ENABLE_GCC_WARNING("-Wstrict-prototypes")
 
 static SECStatus always_accept_cert_cb(void *, PRFileDesc *, PRBool, PRBool);
 
@@ -369,6 +369,8 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain,
 
   (void)tls;
   PRErrorCode code = PORT_GetError();
+  if (tls)
+    tls->last_error = code;
 
   const char *addr = tls ? tls->address : NULL;
   const char *string = PORT_ErrorToString(code);
@@ -391,6 +393,17 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain,
            with, addr);
   }
 }
+const char *
+tor_tls_get_last_error_msg(const tor_tls_t *tls)
+{
+  IF_BUG_ONCE(!tls) {
+    return NULL;
+  }
+  if (tls->last_error == 0) {
+    return NULL;
+  }
+  return PORT_ErrorToString((PRErrorCode)tls->last_error);
+}
 
 tor_tls_t *
 tor_tls_new(tor_socket_t sock, int is_server)
diff --git a/src/lib/tls/tortls_openssl.c b/src/lib/tls/tortls_openssl.c
index c5031a00aa..5c92df1259 100644
--- a/src/lib/tls/tortls_openssl.c
+++ b/src/lib/tls/tortls_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -25,7 +25,7 @@
    * <winsock.h> and mess things up, in at least some openssl versions. */
   #include <winsock2.h>
   #include <ws2tcpip.h>
-#endif
+#endif /* defined(_WIN32) */
 
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rand.h"
@@ -37,7 +37,7 @@
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/opensslv.h>
 
@@ -54,7 +54,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "lib/tls/tortls.h"
 #include "lib/tls/tortls_st.h"
@@ -245,10 +245,30 @@ tls_log_errors(tor_tls_t *tls, int severity, int domain, const char *doing)
   unsigned long err;
 
   while ((err = ERR_get_error()) != 0) {
+    if (tls)
+      tls->last_error = err;
     tor_tls_log_one_error(tls, err, severity, domain, doing);
   }
 }
 
+/**
+ * Return a string representing more detail about the last error received
+ * on TLS.
+ *
+ * May return null if no error was found.
+ **/
+const char *
+tor_tls_get_last_error_msg(const tor_tls_t *tls)
+{
+  IF_BUG_ONCE(!tls) {
+    return NULL;
+  }
+  if (tls->last_error == 0) {
+    return NULL;
+  }
+  return (const char*)ERR_reason_error_string(tls->last_error);
+}
+
 #define CATCH_SYSCALL 1
 #define CATCH_ZERO    2
 
@@ -318,7 +338,7 @@ tor_tls_init(void)
 #else
     SSL_library_init();
     SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
@@ -383,7 +403,7 @@ static const char SERVER_CIPHER_LIST[] =
    * conclude that it has no valid ciphers if it's running with TLS1.3.
    */
   TLS1_3_TXT_AES_128_GCM_SHA256 ":"
-#endif
+#endif /* defined(TLS1_3_TXT_AES_128_GCM_SHA256) */
   TLS1_TXT_DHE_RSA_WITH_AES_256_SHA ":"
   TLS1_TXT_DHE_RSA_WITH_AES_128_SHA;
 
@@ -464,7 +484,9 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 /** List of ciphers that clients should advertise, omitting items that
  * our OpenSSL doesn't know about. */
 static const char CLIENT_CIPHER_LIST[] =
-#include "ciphers.inc"
+#ifndef COCCI
+#include "lib/tls/ciphers.inc"
+#endif
   /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version
    * of any cipher we say. */
   "!SSLv2"
@@ -657,7 +679,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
     if (r < 0)
       goto error;
   }
-#else
+#else /* !(defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SE...)) */
   if (! is_client) {
     int nid;
     EC_KEY *ec_key;
@@ -673,7 +695,7 @@ tor_tls_context_new(crypto_pk_t *identity, unsigned int key_lifetime,
       SSL_CTX_set_tmp_ecdh(result->ctx, ec_key);
     EC_KEY_free(ec_key);
   }
-#endif
+#endif /* defined(SSL_CTX_set1_groups_list) || defined(HAVE_SSL_CTX_SET1...) */
   SSL_CTX_set_verify(result->ctx, SSL_VERIFY_PEER,
                      always_accept_verify_cb);
   /* let us realloc bufs that we're writing from */
@@ -764,7 +786,7 @@ find_cipher_by_id(const SSL *ssl, const SSL_METHOD *m, uint16_t cipher)
       tor_assert((SSL_CIPHER_get_id(c) & 0xffff) == cipher);
     return c != NULL;
   }
-#else /* !(defined(HAVE_SSL_CIPHER_FIND)) */
+#else /* !defined(HAVE_SSL_CIPHER_FIND) */
 
 # if defined(HAVE_STRUCT_SSL_METHOD_ST_GET_CIPHER_BY_CHAR)
   if (m && m->get_cipher_by_char) {
@@ -1062,7 +1084,7 @@ tor_tls_new(tor_socket_t sock, int isServer)
     /* We can't actually use TLS 1.3 until this bug is fixed. */
     SSL_set_max_proto_version(result->ssl, TLS1_2_VERSION);
   }
-#endif
+#endif /* defined(SSL_CTRL_SET_MAX_PROTO_VERSION) */
 
   if (!SSL_set_cipher_list(result->ssl,
                      isServer ? SERVER_CIPHER_LIST : CLIENT_CIPHER_LIST)) {
@@ -1715,7 +1737,7 @@ tor_tls_export_key_material,(tor_tls_t *tls, uint8_t *secrets_out,
     else
       return -1;
   }
-#endif
+#endif /* defined(TLS1_3_VERSION) */
 
   return (r == 1) ? 0 : -1;
 }
diff --git a/src/lib/tls/tortls_st.h b/src/lib/tls/tortls_st.h
index 3f7ea8ac6a..34abe52ee3 100644
--- a/src/lib/tls/tortls_st.h
+++ b/src/lib/tls/tortls_st.h
@@ -1,11 +1,19 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TORTLS_ST_H
 #define TOR_TORTLS_ST_H
 
+/**
+ * @file tortls_st.h
+ * @brief Structure declarations for internal TLS types.
+ *
+ * These should generally be treated as opaque outside of the
+ * lib/tls module.
+ **/
+
 #include "lib/net/socket.h"
 
 #define TOR_TLS_MAGIC 0x71571571
@@ -59,17 +67,20 @@ struct tor_tls_t {
    */
   unsigned long last_write_count;
   unsigned long last_read_count;
+  /** Most recent error value from ERR_get_error(). */
+  unsigned long last_error;
   /** If set, a callback to invoke whenever the client tries to renegotiate
    * the handshake. */
   void (*negotiated_callback)(tor_tls_t *tls, void *arg);
   /** Argument to pass to negotiated_callback. */
   void *callback_arg;
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 #ifdef ENABLE_NSS
   /** Last values retried from tor_get_prfiledesc_byte_counts(). */
   uint64_t last_write_count;
   uint64_t last_read_count;
+  long last_error;
 #endif
 };
 
-#endif
+#endif /* !defined(TOR_TORTLS_ST_H) */
diff --git a/src/lib/tls/tortls_sys.h b/src/lib/tls/tortls_sys.h
new file mode 100644
index 0000000000..177c198f71
--- /dev/null
+++ b/src/lib/tls/tortls_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file tortls_sys.h
+ * \brief Declare subsystem object for the tortls module
+ **/
+
+#ifndef TOR_TORTLS_SYS_H
+#define TOR_TORTLS_SYS_H
+
+extern const struct subsys_fns_t sys_tortls;
+
+#endif /* !defined(TOR_TORTLS_SYS_H) */
diff --git a/src/lib/tls/x509.c b/src/lib/tls/x509.c
index b4a0f8dabf..2515499298 100644
--- a/src/lib/tls/x509.c
+++ b/src/lib/tls/x509.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/tls/x509.h b/src/lib/tls/x509.h
index 5e6660de5c..5919b9089d 100644
--- a/src/lib/tls/x509.h
+++ b/src/lib/tls/x509.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_X509_H
@@ -35,7 +35,7 @@ struct tor_x509_cert_t {
   common_digests_t cert_digests;
   common_digests_t pkey_digests;
 };
-#endif
+#endif /* defined(TOR_X509_PRIVATE) */
 
 void tor_tls_pick_certificate_lifetime(time_t now,
                                        unsigned cert_lifetime,
@@ -47,7 +47,7 @@ tor_x509_cert_t *tor_x509_cert_replace_expiration(
                                                const tor_x509_cert_t *inp,
                                                time_t new_expiration_time,
                                                crypto_pk_t *signing_key);
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
 
 tor_x509_cert_t *tor_x509_cert_dup(const tor_x509_cert_t *cert);
 
@@ -72,4 +72,4 @@ int tor_tls_cert_is_valid(int severity,
                           time_t now,
                           int check_rsa_1024);
 
-#endif
+#endif /* !defined(TOR_X509_H) */
diff --git a/src/lib/tls/x509_internal.h b/src/lib/tls/x509_internal.h
index bf2bec9689..145be7e71c 100644
--- a/src/lib/tls/x509_internal.h
+++ b/src/lib/tls/x509_internal.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_X509_INTERNAL_H
@@ -50,4 +50,4 @@ int tor_x509_cert_set_cached_der_encoding(tor_x509_cert_t *cert);
 #define tor_x509_cert_set_cached_der_encoding(cert) (0)
 #endif
 
-#endif
+#endif /* !defined(TOR_X509_INTERNAL_H) */
diff --git a/src/lib/tls/x509_nss.c b/src/lib/tls/x509_nss.c
index fb4af54c52..341bb57104 100644
--- a/src/lib/tls/x509_nss.c
+++ b/src/lib/tls/x509_nss.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -120,13 +120,13 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                      der.data, der.len,
                                      (SECKEYPrivateKey *)signing_key,//const
                                      &cert->signature);
-#else
+#else /* !(0) */
   s = SEC_DerSignData(cert->arena,
                       &signed_der,
                       der.data, der.len,
                       (SECKEYPrivateKey *)signing_key,//const
                       SEC_OID_PKCS1_SHA256_WITH_RSA_ENCRYPTION);
-#endif
+#endif /* 0 */
 
   if (s != SECSuccess)
     goto err;
@@ -145,7 +145,7 @@ tor_tls_create_certificate_internal(crypto_pk_t *rsa,
                                &result_cert->signatureWrap, issuer_pk, NULL);
     tor_assert(cert_ok == SECSuccess);
   }
-#endif
+#endif /* 1 */
 
  err:
   if (subject_spki)
@@ -455,4 +455,4 @@ tor_x509_cert_replace_expiration(const tor_x509_cert_t *inp,
 
   return newcert ? tor_x509_cert_new(newcert) : NULL;
 }
-#endif
+#endif /* defined(TOR_UNIT_TESTS) */
diff --git a/src/lib/tls/x509_openssl.c b/src/lib/tls/x509_openssl.c
index a344279c22..2abf02851d 100644
--- a/src/lib/tls/x509_openssl.c
+++ b/src/lib/tls/x509_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,7 +19,7 @@
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/opensslv.h>
 
@@ -36,7 +36,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/rsa.h>
 #include <openssl/x509.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
@@ -59,12 +59,12 @@ ENABLE_GCC_WARNING(redundant-decls)
 #define X509_get_notAfter(cert) \
     X509_getm_notAfter(cert)
 #endif
-#else /* ! OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0) */
+#else /* !defined(OPENSSL_1_1_API) */
 #define X509_get_notBefore_const(cert) \
   ((const ASN1_TIME*) X509_get_notBefore((X509 *)cert))
 #define X509_get_notAfter_const(cert) \
   ((const ASN1_TIME*) X509_get_notAfter((X509 *)cert))
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 
 /** Return a newly allocated X509 name with commonName <b>cname</b>. */
 static X509_NAME *
diff --git a/src/lib/trace/debug.h b/src/lib/trace/debug.h
index e35616cf50..87b3074e0b 100644
--- a/src/lib/trace/debug.h
+++ b/src/lib/trace/debug.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -27,4 +27,4 @@
                         "\"" XSTR(subsystem) "\" hit. " \
                         "(line "XSTR(__LINE__) ")")
 
-#endif /* TOR_TRACE_LOG_DEBUG_H */
+#endif /* !defined(TOR_TRACE_LOG_DEBUG_H) */
diff --git a/src/lib/trace/events.h b/src/lib/trace/events.h
index 1e1e7b9d16..368f85dd02 100644
--- a/src/lib/trace/events.h
+++ b/src/lib/trace/events.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -34,12 +34,12 @@
 #include "lib/trace/debug.h"
 #endif
 
-#else /* TOR_EVENT_TRACING_ENABLED */
+#else /* !defined(TOR_EVENT_TRACING_ENABLED) */
 
 /* Reaching this point, we NOP every event declaration because event tracing
  * is not been enabled at compile time. */
 #define tor_trace(subsystem, name, args...)
 
-#endif /* TOR_EVENT_TRACING_ENABLED */
+#endif /* defined(TOR_EVENT_TRACING_ENABLED) */
 
-#endif /* TOR_TRACE_EVENTS_H */
+#endif /* !defined(TOR_TRACE_EVENTS_H) */
diff --git a/src/lib/trace/include.am b/src/lib/trace/include.am
index 6f10c98744..98098c87f4 100644
--- a/src/lib/trace/include.am
+++ b/src/lib/trace/include.am
@@ -2,6 +2,7 @@
 noinst_LIBRARIES += \
 	src/lib/libtor-trace.a
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 TRACEHEADERS = \
 	src/lib/trace/trace.h \
 	src/lib/trace/events.h
@@ -11,7 +12,7 @@ TRACEHEADERS += \
 	src/lib/trace/debug.h
 endif
 
-# Library source files.
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_trace_a_SOURCES = \
 	src/lib/trace/trace.c
 
diff --git a/src/lib/trace/lib_trace.md b/src/lib/trace/lib_trace.md
new file mode 100644
index 0000000000..a7a32529b0
--- /dev/null
+++ b/src/lib/trace/lib_trace.md
@@ -0,0 +1,6 @@
+@dir /lib/trace
+@brief lib/trace: Function-tracing functionality API.
+
+This module is used for adding "trace" support (low-granularity function
+logging) to Tor.  Right now it doesn't have many users.
+
diff --git a/src/lib/trace/trace.c b/src/lib/trace/trace.c
index 18be63c5a8..4e5c66b4c6 100644
--- a/src/lib/trace/trace.c
+++ b/src/lib/trace/trace.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/trace/trace.h b/src/lib/trace/trace.h
index 606d435568..5e24678c3c 100644
--- a/src/lib/trace/trace.h
+++ b/src/lib/trace/trace.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -11,4 +11,4 @@
 
 void tor_trace_init(void);
 
-#endif // TOR_TRACE_TRACE_H
+#endif /* !defined(TOR_TRACE_TRACE_H) */
diff --git a/src/lib/version/.may_include b/src/lib/version/.may_include
new file mode 100644
index 0000000000..d159ceb41f
--- /dev/null
+++ b/src/lib/version/.may_include
@@ -0,0 +1,3 @@
+orconfig.h
+micro-revision.i
+lib/version/*.h
\ No newline at end of file
diff --git a/src/lib/log/git_revision.c b/src/lib/version/git_revision.c
index 7d27549cad..09f11aa316 100644
--- a/src/lib/log/git_revision.c
+++ b/src/lib/version/git_revision.c
@@ -1,24 +1,39 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#include "lib/log/git_revision.h"
+#include "lib/version/git_revision.h"
+
+/**
+ * @file git_revision.c
+ * @brief Strings to describe the current Git commit.
+ **/
 
 /** String describing which Tor Git repository version the source was
  * built from.  This string is generated by a bit of shell kludging in
  * src/core/include.am, and is usually right.
  */
 const char tor_git_revision[] =
+#ifndef COCCI
 #ifndef _MSC_VER
 #include "micro-revision.i"
 #endif
+#endif
   "";
 
+/**
+ * String appended to Tor bug messages describing the Tor version.
+ *
+ * It has the form "(on Tor 0.4.3.1-alpha)" or
+ * "(on Tor 0.4.3.1-alpha git-b994397f1af193f8)"
+ **/
 const char tor_bug_suffix[] = " (on Tor " VERSION
+#ifndef COCCI
 #ifndef _MSC_VER
   " "
 #include "micro-revision.i"
 #endif
+#endif /* !defined(COCCI) */
   ")";
diff --git a/src/lib/log/git_revision.h b/src/lib/version/git_revision.h
index 79e3c6684b..80b6c4734e 100644
--- a/src/lib/log/git_revision.h
+++ b/src/lib/version/git_revision.h
@@ -1,11 +1,16 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_GIT_REVISION_H
 #define TOR_GIT_REVISION_H
 
+/**
+ * @file git_revision.h
+ * @brief Header for git_revision.c
+ **/
+
 extern const char tor_git_revision[];
 extern const char tor_bug_suffix[];
 
diff --git a/src/lib/version/include.am b/src/lib/version/include.am
new file mode 100644
index 0000000000..0ae31be1b2
--- /dev/null
+++ b/src/lib/version/include.am
@@ -0,0 +1,27 @@
+
+noinst_LIBRARIES += src/lib/libtor-version.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-version-testing.a
+endif
+
+# ADD_C_FILE: INSERT SOURCES HERE.
+src_lib_libtor_version_a_SOURCES =			\
+	src/lib/version/git_revision.c			\
+	src/lib/version/version.c
+
+src_lib_libtor_version_testing_a_SOURCES = \
+	$(src_lib_libtor_version_a_SOURCES)
+src_lib_libtor_version_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_version_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+# Declare that these object files depend on micro-revision.i.  Without this
+# rule, we could try to build them before micro-revision.i was created.
+src/lib/version/git_revision.$(OBJEXT) \
+  src/lib/version/src_lib_libtor_version_testing_a-git_revision.$(OBJEXT): \
+	micro-revision.i
+
+# ADD_C_FILE: INSERT HEADERS HERE.
+noinst_HEADERS +=					\
+	src/lib/version/git_revision.h			\
+	src/lib/version/torversion.h
diff --git a/src/lib/version/lib_version.md b/src/lib/version/lib_version.md
new file mode 100644
index 0000000000..ccc45920f9
--- /dev/null
+++ b/src/lib/version/lib_version.md
@@ -0,0 +1,2 @@
+@dir /lib/version
+@brief lib/version: holds the current version of Tor.
diff --git a/src/lib/version/torversion.h b/src/lib/version/torversion.h
new file mode 100644
index 0000000000..679df74381
--- /dev/null
+++ b/src/lib/version/torversion.h
@@ -0,0 +1,17 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_VERSION_H
+#define TOR_VERSION_H
+
+/**
+ * @file torversion.h
+ * @brief Header for version.c.
+ **/
+
+const char *get_version(void);
+const char *get_short_version(void);
+
+#endif /* !defined(TOR_VERSION_H) */
diff --git a/src/lib/version/version.c b/src/lib/version/version.c
new file mode 100644
index 0000000000..ec1d0bea2f
--- /dev/null
+++ b/src/lib/version/version.c
@@ -0,0 +1,59 @@
+/* Copyright 2001-2004 Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "lib/version/torversion.h"
+#include "lib/version/git_revision.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * @file version.c
+ * @brief Functions to get the version of Tor.
+ **/
+
+/** A shorter version of this Tor process's version, for export in our router
+ *  descriptor.  (Does not include the git version, if any.) */
+static const char the_short_tor_version[] =
+  VERSION
+#ifdef TOR_BUILD_TAG
+  " ("TOR_BUILD_TAG")"
+#endif
+  "";
+
+/**
+ * Longest possible version length. We make this a constant so that we
+ * can statically allocate the_tor_version.
+ **/
+#define MAX_VERSION_LEN 128
+
+/** The version of this Tor process, possibly including git version */
+static char the_tor_version[MAX_VERSION_LEN] = "";
+
+/** Return the current Tor version. */
+const char *
+get_version(void)
+{
+  if (the_tor_version[0] == 0) {
+    if (strlen(tor_git_revision)) {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s (git-%s)", the_short_tor_version, tor_git_revision);
+    } else {
+      snprintf(the_tor_version, sizeof(the_tor_version),
+               "%s", the_short_tor_version);
+    }
+    the_tor_version[sizeof(the_tor_version)-1] = 0;
+  }
+
+  return the_tor_version;
+}
+
+/** Return the current Tor version, without any git tag. */
+const char *
+get_short_version(void)
+{
+  return the_short_tor_version;
+}
diff --git a/src/lib/wallclock/.may_include b/src/lib/wallclock/.may_include
index dc010da063..ce7a26472b 100644
--- a/src/lib/wallclock/.may_include
+++ b/src/lib/wallclock/.may_include
@@ -3,4 +3,5 @@ lib/cc/*.h
 lib/err/*.h
 lib/wallclock/*.h
 lib/string/*.h
+lib/subsys/*.h
 lib/testsupport/*.h
diff --git a/src/lib/wallclock/approx_time.c b/src/lib/wallclock/approx_time.c
index ee498702d5..c815f20e51 100644
--- a/src/lib/wallclock/approx_time.c
+++ b/src/lib/wallclock/approx_time.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,7 +9,9 @@
  **/
 
 #include "orconfig.h"
+#include "lib/subsys/subsys.h"
 #include "lib/wallclock/approx_time.h"
+#include "lib/wallclock/wallclock_sys.h"
 
 /* =====
  * Cached time
@@ -41,3 +43,26 @@ update_approx_time(time_t now)
   cached_approx_time = now;
 }
 #endif /* !defined(TIME_IS_FAST) */
+
+/**
+ * Initialize the "wallclock" subsystem by setting the current cached time.
+ **/
+static int
+subsys_wallclock_initialize(void)
+{
+  update_approx_time(time(NULL));
+  return 0;
+}
+
+/**
+ * Subsystem function table describing the "wallclock" subsystem.
+ **/
+const subsys_fns_t sys_wallclock = {
+  .name = "wallclock",
+  SUBSYS_DECLARE_LOCATION(),
+  .supported = true,
+  /* Approximate time is a diagnostic feature, we want it to init right after
+   * low-level error handling. */
+  .level = -98,
+  .initialize = subsys_wallclock_initialize,
+};
diff --git a/src/lib/wallclock/approx_time.h b/src/lib/wallclock/approx_time.h
index e6b53f2c27..42040a1f52 100644
--- a/src/lib/wallclock/approx_time.h
+++ b/src/lib/wallclock/approx_time.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -22,4 +22,4 @@ time_t approx_time(void);
 void update_approx_time(time_t now);
 #endif /* defined(TIME_IS_FAST) */
 
-#endif
+#endif /* !defined(TOR_APPROX_TIME_H) */
diff --git a/src/lib/wallclock/include.am b/src/lib/wallclock/include.am
index 1961639bd7..2b50d6ccbb 100644
--- a/src/lib/wallclock/include.am
+++ b/src/lib/wallclock/include.am
@@ -5,6 +5,7 @@ if UNITTESTS_ENABLED
 noinst_LIBRARIES += src/lib/libtor-wallclock-testing.a
 endif
 
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_lib_libtor_wallclock_a_SOURCES =			\
 	src/lib/wallclock/approx_time.c			\
 	src/lib/wallclock/time_to_tm.c			\
@@ -15,8 +16,10 @@ src_lib_libtor_wallclock_testing_a_SOURCES = \
 src_lib_libtor_wallclock_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_lib_libtor_wallclock_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS +=					\
 	src/lib/wallclock/approx_time.h			\
 	src/lib/wallclock/timeval.h			\
 	src/lib/wallclock/time_to_tm.h			\
-	src/lib/wallclock/tor_gettimeofday.h
+	src/lib/wallclock/tor_gettimeofday.h		\
+	src/lib/wallclock/wallclock_sys.h
diff --git a/src/lib/wallclock/lib_wallclock.md b/src/lib/wallclock/lib_wallclock.md
new file mode 100644
index 0000000000..f21721f6f6
--- /dev/null
+++ b/src/lib/wallclock/lib_wallclock.md
@@ -0,0 +1,11 @@
+@dir /lib/wallclock
+@brief lib/wallclock: Inspect and manipulate the current time.
+
+This module handles our concept of "what time is it" or "what time does the
+world agree it is?"  Generally, if you want something derived from UTC, this
+is the module for you.
+
+For versions of the time that are more local, more monotonic, or more
+accurate, see \refdir{lib/time}.  For parsing and encoding times and dates,
+see \refdir{lib/encoding}.
+
diff --git a/src/lib/wallclock/time_to_tm.c b/src/lib/wallclock/time_to_tm.c
index f7cb21827b..8c747b4c7b 100644
--- a/src/lib/wallclock/time_to_tm.c
+++ b/src/lib/wallclock/time_to_tm.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -198,3 +198,4 @@ tor_gmtime_r_msg(const time_t *timep, struct tm *result, char **err_out)
   return correct_tm(0, timep, result, r, err_out);
 }
 #endif /* defined(HAVE_GMTIME_R) || ... */
+/**@}*/
diff --git a/src/lib/wallclock/time_to_tm.h b/src/lib/wallclock/time_to_tm.h
index abe78c0efe..bfa8fa3689 100644
--- a/src/lib/wallclock/time_to_tm.h
+++ b/src/lib/wallclock/time_to_tm.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,4 +19,4 @@ struct tm *tor_localtime_r_msg(const time_t *timep, struct tm *result,
 struct tm *tor_gmtime_r_msg(const time_t *timep, struct tm *result,
                             char **err_out);
 
-#endif
+#endif /* !defined(TOR_WALLCLOCK_TIME_TO_TM_H) */
diff --git a/src/lib/wallclock/timeval.h b/src/lib/wallclock/timeval.h
index 4967e939bf..d7d5bda99f 100644
--- a/src/lib/wallclock/timeval.h
+++ b/src/lib/wallclock/timeval.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,6 +20,27 @@
 #include <sys/time.h>
 #endif
 
+#ifdef TOR_COVERAGE
+/* For coverage builds, we use a slower definition of these macros without
+ * branches, to make coverage consistent. */
+#undef timeradd
+#undef timersub
+#define timeradd(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec + (tv2)->tv_sec;    \
+    (tvout)->tv_usec = (tv1)->tv_usec + (tv2)->tv_usec; \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;      \
+    (tvout)->tv_usec %= 1000000;                        \
+  } while (0)
+#define timersub(tv1,tv2,tvout) \
+  do {                          \
+    (tvout)->tv_sec = (tv1)->tv_sec - (tv2)->tv_sec - 1;            \
+    (tvout)->tv_usec = (tv1)->tv_usec - (tv2)->tv_usec + 1000000;   \
+    (tvout)->tv_sec += (tvout)->tv_usec / 1000000;                  \
+    (tvout)->tv_usec %= 1000000;                                    \
+  } while (0)
+#endif /* defined(TOR_COVERAGE) */
+
 #ifndef timeradd
 /** Replacement for timeradd on platforms that do not have it: sets tvout to
  * the sum of tv1 and tv2. */
@@ -48,6 +69,7 @@
   } while (0)
 #endif /* !defined(timersub) */
 
+#ifndef COCCI
 #ifndef timercmp
 /** Replacement for timercmp on platforms that do not have it: returns true
  * iff the relational operator "op" makes the expression tv1 op tv2 true.
@@ -61,5 +83,6 @@
    ((tv1)->tv_usec op (tv2)->tv_usec) :         \
    ((tv1)->tv_sec op (tv2)->tv_sec))
 #endif /* !defined(timercmp) */
+#endif /* !defined(COCCI) */
 
-#endif
+#endif /* !defined(TOR_TIMEVAL_H) */
diff --git a/src/lib/wallclock/tor_gettimeofday.c b/src/lib/wallclock/tor_gettimeofday.c
index 63538f3b81..a07f83220d 100644
--- a/src/lib/wallclock/tor_gettimeofday.c
+++ b/src/lib/wallclock/tor_gettimeofday.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/lib/wallclock/tor_gettimeofday.h b/src/lib/wallclock/tor_gettimeofday.h
index c7fff9747a..c1a8afca3a 100644
--- a/src/lib/wallclock/tor_gettimeofday.h
+++ b/src/lib/wallclock/tor_gettimeofday.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2003-2004, Roger Dingledine
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -17,4 +17,4 @@ struct timeval;
 
 MOCK_DECL(void, tor_gettimeofday, (struct timeval *timeval));
 
-#endif
+#endif /* !defined(TOR_GETTIMEOFDAY_H) */
diff --git a/src/lib/wallclock/wallclock_sys.h b/src/lib/wallclock/wallclock_sys.h
new file mode 100644
index 0000000000..3997d11e7a
--- /dev/null
+++ b/src/lib/wallclock/wallclock_sys.h
@@ -0,0 +1,14 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file wallclock_sys.h
+ * \brief Declare subsystem object for the wallclock module.
+ **/
+
+#ifndef TOR_WALLCLOCK_SYS_H
+#define TOR_WALLCLOCK_SYS_H
+
+extern const struct subsys_fns_t sys_wallclock;
+
+#endif /* !defined(TOR_WALLCLOCK_SYS_H) */
diff --git a/src/mainpage.md b/src/mainpage.md
new file mode 100644
index 0000000000..2c4c494354
--- /dev/null
+++ b/src/mainpage.md
@@ -0,0 +1,150 @@
+@mainpage Tor source reference
+
+@tableofcontents
+
+@section welcome Welcome to Tor
+
+(For an up-to-date rendered copy of this documentation, see
+https://src-ref.docs.torproject.org/tor/index.html .)
+
+This documentation describes the general structure of the Tor codebase, how
+it fits together, what functionality is available for extending Tor, and
+gives some notes on how Tor got that way.  It also includes a reference for
+nearly every function, type, file, and module in the Tor source code.  The
+high-level documentation is a work in progress.
+
+Tor itself remains a work in progress too: We've been working on it for
+nearly two decades, and we've learned a lot about good coding since we first
+started.  This means, however, that some of the older pieces of Tor will have
+some "code smell" in them that could stand a brisk refactoring.  So when we
+describe a piece of code, we'll sometimes give a note on how it got that way,
+and whether we still think that's a good idea.
+
+This document is not an overview of the Tor protocol.  For that, see the
+design paper and the specifications at https://spec.torproject.org/ .
+
+For more information about Tor's coding standards and some helpful
+development tools, see
+[doc/HACKING](https://gitweb.torproject.org/tor.git/tree/doc/HACKING) in the
+Tor repository.
+
+@section topics Topic-related documentation
+
+@subpage intro
+
+@subpage arch_goals
+
+@subpage initialization
+
+@subpage dataflow
+
+@subpage certificates
+
+@subpage threading
+
+@subpage strings
+
+@subpage time_periodic
+
+@subpage configuration
+
+@subpage publish_subscribe
+
+@page intro A high-level overview
+
+@tableofcontents
+
+@section highlevel The very high level
+
+Ultimately, Tor runs as an event-driven network daemon: it responds to
+network events, signals, and timers by sending and receiving things over
+the network.  Clients, relays, and directory authorities all use the
+same codebase: the Tor process will run as a client, relay, or authority
+depending on its configuration.
+
+Tor has a few major dependencies, including Libevent (used to tell which
+sockets are readable and writable), OpenSSL or NSS (used for many encryption
+functions, and to implement the TLS protocol), and zlib (used to
+compress and uncompress directory information).
+
+Most of Tor's work today is done in a single event-driven main thread.
+Tor also spawns one or more worker threads to handle CPU-intensive
+tasks.  (Right now, this only includes circuit encryption and the more
+expensive compression algorithms.)
+
+On startup, Tor initializes its libraries, reads and responds to its
+configuration files, and launches a main event loop.  At first, the only
+events that Tor listens for are a few signals (like TERM and HUP), and
+one or more listener sockets (for different kinds of incoming
+connections).  Tor also configures several timers to handle periodic
+events.  As Tor runs over time, other events will open, and new events
+will be scheduled.
+
+The codebase is divided into a few top-level subdirectories, each of
+which contains several sub-modules.
+
+   - `ext` -- Code maintained elsewhere that we include in the Tor
+     source distribution.
+
+   - \refdir{lib} -- Lower-level utility code, not necessarily
+     tor-specific.
+
+   - `trunnel` -- Automatically generated code (from the Trunnel
+     tool): used to parse and encode binary formats.
+
+   - \refdir{core} -- Networking code that is implements the central
+     parts of the Tor protocol and main loop.
+
+   - \refdir{feature} -- Aspects of Tor (like directory management,
+     running a relay, running a directory authorities, managing a list of
+     nodes, running and using onion services) that are built on top of the
+     mainloop code.
+
+   - \refdir{app} -- Highest-level functionality; responsible for setting
+     up and configuring the Tor daemon, making sure all the lower-level
+     modules start up when required, and so on.
+
+   - \refdir{tools} -- Binaries other than Tor that we produce.
+      Currently this is tor-resolve, tor-gencert, and the tor_runner.o helper
+      module.
+
+   - `test` -- unit tests, regression tests, and a few integration
+     tests.
+
+In theory, the above parts of the codebase are sorted from highest-level to
+lowest-level, where high-level code is only allowed to invoke lower-level
+code, and lower-level code never includes or depends on code of a higher
+level.  In practice, this refactoring is incomplete: The modules in
+\refdir{lib} are well-factored, but there are many layer violations ("upward
+dependencies") in \refdir{core} and \refdir{feature}.
+We aim to eliminate those over time.
+
+@section keyabstractions Some key high-level abstractions
+
+The most important abstractions at Tor's high-level are Connections,
+Channels, Circuits, and Nodes.
+
+A 'Connection' (connection_t) represents a stream-based information flow.
+Most connections are TCP connections to remote Tor servers and clients. (But
+as a shortcut, a relay will sometimes make a connection to itself without
+actually using a TCP connection.  More details later on.)  Connections exist
+in different varieties, depending on what functionality they provide.  The
+principle types of connection are edge_connection_t (eg a socks connection or
+a connection from an exit relay to a destination), or_connection_t (a TLS
+stream connecting to a relay), dir_connection_t (an HTTP connection to learn
+about the network), and control_connection_t (a connection from a
+controller).
+
+A 'Circuit' (circuit_t) is persistent tunnel through the Tor network,
+established with public-key cryptography, and used to send cells one or more
+hops.  Clients keep track of multi-hop circuits (origin_circuit_t), and the
+cryptography associated with each hop.  Relays, on the other hand, keep track
+only of their hop of each circuit (or_circuit_t).
+
+A 'Channel' (channel_t) is an abstract view of sending cells to and from a
+Tor relay.  Currently, all channels are implemented using OR connections
+(channel_tls_t).  If we switch to other strategies in the future, we'll have
+more connection types.
+
+A 'Node' (node_t) is a view of a Tor instance's current knowledge and opinions
+about a Tor relay or bridge.
diff --git a/src/rust/build.rs b/src/rust/build.rs
index 123d5c0682..5626b35f75 100644
--- a/src/rust/build.rs
+++ b/src/rust/build.rs
@@ -149,8 +149,9 @@ pub fn main() {
             cfg.component("tor-sandbox-testing");
             cfg.component("tor-encoding-testing");
             cfg.component("tor-fs-testing");
-            cfg.component("tor-time-testing");
             cfg.component("tor-net-testing");
+            cfg.component("tor-buf-testing");
+            cfg.component("tor-time-testing");
             cfg.component("tor-thread-testing");
             cfg.component("tor-memarea-testing");
             cfg.component("tor-log-testing");
@@ -162,6 +163,7 @@ pub fn main() {
             cfg.component("tor-malloc");
             cfg.component("tor-wallclock");
             cfg.component("tor-err-testing");
+            cfg.component("tor-version-testing");
             cfg.component("tor-intmath-testing");
             cfg.component("tor-ctime-testing");
             cfg.component("curve25519_donna");
diff --git a/src/rust/protover/ffi.rs b/src/rust/protover/ffi.rs
index 6ee63adb10..14170d0353 100644
--- a/src/rust/protover/ffi.rs
+++ b/src/rust/protover/ffi.rs
@@ -30,6 +30,8 @@ fn translate_to_rust(c_proto: uint32_t) -> Result<Protocol, ProtoverError> {
         7 => Ok(Protocol::Desc),
         8 => Ok(Protocol::Microdesc),
         9 => Ok(Protocol::Cons),
+        10 => Ok(Protocol::Padding),
+        11 => Ok(Protocol::FlowCtrl),
         _ => Err(ProtoverError::UnknownProtocol),
     }
 }
diff --git a/src/rust/protover/protover.rs b/src/rust/protover/protover.rs
index 536667f61b..06c4dd2398 100644
--- a/src/rust/protover/protover.rs
+++ b/src/rust/protover/protover.rs
@@ -46,6 +46,8 @@ pub enum Protocol {
     LinkAuth,
     Microdesc,
     Relay,
+    Padding,
+    FlowCtrl,
 }
 
 impl fmt::Display for Protocol {
@@ -73,6 +75,8 @@ impl FromStr for Protocol {
             "LinkAuth" => Ok(Protocol::LinkAuth),
             "Microdesc" => Ok(Protocol::Microdesc),
             "Relay" => Ok(Protocol::Relay),
+            "Padding" => Ok(Protocol::Padding),
+            "FlowCtrl" => Ok(Protocol::FlowCtrl),
             _ => Err(ProtoverError::UnknownProtocol),
         }
     }
@@ -157,12 +161,14 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
             "Cons=1-2 \
              Desc=1-2 \
              DirCache=1-2 \
+             FlowCtrl=1 \
              HSDir=1-2 \
              HSIntro=3-4 \
              HSRend=1-2 \
              Link=1-5 \
              LinkAuth=3 \
              Microdesc=1-2 \
+             Padding=2 \
              Relay=1-2"
         )
     } else {
@@ -170,12 +176,14 @@ pub(crate) fn get_supported_protocols_cstr() -> &'static CStr {
             "Cons=1-2 \
              Desc=1-2 \
              DirCache=1-2 \
+             FlowCtrl=1 \
              HSDir=1-2 \
              HSIntro=3-4 \
              HSRend=1-2 \
              Link=1-5 \
              LinkAuth=1,3 \
              Microdesc=1-2 \
+             Padding=2 \
              Relay=1-2"
         )
     }
diff --git a/src/rust/tor_log/tor_log.rs b/src/rust/tor_log/tor_log.rs
index 98fccba5a9..bbaf97129c 100644
--- a/src/rust/tor_log/tor_log.rs
+++ b/src/rust/tor_log/tor_log.rs
@@ -99,14 +99,14 @@ pub mod log {
     /// Domain log types. These mirror definitions in src/lib/log/log.h
     /// C_RUST_COUPLED: src/lib/log/log.c, log severity types
     extern "C" {
-        static LD_NET_: u32;
-        static LD_GENERAL_: u32;
+        static LD_NET_: u64;
+        static LD_GENERAL_: u64;
     }
 
     /// Translate Rust defintions of log domain levels to C. This exposes a 1:1
     /// mapping between types.
     #[inline]
-    pub unsafe fn translate_domain(domain: LogDomain) -> u32 {
+    pub unsafe fn translate_domain(domain: LogDomain) -> u64 {
         match domain {
             LogDomain::Net => LD_NET_,
             LogDomain::General => LD_GENERAL_,
@@ -128,7 +128,7 @@ pub mod log {
     extern "C" {
         pub fn tor_log_string(
             severity: c_int,
-            domain: u32,
+            domain: u64,
             function: *const c_char,
             string: *const c_char,
         );
diff --git a/src/rust/tor_util/strings.rs b/src/rust/tor_util/strings.rs
index 2a19458f46..ede42c6ea8 100644
--- a/src/rust/tor_util/strings.rs
+++ b/src/rust/tor_util/strings.rs
@@ -105,11 +105,7 @@ macro_rules! cstr {
     ($($bytes:expr),*) => (
         ::std::ffi::CStr::from_bytes_with_nul(
             concat!($($bytes),*, "\0").as_bytes()
-        ).unwrap_or(
-            unsafe{
-                ::std::ffi::CStr::from_bytes_with_nul_unchecked(b"\0")
-            }
-        )
+        ).unwrap_or_default()
     )
 }
 
diff --git a/src/test/.may_include b/src/test/.may_include
new file mode 100644
index 0000000000..11c5ffbb14
--- /dev/null
+++ b/src/test/.may_include
@@ -0,0 +1,2 @@
+*.h
+*.inc
diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake
index cfbe281b94..ca6a84cf8a 100644
--- a/src/test/Makefile.nmake
+++ b/src/test/Makefile.nmake
@@ -1,4 +1,4 @@
-all: test.exe test-child.exe bench.exe
+all: test.exe bench.exe
 
 CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \
     /I ..\ext
@@ -19,6 +19,7 @@ TEST_OBJECTS = test.obj test_addr.obj test_channel.obj test_channeltls.obj \
 	test_cell_formats.obj test_relay.obj test_replay.obj \
 	test_channelpadding.obj \
 	test_circuitstats.obj \
+	test_circuitpadding.obj \
 	test_scheduler.obj test_introduce.obj test_hs.obj tinytest.obj
 
 tinytest.obj: ..\ext\tinytest.c
@@ -30,8 +31,5 @@ test.exe: $(TEST_OBJECTS)
 bench.exe: bench.obj
 	$(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@
 
-test-child.exe: test-child.obj
-	$(CC) $(CFLAGS) test-child.obj /Fe$@
-
 clean:
-	del *.obj *.lib test.exe bench.exe test-child.exe
+	del *.obj *.lib test.exe bench.exe
diff --git a/src/test/bench.c b/src/test/bench.c
index 06c616c3b0..7a8c04e802 100644
--- a/src/test/bench.c
+++ b/src/test/bench.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,16 +14,19 @@
 #include "core/crypto/onion_tap.h"
 #include "core/crypto/relay_crypto.h"
 
+#include "lib/intmath/weakrng.h"
+
 #ifdef ENABLE_OPENSSL
 #include <openssl/opensslv.h>
 #include <openssl/evp.h>
 #include <openssl/ec.h>
 #include <openssl/ecdh.h>
 #include <openssl/obj_mac.h>
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include "core/or/circuitlist.h"
 #include "app/config/config.h"
+#include "app/main/subsysmgr.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "core/crypto/onion_ntor.h"
@@ -38,6 +41,9 @@
 #include "lib/crypt_ops/digestset.h"
 #include "lib/crypt_ops/crypto_init.h"
 
+#include "feature/dirparse/microdesc_parse.h"
+#include "feature/nodelist/microdesc.h"
+
 #if defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID)
 static uint64_t nanostart;
 static inline uint64_t
@@ -332,6 +338,65 @@ bench_ed25519(void)
 }
 
 static void
+bench_rand_len(int len)
+{
+  const int N = 100000;
+  int i;
+  char *buf = tor_malloc(len);
+  uint64_t start,end;
+
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_rand(buf, len);
+  }
+  end = perftime();
+  printf("crypto_rand(%d): %f nsec.\n", len, NANOCOUNT(start,end,N));
+
+  crypto_fast_rng_t *fr = crypto_fast_rng_new();
+  start = perftime();
+  for (i = 0; i < N; ++i) {
+    crypto_fast_rng_getbytes(fr,(uint8_t*)buf,len);
+  }
+  end = perftime();
+  printf("crypto_fast_rng_getbytes(%d): %f nsec.\n", len,
+         NANOCOUNT(start,end,N));
+  crypto_fast_rng_free(fr);
+
+  if (len <= 32) {
+    start = perftime();
+    for (i = 0; i < N; ++i) {
+      crypto_strongest_rand((uint8_t*)buf, len);
+    }
+    end = perftime();
+    printf("crypto_strongest_rand(%d): %f nsec.\n", len,
+           NANOCOUNT(start,end,N));
+  }
+
+  if (len == 4) {
+    tor_weak_rng_t weak;
+    tor_init_weak_random(&weak, 1337);
+
+    start = perftime();
+    uint32_t t=0;
+    for (i = 0; i < N; ++i) {
+      t += tor_weak_random(&weak);
+    }
+    end = perftime();
+    printf("weak_rand(4): %f nsec.\n", NANOCOUNT(start,end,N));
+  }
+
+  tor_free(buf);
+}
+
+static void
+bench_rand(void)
+{
+  bench_rand_len(4);
+  bench_rand_len(16);
+  bench_rand_len(128);
+}
+
+static void
 bench_cell_aes(void)
 {
   uint64_t start, end;
@@ -636,7 +701,42 @@ bench_ecdh_p224(void)
 {
   bench_ecdh_impl(NID_secp224r1, "P-224");
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
+
+static void
+bench_md_parse(void)
+{
+  uint64_t start, end;
+  const int N = 100000;
+  // selected arbitrarily
+  const char md_text[] =
+    "@last-listed 2018-12-14 18:14:14\n"
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMHkZeXNDX/49JqM2BVLmh1Fnb5iMVnatvZZTLJyedqDLkbXZ1WKP5oh\n"
+    "7ec14dj/k3ntpwHD4s2o3Lb6nfagWbug4+F/rNJ7JuFru/PSyOvDyHGNAuegOXph\n"
+    "3gTGjdDpv/yPoiadGebbVe8E7n6hO+XxM2W/4dqheKimF0/s9B7HAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "ntor-onion-key QgF/EjqlNG1wRHLIop/nCekEH+ETGZSgYOhu26eiTF4=\n"
+    "family $00E9A86E7733240E60D8435A7BBD634A23894098 "
+    "$329BD7545DEEEBBDC8C4285F243916F248972102 "
+    "$69E06EBB2573A4F89330BDF8BC869794A3E10E4D "
+    "$DCA2A3FAE50B3729DAA15BC95FB21AF03389818B\n"
+    "p accept 53,80,443,5222-5223,25565\n"
+    "id ed25519 BzffzY99z6Q8KltcFlUTLWjNTBU7yKK+uQhyi1Ivb3A\n";
+
+  reset_perftime();
+  start = perftime();
+  for (int i = 0; i < N; ++i) {
+    smartlist_t *s = microdescs_parse_from_string(md_text, NULL, 1,
+                                                  SAVED_IN_CACHE, NULL);
+    SMARTLIST_FOREACH(s, microdesc_t *, md, microdesc_free(md));
+    smartlist_free(s);
+  }
+
+  end = perftime();
+  printf("Microdesc parse: %f nsec\n", NANOCOUNT(start, end, N));
+}
 
 typedef void (*bench_fn)(void);
 
@@ -656,6 +756,7 @@ static struct benchmark_t benchmarks[] = {
   ENT(onion_TAP),
   ENT(onion_ntor),
   ENT(ed25519),
+  ENT(rand),
 
   ENT(cell_aes),
   ENT(cell_ops),
@@ -665,6 +766,8 @@ static struct benchmark_t benchmarks[] = {
   ENT(ecdh_p256),
   ENT(ecdh_p224),
 #endif
+
+  ENT(md_parse),
   {NULL,NULL,0}
 };
 
@@ -690,9 +793,10 @@ main(int argc, const char **argv)
   char *errmsg;
   or_options_t *options;
 
-  tor_threads_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
+
   tor_compress_init();
-  init_logging(1);
 
   if (argc == 4 && !strcmp(argv[1], "diff")) {
     const int N = 200;
@@ -702,11 +806,13 @@ main(int argc, const char **argv)
       perror("X");
       return 1;
     }
+    size_t f1len = strlen(f1);
+    size_t f2len = strlen(f2);
     for (i = 0; i < N; ++i) {
-      char *diff = consensus_diff_generate(f1, f2);
+      char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
       tor_free(diff);
     }
-    char *diff = consensus_diff_generate(f1, f2);
+    char *diff = consensus_diff_generate(f1, f1len, f2, f2len);
     printf("%s", diff);
     tor_free(f1);
     tor_free(f2);
@@ -737,7 +843,6 @@ main(int argc, const char **argv)
 
   init_protocol_warning_severity_level();
   options = options_new();
-  init_logging(1);
   options->command = CMD_RUN_UNITTESTS;
   options->DataDirectory = tor_strdup("");
   options->KeyDirectory = tor_strdup("");
diff --git a/src/test/bt_test.py b/src/test/bt_test.py
index f9ca79efde..d728f13596 100755
--- a/src/test/bt_test.py
+++ b/src/test/bt_test.py
@@ -15,7 +15,11 @@ OK
 
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 
 
diff --git a/src/test/conf_examples/badnick_1/error b/src/test/conf_examples/badnick_1/error
new file mode 100644
index 0000000000..3e92ddc832
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/error
@@ -0,0 +1 @@
+nicknames must be between 1 and 19 characters inclusive
diff --git a/src/test/conf_examples/badnick_1/expected_log_no_dirauth_relay b/src/test/conf_examples/badnick_1/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..9190a3326b
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*badnick_1[./]*torrc
diff --git a/src/test/conf_examples/badnick_1/expected_no_dirauth_relay b/src/test/conf_examples/badnick_1/expected_no_dirauth_relay
new file mode 100644
index 0000000000..b00be15c2e
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/expected_no_dirauth_relay
@@ -0,0 +1 @@
+Nickname TooManyCharactersInThisNickname
diff --git a/src/test/conf_examples/badnick_1/torrc b/src/test/conf_examples/badnick_1/torrc
new file mode 100644
index 0000000000..087e3f2ff1
--- /dev/null
+++ b/src/test/conf_examples/badnick_1/torrc
@@ -0,0 +1,4 @@
+# This nickname is too long; we won't accept it.
+# (Unless the relay module is disabled, because Nickname is a
+# relay-only option. We'll ignore all relay-only options in #32395.)
+Nickname TooManyCharactersInThisNickname
diff --git a/src/test/conf_examples/badnick_2/error b/src/test/conf_examples/badnick_2/error
new file mode 100644
index 0000000000..ceac99f012
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/error
@@ -0,0 +1 @@
+must contain only the characters \[a-zA-Z0-9\]
diff --git a/src/test/conf_examples/badnick_2/expected_log_no_dirauth_relay b/src/test/conf_examples/badnick_2/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..a15c7b02cb
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*badnick_2[./]*torrc
diff --git a/src/test/conf_examples/badnick_2/expected_no_dirauth_relay b/src/test/conf_examples/badnick_2/expected_no_dirauth_relay
new file mode 100644
index 0000000000..08dcdc33a9
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/expected_no_dirauth_relay
@@ -0,0 +1 @@
+Nickname has a space
diff --git a/src/test/conf_examples/badnick_2/torrc b/src/test/conf_examples/badnick_2/torrc
new file mode 100644
index 0000000000..51a5f96c00
--- /dev/null
+++ b/src/test/conf_examples/badnick_2/torrc
@@ -0,0 +1,4 @@
+# this nickname has spaces in it and won't work.
+# (Unless the relay module is disabled, because Nickname is a
+# relay-only option. We'll ignore all relay-only options in #32395.)
+Nickname has a space
diff --git a/src/test/conf_examples/bridgeauth_1/error_no_dirauth b/src/test/conf_examples/bridgeauth_1/error_no_dirauth
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/error_no_dirauth
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/bridgeauth_1/error_no_dirauth_relay b/src/test/conf_examples/bridgeauth_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/bridgeauth_1/expected b/src/test/conf_examples/bridgeauth_1/expected
new file mode 100644
index 0000000000..d43aaf2c8b
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/expected
@@ -0,0 +1,7 @@
+Address 198.51.100.123
+AuthoritativeDirectory 1
+BridgeAuthoritativeDir 1
+ContactInfo tor_parse_test@example.com
+DirPort 80
+Nickname Unnamed
+ORPort 443
diff --git a/src/test/conf_examples/bridgeauth_1/expected_log b/src/test/conf_examples/bridgeauth_1/expected_log
new file mode 100644
index 0000000000..cabe9d3f89
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*bridgeauth_1[./]*torrc
diff --git a/src/test/conf_examples/bridgeauth_1/torrc b/src/test/conf_examples/bridgeauth_1/torrc
new file mode 100644
index 0000000000..740bc6c2eb
--- /dev/null
+++ b/src/test/conf_examples/bridgeauth_1/torrc
@@ -0,0 +1,8 @@
+AuthoritativeDirectory 1
+BridgeAuthoritativeDir 1
+
+ContactInfo tor_parse_test@example.com
+
+Address 198.51.100.123
+ORPort 443
+DirPort 80
diff --git a/src/test/conf_examples/contactinfo_notutf8/error b/src/test/conf_examples/contactinfo_notutf8/error
new file mode 100644
index 0000000000..6d165152ce
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/error
@@ -0,0 +1 @@
+ContactInfo config option must be UTF-8
diff --git a/src/test/conf_examples/contactinfo_notutf8/expected_log_no_dirauth_relay b/src/test/conf_examples/contactinfo_notutf8/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..caa07aca40
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*contactinfo_notutf8[./]*torrc
diff --git a/src/test/conf_examples/contactinfo_notutf8/expected_no_dirauth_relay b/src/test/conf_examples/contactinfo_notutf8/expected_no_dirauth_relay
new file mode 100644
index 0000000000..cc8bd7b8e3
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/expected_no_dirauth_relay
@@ -0,0 +1 @@
+ContactInfo "\304\353\304\353\304\353@example.com"
diff --git a/src/test/conf_examples/contactinfo_notutf8/torrc b/src/test/conf_examples/contactinfo_notutf8/torrc
new file mode 100644
index 0000000000..0176a56a97
--- /dev/null
+++ b/src/test/conf_examples/contactinfo_notutf8/torrc
@@ -0,0 +1,5 @@
+# We only accept ContactInfo in UTF-8 (or 7-bit ASCII)
+# (Unless the relay module is disabled, because ContactInfo is a
+# relay-only option. We'll ignore all relay-only options in #32395.
+# But at the moment, tor canonicalises and quotes the string.)
+ContactInfo ÄëÄëÄë@example.com
diff --git a/src/test/conf_examples/controlsock/error b/src/test/conf_examples/controlsock/error
new file mode 100644
index 0000000000..8fbea37894
--- /dev/null
+++ b/src/test/conf_examples/controlsock/error
@@ -0,0 +1 @@
+not supported on this OS\|without setting a ControlSocket
diff --git a/src/test/conf_examples/controlsock/torrc b/src/test/conf_examples/controlsock/torrc
new file mode 100644
index 0000000000..dd3cb7ede5
--- /dev/null
+++ b/src/test/conf_examples/controlsock/torrc
@@ -0,0 +1 @@
+ControlSocketsGroupWritable 1
diff --git a/src/test/conf_examples/crypto_accel/expected b/src/test/conf_examples/crypto_accel/expected
new file mode 100644
index 0000000000..ea80ca19dc
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected
@@ -0,0 +1,2 @@
+AccelName nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel/expected_log b/src/test/conf_examples/crypto_accel/expected_log
new file mode 100644
index 0000000000..7fab0c8dad
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected_log
@@ -0,0 +1 @@
+Unable to load dynamic OpenSSL engine "nonexistent_chartreuse_accelerator"
diff --git a/src/test/conf_examples/crypto_accel/expected_log_nss b/src/test/conf_examples/crypto_accel/expected_log_nss
new file mode 100644
index 0000000000..c0fe7b003c
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected_log_nss
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, NSS .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/crypto_accel/expected_nss b/src/test/conf_examples/crypto_accel/expected_nss
new file mode 100644
index 0000000000..ea80ca19dc
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/expected_nss
@@ -0,0 +1,2 @@
+AccelName nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel/torrc b/src/test/conf_examples/crypto_accel/torrc
new file mode 100644
index 0000000000..9ca18903b7
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel/torrc
@@ -0,0 +1,3 @@
+
+AccelName nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel_req/error b/src/test/conf_examples/crypto_accel_req/error
new file mode 100644
index 0000000000..e12e002915
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/error
@@ -0,0 +1 @@
+Unable to load required dynamic OpenSSL engine "nonexistent_chartreuse_accelerator"
diff --git a/src/test/conf_examples/crypto_accel_req/expected_log_nss b/src/test/conf_examples/crypto_accel_req/expected_log_nss
new file mode 100644
index 0000000000..c0fe7b003c
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/expected_log_nss
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, NSS .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/crypto_accel_req/expected_nss b/src/test/conf_examples/crypto_accel_req/expected_nss
new file mode 100644
index 0000000000..f3e172f640
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/expected_nss
@@ -0,0 +1,2 @@
+AccelName !nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/crypto_accel_req/torrc b/src/test/conf_examples/crypto_accel_req/torrc
new file mode 100644
index 0000000000..981d9116fc
--- /dev/null
+++ b/src/test/conf_examples/crypto_accel_req/torrc
@@ -0,0 +1,3 @@
+
+AccelName !nonexistent_chartreuse_accelerator
+HardwareAccel 1
diff --git a/src/test/conf_examples/dirauth_1/error_no_dirauth b/src/test/conf_examples/dirauth_1/error_no_dirauth
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/error_no_dirauth
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/dirauth_1/error_no_dirauth_relay b/src/test/conf_examples/dirauth_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..e6bd5db69c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with dirauth mode disabled.
diff --git a/src/test/conf_examples/dirauth_1/expected b/src/test/conf_examples/dirauth_1/expected
new file mode 100644
index 0000000000..901f7d947f
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/expected
@@ -0,0 +1,8 @@
+Address 192.0.2.1
+AuthoritativeDirectory 1
+ContactInfo tor_parse_test@example.net
+DirPort 9030
+DownloadExtraInfo 1
+Nickname Unnamed
+ORPort 9001
+V3AuthoritativeDirectory 1
diff --git a/src/test/conf_examples/dirauth_1/expected_log b/src/test/conf_examples/dirauth_1/expected_log
new file mode 100644
index 0000000000..b788be2e33
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*dirauth_1[./]*torrc
diff --git a/src/test/conf_examples/dirauth_1/torrc b/src/test/conf_examples/dirauth_1/torrc
new file mode 100644
index 0000000000..b870e6e8e0
--- /dev/null
+++ b/src/test/conf_examples/dirauth_1/torrc
@@ -0,0 +1,8 @@
+AuthoritativeDirectory 1
+V3AuthoritativeDirectory 1
+
+ContactInfo tor_parse_test@example.net
+
+Address 192.0.2.1
+ORPort 9001
+DirPort 9030
diff --git a/src/test/conf_examples/dirauth_2/expected b/src/test/conf_examples/dirauth_2/expected
new file mode 100644
index 0000000000..19ab024ed3
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected
@@ -0,0 +1 @@
+AuthDirMaxServersPerAddr 8
diff --git a/src/test/conf_examples/dirauth_2/expected_log b/src/test/conf_examples/dirauth_2/expected_log
new file mode 100644
index 0000000000..88611fee9d
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_log
@@ -0,0 +1 @@
+Read configuration file
diff --git a/src/test/conf_examples/dirauth_2/expected_log_no_dirauth b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth
new file mode 100644
index 0000000000..01110c5d8c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth
@@ -0,0 +1 @@
+This copy of Tor was built without support for the option "AuthDirMaxServersPerAddr". Skipping.
\ No newline at end of file
diff --git a/src/test/conf_examples/dirauth_2/expected_log_no_dirauth_relay b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..01110c5d8c
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+This copy of Tor was built without support for the option "AuthDirMaxServersPerAddr". Skipping.
\ No newline at end of file
diff --git a/src/test/conf_examples/dirauth_2/expected_no_dirauth b/src/test/conf_examples/dirauth_2/expected_no_dirauth
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_no_dirauth
diff --git a/src/test/conf_examples/dirauth_2/expected_no_dirauth_relay b/src/test/conf_examples/dirauth_2/expected_no_dirauth_relay
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/expected_no_dirauth_relay
diff --git a/src/test/conf_examples/dirauth_2/torrc b/src/test/conf_examples/dirauth_2/torrc
new file mode 100644
index 0000000000..bd1cdbc8b9
--- /dev/null
+++ b/src/test/conf_examples/dirauth_2/torrc
@@ -0,0 +1,5 @@
+#
+# This will get accepted if the module is enabled, and ignored if the module
+# is disabled.
+#
+AuthDirMaxServersPerAddr 8
diff --git a/src/test/conf_examples/empty_1/expected b/src/test/conf_examples/empty_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_1/expected
diff --git a/src/test/conf_examples/empty_1/expected_log b/src/test/conf_examples/empty_1/expected_log
new file mode 100644
index 0000000000..4c6b00069f
--- /dev/null
+++ b/src/test/conf_examples/empty_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*empty_1[./]*torrc
diff --git a/src/test/conf_examples/empty_1/torrc b/src/test/conf_examples/empty_1/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_1/torrc
diff --git a/src/test/conf_examples/empty_2/cmdline b/src/test/conf_examples/empty_2/cmdline
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/cmdline
diff --git a/src/test/conf_examples/empty_2/expected b/src/test/conf_examples/empty_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/expected
diff --git a/src/test/conf_examples/empty_2/expected_log b/src/test/conf_examples/empty_2/expected_log
new file mode 100644
index 0000000000..9c846a03f3
--- /dev/null
+++ b/src/test/conf_examples/empty_2/expected_log
@@ -0,0 +1 @@
+Read configuration file .*empty_2[./]*torrc\.defaults
diff --git a/src/test/conf_examples/empty_2/torrc b/src/test/conf_examples/empty_2/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/torrc
diff --git a/src/test/conf_examples/empty_2/torrc.defaults b/src/test/conf_examples/empty_2/torrc.defaults
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_2/torrc.defaults
diff --git a/src/test/conf_examples/empty_3/expected b/src/test/conf_examples/empty_3/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_3/expected
diff --git a/src/test/conf_examples/empty_3/expected_log b/src/test/conf_examples/empty_3/expected_log
new file mode 100644
index 0000000000..e3f2365893
--- /dev/null
+++ b/src/test/conf_examples/empty_3/expected_log
@@ -0,0 +1 @@
+Processing configuration path \".*included\" at recursion level 1\.
diff --git a/src/test/conf_examples/empty_3/included/empty b/src/test/conf_examples/empty_3/included/empty
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/empty_3/included/empty
diff --git a/src/test/conf_examples/empty_3/torrc b/src/test/conf_examples/empty_3/torrc
new file mode 100644
index 0000000000..049b438903
--- /dev/null
+++ b/src/test/conf_examples/empty_3/torrc
@@ -0,0 +1 @@
+%include "included"
diff --git a/src/test/conf_examples/empty_4/error b/src/test/conf_examples/empty_4/error
new file mode 100644
index 0000000000..e6c2f7d885
--- /dev/null
+++ b/src/test/conf_examples/empty_4/error
@@ -0,0 +1 @@
+Unable to open configuration file
\ No newline at end of file
diff --git a/src/test/conf_examples/example_1/expected b/src/test/conf_examples/example_1/expected
new file mode 100644
index 0000000000..9d6688a565
--- /dev/null
+++ b/src/test/conf_examples/example_1/expected
@@ -0,0 +1,2 @@
+ContactInfo tor_tellini@example.com
+SocksPort 80
diff --git a/src/test/conf_examples/example_1/expected_log b/src/test/conf_examples/example_1/expected_log
new file mode 100644
index 0000000000..8f83eec988
--- /dev/null
+++ b/src/test/conf_examples/example_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*example_1[./]*torrc
diff --git a/src/test/conf_examples/example_1/torrc b/src/test/conf_examples/example_1/torrc
new file mode 100644
index 0000000000..bff7fa0aa2
--- /dev/null
+++ b/src/test/conf_examples/example_1/torrc
@@ -0,0 +1,5 @@
+
+# Here is a simple example torrc.
+  SocksPort  80
+
+ContactInfo "tor_tellini@example.com"
diff --git a/src/test/conf_examples/example_2/error b/src/test/conf_examples/example_2/error
new file mode 100644
index 0000000000..ce18b68db4
--- /dev/null
+++ b/src/test/conf_examples/example_2/error
@@ -0,0 +1 @@
+Unknown option 'JumpingJellyjars'
diff --git a/src/test/conf_examples/example_2/torrc b/src/test/conf_examples/example_2/torrc
new file mode 100644
index 0000000000..8ec8133b24
--- /dev/null
+++ b/src/test/conf_examples/example_2/torrc
@@ -0,0 +1 @@
+JumpingJellyjars 1
diff --git a/src/test/conf_examples/example_3/cmdline b/src/test/conf_examples/example_3/cmdline
new file mode 100644
index 0000000000..5b2fadcebb
--- /dev/null
+++ b/src/test/conf_examples/example_3/cmdline
@@ -0,0 +1 @@
+--socksport 99
diff --git a/src/test/conf_examples/example_3/expected b/src/test/conf_examples/example_3/expected
new file mode 100644
index 0000000000..867fb8bcc8
--- /dev/null
+++ b/src/test/conf_examples/example_3/expected
@@ -0,0 +1 @@
+SocksPort 99
diff --git a/src/test/conf_examples/example_3/expected_log b/src/test/conf_examples/example_3/expected_log
new file mode 100644
index 0000000000..807f9c2fc8
--- /dev/null
+++ b/src/test/conf_examples/example_3/expected_log
@@ -0,0 +1 @@
+Read configuration file .*example_3[./]*torrc
diff --git a/src/test/conf_examples/example_3/torrc b/src/test/conf_examples/example_3/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/example_3/torrc
diff --git a/src/test/conf_examples/include_1/error_no_dirauth_relay b/src/test/conf_examples/include_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/include_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/include_1/expected b/src/test/conf_examples/include_1/expected
new file mode 100644
index 0000000000..4bbf52ce9f
--- /dev/null
+++ b/src/test/conf_examples/include_1/expected
@@ -0,0 +1,3 @@
+ContactInfo includefile@example.com
+Nickname nested
+ORPort 8008
diff --git a/src/test/conf_examples/include_1/expected_log b/src/test/conf_examples/include_1/expected_log
new file mode 100644
index 0000000000..0791a494d2
--- /dev/null
+++ b/src/test/conf_examples/include_1/expected_log
@@ -0,0 +1 @@
+Processing configuration path \".*nested\.inc\" at recursion level 2\.
diff --git a/src/test/conf_examples/include_1/included.inc b/src/test/conf_examples/include_1/included.inc
new file mode 100644
index 0000000000..8d1834345d
--- /dev/null
+++ b/src/test/conf_examples/include_1/included.inc
@@ -0,0 +1,4 @@
+
+ContactInfo includefile@example.com
+
+%include "nested.inc"
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/nested.inc b/src/test/conf_examples/include_1/nested.inc
new file mode 100644
index 0000000000..789b044a2b
--- /dev/null
+++ b/src/test/conf_examples/include_1/nested.inc
@@ -0,0 +1,2 @@
+
+Nickname nested
\ No newline at end of file
diff --git a/src/test/conf_examples/include_1/torrc b/src/test/conf_examples/include_1/torrc
new file mode 100644
index 0000000000..2ed4074f6e
--- /dev/null
+++ b/src/test/conf_examples/include_1/torrc
@@ -0,0 +1,4 @@
+
+%include "included.inc"
+
+ORPort 8008
diff --git a/src/test/conf_examples/include_bug_31408/error_no_dirauth_relay b/src/test/conf_examples/include_bug_31408/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/include_bug_31408/expected b/src/test/conf_examples/include_bug_31408/expected
new file mode 100644
index 0000000000..2e822f1a78
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/expected
@@ -0,0 +1,2 @@
+Nickname test31408
+ORPort 31408
diff --git a/src/test/conf_examples/include_bug_31408/expected_log b/src/test/conf_examples/include_bug_31408/expected_log
new file mode 100644
index 0000000000..e3f2365893
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/expected_log
@@ -0,0 +1 @@
+Processing configuration path \".*included\" at recursion level 1\.
diff --git a/src/test/conf_examples/include_bug_31408/included/01_nickname.inc b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
new file mode 100644
index 0000000000..508dd89a35
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/01_nickname.inc
@@ -0,0 +1 @@
+Nickname test31408
diff --git a/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
new file mode 100644
index 0000000000..140e927f19
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/included/02_no_configs.inc
@@ -0,0 +1,3 @@
+# Bug 31048 is triggered when the last file in a config directory:
+#   * contains no configuration options,
+#   * but is non-empty: that is, it contains comments or whitespace.
diff --git a/src/test/conf_examples/include_bug_31408/torrc b/src/test/conf_examples/include_bug_31408/torrc
new file mode 100644
index 0000000000..a42685e93c
--- /dev/null
+++ b/src/test/conf_examples/include_bug_31408/torrc
@@ -0,0 +1,2 @@
+%include "included"
+ORPort 31408
diff --git a/src/test/conf_examples/large_1/error_no_dirauth_relay b/src/test/conf_examples/large_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/large_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/large_1/expected b/src/test/conf_examples/large_1/expected
new file mode 100644
index 0000000000..99a12ffc84
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected
@@ -0,0 +1,158 @@
+AccountingMax 10737418240
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2147483647
+BandwidthRate 1610612736
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirPortFrontPage /dirport/frontpage
+DirPort 99
+DirReqStatistics 0
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 1260
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 300
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPortCookieAuthFile /foobar
+ExtORPort 99
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 691200
+HeartbeatPeriod 2700
+IPv6Exit 1
+KeepalivePeriod 540
+KeyDirectory /keyz
+KISTSchedRunInterval 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 60000
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 3600
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 2629728
+MaxMemInQueues 314572800
+MaxOnionQueueDelay 60000
+MaxUnparseableDescSizeToLog 1048576
+MiddleNodes grommit,truffle,parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 7200
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PerConnBWBurst 10485760
+PerConnBWRate 102400
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 600
+RephistTrackTime 600
+SafeLogging 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10
+SigningKeyLifetime 4838400
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 600
+SSLKeyLifetime 86400
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1000
+TrackHostExits www.example.com
+TrackHostExitsExpire 3600
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/large_1/expected_log b/src/test/conf_examples/large_1/expected_log
new file mode 100644
index 0000000000..21248bb5e4
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected_log
@@ -0,0 +1 @@
+Your log may contain sensitive information
diff --git a/src/test/conf_examples/large_1/expected_log_no_dirauth b/src/test/conf_examples/large_1/expected_log_no_dirauth
new file mode 100644
index 0000000000..0b74de4e40
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected_log_no_dirauth
@@ -0,0 +1 @@
+This copy of Tor was built without support for the option "ConsensusParams". Skipping.
diff --git a/src/test/conf_examples/large_1/expected_no_dirauth b/src/test/conf_examples/large_1/expected_no_dirauth
new file mode 100644
index 0000000000..26a33bdc7c
--- /dev/null
+++ b/src/test/conf_examples/large_1/expected_no_dirauth
@@ -0,0 +1,157 @@
+AccountingMax 10737418240
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2147483647
+BandwidthRate 1610612736
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirPortFrontPage /dirport/frontpage
+DirPort 99
+DirReqStatistics 0
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 1260
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 300
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPortCookieAuthFile /foobar
+ExtORPort 99
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 691200
+HeartbeatPeriod 2700
+IPv6Exit 1
+KeepalivePeriod 540
+KeyDirectory /keyz
+KISTSchedRunInterval 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 60000
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 3600
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 2629728
+MaxMemInQueues 314572800
+MaxOnionQueueDelay 60000
+MaxUnparseableDescSizeToLog 1048576
+MiddleNodes grommit,truffle,parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 7200
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PerConnBWBurst 10485760
+PerConnBWRate 102400
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 600
+RephistTrackTime 600
+SafeLogging 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10
+SigningKeyLifetime 4838400
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 600
+SSLKeyLifetime 86400
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1000
+TrackHostExits www.example.com
+TrackHostExitsExpire 3600
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/large_1/torrc b/src/test/conf_examples/large_1/torrc
new file mode 100644
index 0000000000..20ddf00e16
--- /dev/null
+++ b/src/test/conf_examples/large_1/torrc
@@ -0,0 +1,166 @@
+AccountingMax 10 GB
+AccountingRule sum
+AccountingStart day 05:15
+Address 128.66.8.8
+AllowNonRFC953Hostnames 1
+AndroidIdentityTag droidy
+AutomapHostsOnResolve 1
+AutomapHostsSuffixes .onions
+AvoidDiskWrites 1
+BandwidthBurst 2 GB
+BandwidthRate 1.5 GB
+Bridge 128.66.1.10:80
+CacheDirectory /this-is-a-cache
+CellStatistics 1
+CircuitBuildTimeout 200
+CircuitPadding 1
+CircuitsAvailableTimeout 10
+CircuitStreamTimeout 20
+ClientOnly 1
+ClientPreferIPv6DirPort 1
+ClientPreferIPv6ORPort 1
+ClientRejectInternalAddresses 0
+ClientUseIPv4 0
+ClientUseIPv6 1
+ConnDirectionStatistics 1
+ConnectionPadding 1
+ConnLimit 64
+ConsensusParams wombat=7
+ConstrainedSockets 1
+ConstrainedSockSize 10240
+ContactInfo long_config@example.com
+ControlPortFileGroupReadable 1
+ControlPort 9058
+CookieAuthentication 1
+CookieAuthFile /control/cookie
+CookieAuthFileGroupReadable 1
+CountPrivateBandwidth 1
+DataDirectory /data/dir
+DirAllowPrivateAddresses 1
+DirPolicy reject 128.66.1.1/32, accept *:*
+DirReqStatistics 0
+DirPort 99
+DirPortFrontPage /dirport/frontpage
+DisableDebuggerAttachment 0
+DisableNetwork 1
+DisableOOSCheck 0
+DNSPort 53535
+DormantCanceledByStartup 1
+DormantClientTimeout 21 minutes
+DormantOnFirstStartup 1
+DormantTimeoutDisabledByIdleStreams 0
+DoSCircuitCreationBurst 1000
+DoSCircuitCreationDefenseTimePeriod 5 minutes
+DoSCircuitCreationDefenseType 2
+DoSCircuitCreationEnabled 1
+DoSCircuitCreationMinConnections 10
+DoSCircuitCreationRate 100
+DoSConnectionDefenseType 2
+DoSConnectionEnabled 1
+DoSConnectionMaxConcurrentCount 6
+DoSRefuseSingleHopClientRendezvous 0
+DownloadExtraInfo 1
+EnforceDistinctSubnets 0
+EntryNodes potrzebie,triffid,cromulent
+EntryStatistics 1
+ExcludeExitNodes blaznort,kriffid,zeppelin
+ExcludeNodes 128.66.7.6
+ExitNodes 128.66.7.7,128.66.128.0/17,exitexit
+ExitPolicy accept *:80,reject *:*
+ExitPolicyRejectLocalInterfaces 1
+ExitPolicyRejectPrivate 0
+ExitPortStatistics 1
+ExitRelay 1
+ExtendAllowPrivateAddresses 1
+ExtendByEd25519ID 1
+ExtORPort 99
+ExtORPortCookieAuthFile /foobar
+ExtraInfoStatistics 1
+FascistFirewall 1
+FetchDirInfoEarly 1
+FetchDirInfoExtraEarly 1
+FetchHidServDescriptors 1
+FetchServerDescriptors 1
+FetchUselessDescriptors 1
+FirewallPorts 80,443,999
+GeoIPExcludeUnknown 1
+GeoIPFile /geoip
+GuardfractionFile /gff
+GuardLifetime 8 days
+HeartbeatPeriod 45 minutes
+IPv6Exit 1
+KeepalivePeriod 9 minutes
+KeyDirectory /keyz
+KISTSchedRunInterval 1 msec
+LearnCircuitBuildTimeout 1
+Log notice file /logfile
+Log info file /logfile-verbose
+LogTimeGranularity 1 minute
+LongLivedPorts 9090
+MainloopStats 1
+MapAddress www.example.com:10.0.0.6
+MaxAdvertisedBandwidth 100
+MaxCircuitDirtiness 1 hour
+MaxClientCircuitsPending 127
+MaxConsensusAgeForDiffs 1 month
+MaxMemInQueues 300 MB
+MaxOnionQueueDelay 60 seconds
+MaxUnparseableDescSizeToLog 1 MB
+MiddleNodes grommit, truffle, parcheesi
+MyFamily $ffffffffffffffffffffffffffffffffffffffff
+NewCircuitPeriod 2 hours
+Nickname nickname
+NodeFamily $ffffffffffffffffffffffffffffffffffffffff,$dddddddddddddddddddddddddddddddddddddddd
+NumCPUs 3
+NumDirectoryGuards 4
+NumEntryGuards 5
+NumPrimaryGuards 8
+OfflineMasterKey 1
+OptimisticData 1
+ORPort 2222
+OutboundBindAddress 10.0.0.7
+OutboundBindAddressExit 10.0.0.8
+OutboundBindAddressOR 10.0.0.9
+PaddingStatistics 1
+PerConnBWBurst 10 MB
+PerConnBWRate 100 kb
+PidFile /piddy
+ProtocolWarnings 1
+PublishHidServDescriptors 0
+PublishServerDescriptor 0
+ReachableAddresses 0.0.0.0, *:*
+ReachableDirAddresses 128.0.0.0/1
+ReachableORAddresses 128.0.0.0/8
+RejectPlaintextPorts 23
+RelayBandwidthBurst 10000
+RelayBandwidthRate 1000
+RendPostPeriod 10 minutes
+RephistTrackTime 10 minutes
+SafeLogging 0
+SafeSocks 0
+Schedulers Vanilla,KISTLite,Kist
+ShutdownWaitLength 10 seconds
+SigningKeyLifetime 8 weeks
+Socks5Proxy 128.66.99.99:99
+Socks5ProxyPassword flynn
+Socks5ProxyUsername spaceparanoids
+SocksPolicy accept 127.0.0.0/24, reject *:*
+SocksPort 9099
+SocksTimeout 10 minutes
+SSLKeyLifetime 1 day
+StrictNodes 1
+SyslogIdentityTag tortor
+TestSocks 1
+TokenBucketRefillInterval 1 second
+TrackHostExits www.example.com
+TrackHostExitsExpire 1 hour
+TruncateLogFile 1
+UnixSocksGroupWritable 1
+UpdateBridgesFromAuthority 1
+UseDefaultFallbackDirs 0
+UseEntryGuards 1
+UseGuardFraction 1
+UseMicrodescriptors 0
+VirtualAddrNetworkIPv4 18.66.0.0/16
+VirtualAddrNetworkIPv6 [ff00::]/16
+WarnPlaintextPorts 7,11,23,1001
diff --git a/src/test/conf_examples/lzma_zstd_1/expected b/src/test/conf_examples/lzma_zstd_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log b/src/test/conf_examples/lzma_zstd_1/expected_log
new file mode 100644
index 0000000000..a5531ca21e
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma N/A, and Libzstd N/A
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log_lzma b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma
new file mode 100644
index 0000000000..2947e5991b
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma .*, and Libzstd N/A
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log_lzma_zstd b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma_zstd
new file mode 100644
index 0000000000..e76e4357f8
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log_lzma_zstd
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma .*, and Libzstd .*
\ No newline at end of file
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_log_zstd b/src/test/conf_examples/lzma_zstd_1/expected_log_zstd
new file mode 100644
index 0000000000..c8b174423b
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_log_zstd
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, .*, Zlib .*, Liblzma N/A, and Libzstd .*
\ No newline at end of file
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_lzma b/src/test/conf_examples/lzma_zstd_1/expected_lzma
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_lzma
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_lzma_zstd b/src/test/conf_examples/lzma_zstd_1/expected_lzma_zstd
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_lzma_zstd
diff --git a/src/test/conf_examples/lzma_zstd_1/expected_zstd b/src/test/conf_examples/lzma_zstd_1/expected_zstd
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/expected_zstd
diff --git a/src/test/conf_examples/lzma_zstd_1/torrc b/src/test/conf_examples/lzma_zstd_1/torrc
new file mode 100644
index 0000000000..f873d79028
--- /dev/null
+++ b/src/test/conf_examples/lzma_zstd_1/torrc
@@ -0,0 +1 @@
+# This test checks for the optional library list in tor's logs
diff --git a/src/test/conf_examples/missing_cl_arg/cmdline b/src/test/conf_examples/missing_cl_arg/cmdline
new file mode 100644
index 0000000000..7fc4d0a54d
--- /dev/null
+++ b/src/test/conf_examples/missing_cl_arg/cmdline
@@ -0,0 +1 @@
+--hash-password
diff --git a/src/test/conf_examples/missing_cl_arg/error b/src/test/conf_examples/missing_cl_arg/error
new file mode 100644
index 0000000000..61dbeac8aa
--- /dev/null
+++ b/src/test/conf_examples/missing_cl_arg/error
@@ -0,0 +1 @@
+Command-line option '--hash-password' with no value.
diff --git a/src/test/conf_examples/missing_cl_arg/torrc b/src/test/conf_examples/missing_cl_arg/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/missing_cl_arg/torrc
diff --git a/src/test/conf_examples/nss_1/expected b/src/test/conf_examples/nss_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected
diff --git a/src/test/conf_examples/nss_1/expected_log b/src/test/conf_examples/nss_1/expected_log
new file mode 100644
index 0000000000..32e8cfc2f8
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected_log
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, OpenSSL .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/nss_1/expected_log_nss b/src/test/conf_examples/nss_1/expected_log_nss
new file mode 100644
index 0000000000..c0fe7b003c
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected_log_nss
@@ -0,0 +1 @@
+Tor 0.* running on .* with Libevent .*, NSS .*, Zlib .*, Liblzma .*, and Libzstd .*
diff --git a/src/test/conf_examples/nss_1/expected_nss b/src/test/conf_examples/nss_1/expected_nss
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/nss_1/expected_nss
diff --git a/src/test/conf_examples/nss_1/torrc b/src/test/conf_examples/nss_1/torrc
new file mode 100644
index 0000000000..f873d79028
--- /dev/null
+++ b/src/test/conf_examples/nss_1/torrc
@@ -0,0 +1 @@
+# This test checks for the optional library list in tor's logs
diff --git a/src/test/conf_examples/obsolete_1/expected b/src/test/conf_examples/obsolete_1/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/expected
diff --git a/src/test/conf_examples/obsolete_1/expected_log b/src/test/conf_examples/obsolete_1/expected_log
new file mode 100644
index 0000000000..52f6f70a2e
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/expected_log
@@ -0,0 +1 @@
+Skipping obsolete configuration option
diff --git a/src/test/conf_examples/obsolete_1/torrc b/src/test/conf_examples/obsolete_1/torrc
new file mode 100644
index 0000000000..e711fe4065
--- /dev/null
+++ b/src/test/conf_examples/obsolete_1/torrc
@@ -0,0 +1,70 @@
+# These options are obsolete as of 0.4.2
+# Obsolete options without arguments, or with an empty argument,
+# are silently ignored. So we give each one of these options an argument.
+AllowDotExit 1
+AllowInvalidNodes 1
+AllowSingleHopCircuits 1
+AllowSingleHopExits 1
+AlternateHSAuthority 1
+AuthDirBadDir 1
+AuthDirBadDirCCs 1
+AuthDirRejectUnlisted 1
+AuthDirListBadDirs 1
+AuthDirMaxServersPerAuthAddr 1
+CircuitIdleTimeout 1
+ControlListenAddress 1
+DirListenAddress 1
+DisableIOCP 1
+DisableV2DirectoryInfo_ 1
+DynamicDHGroups 1
+DNSListenAddress 1
+TestingEnableTbEmptyEvent 1
+ExcludeSingleHopRelays 1
+FallbackNetworkstatusFile 1
+FastFirstHopPK 1
+FetchV2Networkstatus 1
+Group 1
+HidServDirectoryV2 1
+CloseHSClientCircuitsImmediatelyOnTimeout 1
+CloseHSServiceRendCircuitsImmediatelyOnTimeout 1
+MaxOnionsPending 1
+NamingAuthoritativeDirectory 1
+NATDListenAddress 1
+PredictedPortsRelevanceTime 1
+WarnUnsafeSocks 1
+ORListenAddress 1
+PathBiasDisableRate 1
+PathBiasScaleFactor 1
+PathBiasMultFactor 1
+PathBiasUseCloseCounts 1
+PortForwarding 1
+PortForwardingHelper 1
+PreferTunneledDirConns 1
+RecommendedPackages 1
+RunTesting 1
+SchedulerLowWaterMark__ 1
+SchedulerHighWaterMark__ 1
+SchedulerMaxFlushCells__ 1
+SocksListenAddress 1
+StrictEntryNodes 1
+StrictExitNodes 1
+Support022HiddenServices 1
+Tor2webMode 1
+Tor2webRendezvousPoints 1
+TLSECGroup 1
+TransListenAddress 1
+TunnelDirConns 1
+UseEntryGuardsAsDirGuards 1
+UseNTorHandshake 1
+UserspaceIOCPBuffers 1
+V1AuthoritativeDirectory 1
+V2AuthoritativeDirectory 1
+VoteOnHidServDirectoriesV2 1
+UseFilteringSSLBufferevents 1
+__UseFilteringSSLBufferevents 1
+TestingConsensusMaxDownloadTries 1
+ClientBootstrapConsensusMaxDownloadTries 1
+ClientBootstrapConsensusAuthorityOnlyMaxDownloadTries 1
+TestingDescriptorMaxDownloadTries 1
+TestingMicrodescMaxDownloadTries 1
+TestingCertMaxDownloadTries 1
diff --git a/src/test/conf_examples/obsolete_2/expected b/src/test/conf_examples/obsolete_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/expected
diff --git a/src/test/conf_examples/obsolete_2/expected_log b/src/test/conf_examples/obsolete_2/expected_log
new file mode 100644
index 0000000000..2160355ee9
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/expected_log
@@ -0,0 +1 @@
+Read configuration file .*obsolete_2[./]*torrc
diff --git a/src/test/conf_examples/obsolete_2/torrc b/src/test/conf_examples/obsolete_2/torrc
new file mode 100644
index 0000000000..b83e7a7369
--- /dev/null
+++ b/src/test/conf_examples/obsolete_2/torrc
@@ -0,0 +1,5 @@
+# This option has been obsolete for some time
+# Obsolete options without arguments, or with an empty argument,
+# are silently ignored.
+AllowDotExit
+AllowInvalidNodes ""
diff --git a/src/test/conf_examples/obsolete_3/expected b/src/test/conf_examples/obsolete_3/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/obsolete_3/expected
diff --git a/src/test/conf_examples/obsolete_3/expected_log b/src/test/conf_examples/obsolete_3/expected_log
new file mode 100644
index 0000000000..82d76ec818
--- /dev/null
+++ b/src/test/conf_examples/obsolete_3/expected_log
@@ -0,0 +1 @@
+Skipping obsolete configuration option "AllowDotExit"
diff --git a/src/test/conf_examples/obsolete_3/torrc b/src/test/conf_examples/obsolete_3/torrc
new file mode 100644
index 0000000000..e0efe752bf
--- /dev/null
+++ b/src/test/conf_examples/obsolete_3/torrc
@@ -0,0 +1,4 @@
+# This option has been obsolete for some time
+# Obsolete options without arguments, or with an empty argument,
+# are silently ignored. So we give this option an argument.
+AllowDotExit 1
diff --git a/src/test/conf_examples/ops_1/cmdline b/src/test/conf_examples/ops_1/cmdline
new file mode 100644
index 0000000000..2bb9bfa132
--- /dev/null
+++ b/src/test/conf_examples/ops_1/cmdline
@@ -0,0 +1 @@
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/error_no_dirauth_relay b/src/test/conf_examples/ops_1/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_1/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_1/expected b/src/test/conf_examples/ops_1/expected
new file mode 100644
index 0000000000..84be6a70e2
--- /dev/null
+++ b/src/test/conf_examples/ops_1/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1000
diff --git a/src/test/conf_examples/ops_1/expected_log b/src/test/conf_examples/ops_1/expected_log
new file mode 100644
index 0000000000..b785d7fb52
--- /dev/null
+++ b/src/test/conf_examples/ops_1/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_1[./]*torrc
diff --git a/src/test/conf_examples/ops_1/torrc b/src/test/conf_examples/ops_1/torrc
new file mode 100644
index 0000000000..daf8ae60fe
--- /dev/null
+++ b/src/test/conf_examples/ops_1/torrc
@@ -0,0 +1,3 @@
+# We'll replace this option on the command line.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_2/cmdline b/src/test/conf_examples/ops_2/cmdline
new file mode 100644
index 0000000000..fdd48a045c
--- /dev/null
+++ b/src/test/conf_examples/ops_2/cmdline
@@ -0,0 +1 @@
+/ORPort
diff --git a/src/test/conf_examples/ops_2/expected b/src/test/conf_examples/ops_2/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_2/expected
diff --git a/src/test/conf_examples/ops_2/expected_log b/src/test/conf_examples/ops_2/expected_log
new file mode 100644
index 0000000000..17fbc3ffbf
--- /dev/null
+++ b/src/test/conf_examples/ops_2/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_2[./]*torrc
diff --git a/src/test/conf_examples/ops_2/torrc b/src/test/conf_examples/ops_2/torrc
new file mode 100644
index 0000000000..21fcc93f9a
--- /dev/null
+++ b/src/test/conf_examples/ops_2/torrc
@@ -0,0 +1,3 @@
+# We'll remove this option on the command line, and not replace it.
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_3/cmdline b/src/test/conf_examples/ops_3/cmdline
new file mode 100644
index 0000000000..e4965d26f8
--- /dev/null
+++ b/src/test/conf_examples/ops_3/cmdline
@@ -0,0 +1 @@
++ORPort 1000
diff --git a/src/test/conf_examples/ops_3/error_no_dirauth_relay b/src/test/conf_examples/ops_3/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_3/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_3/expected b/src/test/conf_examples/ops_3/expected
new file mode 100644
index 0000000000..569d26b577
--- /dev/null
+++ b/src/test/conf_examples/ops_3/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9999
+ORPort 1000
diff --git a/src/test/conf_examples/ops_3/expected_log b/src/test/conf_examples/ops_3/expected_log
new file mode 100644
index 0000000000..151498f0df
--- /dev/null
+++ b/src/test/conf_examples/ops_3/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_3[./]*torrc
diff --git a/src/test/conf_examples/ops_3/torrc b/src/test/conf_examples/ops_3/torrc
new file mode 100644
index 0000000000..14adf87d7f
--- /dev/null
+++ b/src/test/conf_examples/ops_3/torrc
@@ -0,0 +1,3 @@
+# We will extend this option on the command line
+
+ORPort 9999
diff --git a/src/test/conf_examples/ops_4/error_no_dirauth_relay b/src/test/conf_examples/ops_4/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_4/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_4/expected b/src/test/conf_examples/ops_4/expected
new file mode 100644
index 0000000000..bf52f6a330
--- /dev/null
+++ b/src/test/conf_examples/ops_4/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/expected_log b/src/test/conf_examples/ops_4/expected_log
new file mode 100644
index 0000000000..7632b2290c
--- /dev/null
+++ b/src/test/conf_examples/ops_4/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_4[./]*torrc\.defaults
diff --git a/src/test/conf_examples/ops_4/torrc b/src/test/conf_examples/ops_4/torrc
new file mode 100644
index 0000000000..dcec2aa95d
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc
@@ -0,0 +1,3 @@
+# This value is unadorned, so replaces the one from defaults.torrc.
+
+ORPort 9099
diff --git a/src/test/conf_examples/ops_4/torrc.defaults b/src/test/conf_examples/ops_4/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_4/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_5/error_no_dirauth_relay b/src/test/conf_examples/ops_5/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/ops_5/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/ops_5/expected b/src/test/conf_examples/ops_5/expected
new file mode 100644
index 0000000000..288721da53
--- /dev/null
+++ b/src/test/conf_examples/ops_5/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 9000
+ORPort 9099
diff --git a/src/test/conf_examples/ops_5/expected_log b/src/test/conf_examples/ops_5/expected_log
new file mode 100644
index 0000000000..ec63cb0638
--- /dev/null
+++ b/src/test/conf_examples/ops_5/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_5[./]*torrc\.defaults
diff --git a/src/test/conf_examples/ops_5/torrc b/src/test/conf_examples/ops_5/torrc
new file mode 100644
index 0000000000..3284fc1c55
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc
@@ -0,0 +1,3 @@
+# This value has a plus, and so extends the one from defaults.torrc.
+
++ORPort 9099
diff --git a/src/test/conf_examples/ops_5/torrc.defaults b/src/test/conf_examples/ops_5/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_5/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/ops_6/expected b/src/test/conf_examples/ops_6/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/ops_6/expected
diff --git a/src/test/conf_examples/ops_6/expected_log b/src/test/conf_examples/ops_6/expected_log
new file mode 100644
index 0000000000..f9b1ca0412
--- /dev/null
+++ b/src/test/conf_examples/ops_6/expected_log
@@ -0,0 +1 @@
+Read configuration file .*ops_6[./]*torrc\.defaults
diff --git a/src/test/conf_examples/ops_6/torrc b/src/test/conf_examples/ops_6/torrc
new file mode 100644
index 0000000000..4d51caaff7
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc
@@ -0,0 +1,3 @@
+# This value has a slash, and so clears the one from defaults.torrc.
+
+/ORPort
diff --git a/src/test/conf_examples/ops_6/torrc.defaults b/src/test/conf_examples/ops_6/torrc.defaults
new file mode 100644
index 0000000000..04cd0393c6
--- /dev/null
+++ b/src/test/conf_examples/ops_6/torrc.defaults
@@ -0,0 +1 @@
+ORPort 9000
diff --git a/src/test/conf_examples/pt_01/expected b/src/test/conf_examples/pt_01/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/pt_01/expected
diff --git a/src/test/conf_examples/pt_01/expected_log b/src/test/conf_examples/pt_01/expected_log
new file mode 100644
index 0000000000..65bfa7a7b7
--- /dev/null
+++ b/src/test/conf_examples/pt_01/expected_log
@@ -0,0 +1 @@
+Linelist option 'ExtORPort' has no value\. Skipping
diff --git a/src/test/conf_examples/pt_01/torrc b/src/test/conf_examples/pt_01/torrc
new file mode 100644
index 0000000000..574bb32a0d
--- /dev/null
+++ b/src/test/conf_examples/pt_01/torrc
@@ -0,0 +1,7 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Empty linelist values are ignored with a warning
+ExtORPort
+ServerTransportPlugin
+ServerTransportListenAddr
+ServerTransportOptions
diff --git a/src/test/conf_examples/pt_02/error b/src/test/conf_examples/pt_02/error
new file mode 100644
index 0000000000..ce28eab729
--- /dev/null
+++ b/src/test/conf_examples/pt_02/error
@@ -0,0 +1 @@
+Invalid ExtORPort configuration
diff --git a/src/test/conf_examples/pt_02/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_02/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..0e48dca7fd
--- /dev/null
+++ b/src/test/conf_examples/pt_02/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_02[./]*torrc
diff --git a/src/test/conf_examples/pt_02/expected_no_dirauth_relay b/src/test/conf_examples/pt_02/expected_no_dirauth_relay
new file mode 100644
index 0000000000..f5cd26e6b7
--- /dev/null
+++ b/src/test/conf_examples/pt_02/expected_no_dirauth_relay
@@ -0,0 +1,8 @@
+ExtORPort illegal_hostname_chars$()^*%(%
+ServerTransportListenAddr bad
+ServerTransportListenAddr bad2 illegal_hostname_chars$()^*%(%
+ServerTransportOptions bad
+ServerTransportOptions bad2 not_kv
+ServerTransportPlugin bad
+ServerTransportPlugin bad2 exec
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_02/torrc b/src/test/conf_examples/pt_02/torrc
new file mode 100644
index 0000000000..825f2c4be4
--- /dev/null
+++ b/src/test/conf_examples/pt_02/torrc
@@ -0,0 +1,13 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Bad options are also ignored
+# (Unless the relay module is disabled, because they are relay-only
+# options. We'll ignore all relay-only options in #32395.)
+ExtORPort illegal_hostname_chars$()^*%(%#%)#(%*
+ServerTransportPlugin bad
+ServerTransportPlugin bad2 exec
+ServerTransportPlugin bad3 exec /
+ServerTransportListenAddr bad
+ServerTransportListenAddr bad2 illegal_hostname_chars$()^*%(%#%)#(%*
+ServerTransportOptions bad
+ServerTransportOptions bad2 not_kv
diff --git a/src/test/conf_examples/pt_03/expected b/src/test/conf_examples/pt_03/expected
new file mode 100644
index 0000000000..f849f2a78f
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected
@@ -0,0 +1 @@
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_03/expected_log b/src/test/conf_examples/pt_03/expected_log
new file mode 100644
index 0000000000..285a189c28
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected_log
@@ -0,0 +1 @@
+We use pluggable transports but the Extended ORPort is disabled
diff --git a/src/test/conf_examples/pt_03/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_03/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..88f4e5bdfb
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_03[./]*torrc
diff --git a/src/test/conf_examples/pt_03/expected_no_dirauth_relay b/src/test/conf_examples/pt_03/expected_no_dirauth_relay
new file mode 100644
index 0000000000..f849f2a78f
--- /dev/null
+++ b/src/test/conf_examples/pt_03/expected_no_dirauth_relay
@@ -0,0 +1 @@
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_03/torrc b/src/test/conf_examples/pt_03/torrc
new file mode 100644
index 0000000000..9868c39b26
--- /dev/null
+++ b/src/test/conf_examples/pt_03/torrc
@@ -0,0 +1,4 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Plugin, but no ExtORPort
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_04/expected b/src/test/conf_examples/pt_04/expected
new file mode 100644
index 0000000000..9087f600e0
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected
@@ -0,0 +1,3 @@
+ExtORPortCookieAuthFile /
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_04/expected_log b/src/test/conf_examples/pt_04/expected_log
new file mode 100644
index 0000000000..5b3ab51d25
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected_log
@@ -0,0 +1 @@
+Tor is not configured as a relay but you specified a ServerTransportPlugin line.*The ServerTransportPlugin line will be ignored
diff --git a/src/test/conf_examples/pt_04/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_04/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..2b989bf320
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_04[./]*torrc
diff --git a/src/test/conf_examples/pt_04/expected_no_dirauth_relay b/src/test/conf_examples/pt_04/expected_no_dirauth_relay
new file mode 100644
index 0000000000..9087f600e0
--- /dev/null
+++ b/src/test/conf_examples/pt_04/expected_no_dirauth_relay
@@ -0,0 +1,3 @@
+ExtORPortCookieAuthFile /
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_04/torrc b/src/test/conf_examples/pt_04/torrc
new file mode 100644
index 0000000000..18bb28f9cf
--- /dev/null
+++ b/src/test/conf_examples/pt_04/torrc
@@ -0,0 +1,6 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a bad cookie auth file
+ExtORPort 1
+ExtORPortCookieAuthFile /
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_05/error_no_dirauth_relay b/src/test/conf_examples/pt_05/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/pt_05/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/pt_05/expected b/src/test/conf_examples/pt_05/expected
new file mode 100644
index 0000000000..61568bb9ac
--- /dev/null
+++ b/src/test/conf_examples/pt_05/expected
@@ -0,0 +1,4 @@
+ExtORPort 1
+Nickname Unnamed
+ORPort 2
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_05/expected_log b/src/test/conf_examples/pt_05/expected_log
new file mode 100644
index 0000000000..c05a0931d6
--- /dev/null
+++ b/src/test/conf_examples/pt_05/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
diff --git a/src/test/conf_examples/pt_05/torrc b/src/test/conf_examples/pt_05/torrc
new file mode 100644
index 0000000000..55c569bb1b
--- /dev/null
+++ b/src/test/conf_examples/pt_05/torrc
@@ -0,0 +1,6 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a valid minimal config
+ORPort 2
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_06/expected b/src/test/conf_examples/pt_06/expected
new file mode 100644
index 0000000000..d5788b92c9
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected
@@ -0,0 +1,6 @@
+ExtORPortCookieAuthFile /
+ExtORPortCookieAuthFileGroupReadable 1
+ExtORPort 1
+ServerTransportListenAddr bad3 127.0.0.1:2
+ServerTransportOptions bad3 a=b
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_06/expected_log b/src/test/conf_examples/pt_06/expected_log
new file mode 100644
index 0000000000..5b3ab51d25
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected_log
@@ -0,0 +1 @@
+Tor is not configured as a relay but you specified a ServerTransportPlugin line.*The ServerTransportPlugin line will be ignored
diff --git a/src/test/conf_examples/pt_06/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_06/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..f35a380c9f
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_06[./]*torrc
diff --git a/src/test/conf_examples/pt_06/expected_no_dirauth_relay b/src/test/conf_examples/pt_06/expected_no_dirauth_relay
new file mode 100644
index 0000000000..d5788b92c9
--- /dev/null
+++ b/src/test/conf_examples/pt_06/expected_no_dirauth_relay
@@ -0,0 +1,6 @@
+ExtORPortCookieAuthFile /
+ExtORPortCookieAuthFileGroupReadable 1
+ExtORPort 1
+ServerTransportListenAddr bad3 127.0.0.1:2
+ServerTransportOptions bad3 a=b
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_06/torrc b/src/test/conf_examples/pt_06/torrc
new file mode 100644
index 0000000000..20cfc329a7
--- /dev/null
+++ b/src/test/conf_examples/pt_06/torrc
@@ -0,0 +1,9 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a config with all the options
+ExtORPort 1
+ExtORPortCookieAuthFile /
+ExtORPortCookieAuthFileGroupReadable 1
+ServerTransportPlugin bad3 exec /
+ServerTransportListenAddr bad3 127.0.0.1:2
+ServerTransportOptions bad3 a=b
diff --git a/src/test/conf_examples/pt_07/error_no_dirauth_relay b/src/test/conf_examples/pt_07/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/pt_07/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/pt_07/expected b/src/test/conf_examples/pt_07/expected
new file mode 100644
index 0000000000..c3a75dc407
--- /dev/null
+++ b/src/test/conf_examples/pt_07/expected
@@ -0,0 +1,4 @@
+ExtORPort 2.2.2.2:1
+Nickname Unnamed
+ORPort 2
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_07/expected_log b/src/test/conf_examples/pt_07/expected_log
new file mode 100644
index 0000000000..5afaf02ba9
--- /dev/null
+++ b/src/test/conf_examples/pt_07/expected_log
@@ -0,0 +1 @@
+You specified a public address .* for ExtORPort
diff --git a/src/test/conf_examples/pt_07/torrc b/src/test/conf_examples/pt_07/torrc
new file mode 100644
index 0000000000..40eaf50e64
--- /dev/null
+++ b/src/test/conf_examples/pt_07/torrc
@@ -0,0 +1,6 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a valid config with a risky ExtORPort address
+ORPort 2
+ExtORPort 2.2.2.2:1
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_08/error b/src/test/conf_examples/pt_08/error
new file mode 100644
index 0000000000..7931bbb4b9
--- /dev/null
+++ b/src/test/conf_examples/pt_08/error
@@ -0,0 +1 @@
+ExtORPort does not support unix sockets
\ No newline at end of file
diff --git a/src/test/conf_examples/pt_08/expected_log_no_dirauth_relay b/src/test/conf_examples/pt_08/expected_log_no_dirauth_relay
new file mode 100644
index 0000000000..79dcbc10ca
--- /dev/null
+++ b/src/test/conf_examples/pt_08/expected_log_no_dirauth_relay
@@ -0,0 +1 @@
+Read configuration file .*pt_08[./]*torrc
diff --git a/src/test/conf_examples/pt_08/expected_no_dirauth_relay b/src/test/conf_examples/pt_08/expected_no_dirauth_relay
new file mode 100644
index 0000000000..9a1a9bd1e4
--- /dev/null
+++ b/src/test/conf_examples/pt_08/expected_no_dirauth_relay
@@ -0,0 +1,2 @@
+ExtORPort unix:/
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_08/torrc b/src/test/conf_examples/pt_08/torrc
new file mode 100644
index 0000000000..6f1d79d706
--- /dev/null
+++ b/src/test/conf_examples/pt_08/torrc
@@ -0,0 +1,7 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try an invalid config with a unix socket for ExtORPort
+# (Unless the relay module is disabled, because they are relay-only
+# options. We'll ignore all relay-only options in #32395.)
+ExtORPort unix:/
+ServerTransportPlugin bad3 exec /
diff --git a/src/test/conf_examples/pt_09/error b/src/test/conf_examples/pt_09/error
new file mode 100644
index 0000000000..882b50a7bc
--- /dev/null
+++ b/src/test/conf_examples/pt_09/error
@@ -0,0 +1 @@
+Error parsing ServerTransportListenAddr address
\ No newline at end of file
diff --git a/src/test/conf_examples/pt_09/error_no_dirauth_relay b/src/test/conf_examples/pt_09/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/pt_09/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/pt_09/torrc b/src/test/conf_examples/pt_09/torrc
new file mode 100644
index 0000000000..50a8e95b95
--- /dev/null
+++ b/src/test/conf_examples/pt_09/torrc
@@ -0,0 +1,7 @@
+# Relay PT tests
+# Options from relay/transport_config.c
+# Try a valid minimal config, with a bad ServerTransportListenAddr
+ORPort 2
+ExtORPort 1
+ServerTransportPlugin bad3 exec /
+ServerTransportListenAddr bad3 [aaaa::bbbb:ccccc]
diff --git a/src/test/conf_examples/relay_01/expected b/src/test/conf_examples/relay_01/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_examples/relay_01/expected
diff --git a/src/test/conf_examples/relay_01/expected_log b/src/test/conf_examples/relay_01/expected_log
new file mode 100644
index 0000000000..32e8c99d27
--- /dev/null
+++ b/src/test/conf_examples/relay_01/expected_log
@@ -0,0 +1 @@
+Linelist option 'ORPort' has no value\. Skipping
diff --git a/src/test/conf_examples/relay_01/torrc b/src/test/conf_examples/relay_01/torrc
new file mode 100644
index 0000000000..da3e85b427
--- /dev/null
+++ b/src/test/conf_examples/relay_01/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Empty linelist values are ignored with a warning
+ORPort
+DirPort
diff --git a/src/test/conf_examples/relay_02/error b/src/test/conf_examples/relay_02/error
new file mode 100644
index 0000000000..dd87d9f7e2
--- /dev/null
+++ b/src/test/conf_examples/relay_02/error
@@ -0,0 +1 @@
+Unrecognized value bad
diff --git a/src/test/conf_examples/relay_02/error_no_dirauth_relay b/src/test/conf_examples/relay_02/error_no_dirauth_relay
new file mode 100644
index 0000000000..dd87d9f7e2
--- /dev/null
+++ b/src/test/conf_examples/relay_02/error_no_dirauth_relay
@@ -0,0 +1 @@
+Unrecognized value bad
diff --git a/src/test/conf_examples/relay_02/torrc b/src/test/conf_examples/relay_02/torrc
new file mode 100644
index 0000000000..3eaa4403a9
--- /dev/null
+++ b/src/test/conf_examples/relay_02/torrc
@@ -0,0 +1,7 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Bad options are also ignored
+ORPort illegal_hostname_chars$()^*%(%#%)#(%*
+DirPort illegal_hostname_chars$()^*%(%#%)#(%*
+DirCache bad
+BridgeRelay bad
diff --git a/src/test/conf_examples/relay_03/error_no_dirauth_relay b/src/test/conf_examples/relay_03/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_03/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_03/expected b/src/test/conf_examples/relay_03/expected
new file mode 100644
index 0000000000..15056a8d1f
--- /dev/null
+++ b/src/test/conf_examples/relay_03/expected
@@ -0,0 +1,2 @@
+DirPort 1
+ORPort 0
diff --git a/src/test/conf_examples/relay_03/expected_log b/src/test/conf_examples/relay_03/expected_log
new file mode 100644
index 0000000000..46ab723e4a
--- /dev/null
+++ b/src/test/conf_examples/relay_03/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_03[./]*torrc
diff --git a/src/test/conf_examples/relay_03/torrc b/src/test/conf_examples/relay_03/torrc
new file mode 100644
index 0000000000..fd7da7bb95
--- /dev/null
+++ b/src/test/conf_examples/relay_03/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# DirPort, but no ORPort
+ORPort 0
+DirPort 1
diff --git a/src/test/conf_examples/relay_04/error_no_dirauth_relay b/src/test/conf_examples/relay_04/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_04/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_04/expected b/src/test/conf_examples/relay_04/expected
new file mode 100644
index 0000000000..1d25374ed1
--- /dev/null
+++ b/src/test/conf_examples/relay_04/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_04/expected_log b/src/test/conf_examples/relay_04/expected_log
new file mode 100644
index 0000000000..c05a0931d6
--- /dev/null
+++ b/src/test/conf_examples/relay_04/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
diff --git a/src/test/conf_examples/relay_04/torrc b/src/test/conf_examples/relay_04/torrc
new file mode 100644
index 0000000000..ff08b2376b
--- /dev/null
+++ b/src/test/conf_examples/relay_04/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal config
+ORPort 1
diff --git a/src/test/conf_examples/relay_05/error_no_dirauth_relay b/src/test/conf_examples/relay_05/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_05/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_05/expected b/src/test/conf_examples/relay_05/expected
new file mode 100644
index 0000000000..ae58cee1af
--- /dev/null
+++ b/src/test/conf_examples/relay_05/expected
@@ -0,0 +1,3 @@
+DirPort 2
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_05/expected_log b/src/test/conf_examples/relay_05/expected_log
new file mode 100644
index 0000000000..483c2e2aae
--- /dev/null
+++ b/src/test/conf_examples/relay_05/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_05[./]*torrc
diff --git a/src/test/conf_examples/relay_05/torrc b/src/test/conf_examples/relay_05/torrc
new file mode 100644
index 0000000000..faeaad32a0
--- /dev/null
+++ b/src/test/conf_examples/relay_05/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal directory mirror config
+ORPort 1
+DirPort 2
diff --git a/src/test/conf_examples/relay_06/error_no_dirauth_relay b/src/test/conf_examples/relay_06/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_06/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_06/expected b/src/test/conf_examples/relay_06/expected
new file mode 100644
index 0000000000..904c7339e0
--- /dev/null
+++ b/src/test/conf_examples/relay_06/expected
@@ -0,0 +1,3 @@
+BridgeRelay 1
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_06/expected_log b/src/test/conf_examples/relay_06/expected_log
new file mode 100644
index 0000000000..70eb18df19
--- /dev/null
+++ b/src/test/conf_examples/relay_06/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_06[./]*torrc
diff --git a/src/test/conf_examples/relay_06/torrc b/src/test/conf_examples/relay_06/torrc
new file mode 100644
index 0000000000..baeae8df5d
--- /dev/null
+++ b/src/test/conf_examples/relay_06/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal bridge config
+ORPort 1
+BridgeRelay 1
diff --git a/src/test/conf_examples/relay_07/error_no_dirauth_relay b/src/test/conf_examples/relay_07/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_07/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_07/expected b/src/test/conf_examples/relay_07/expected
new file mode 100644
index 0000000000..79fa3e5a47
--- /dev/null
+++ b/src/test/conf_examples/relay_07/expected
@@ -0,0 +1,3 @@
+DirCache 0
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_07/expected_log b/src/test/conf_examples/relay_07/expected_log
new file mode 100644
index 0000000000..14729a7ab1
--- /dev/null
+++ b/src/test/conf_examples/relay_07/expected_log
@@ -0,0 +1 @@
+DirCache is disabled and we are configured as a relay
diff --git a/src/test/conf_examples/relay_07/torrc b/src/test/conf_examples/relay_07/torrc
new file mode 100644
index 0000000000..01ac138597
--- /dev/null
+++ b/src/test/conf_examples/relay_07/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid minimal non-directory cache config
+ORPort 1
+DirCache 0
diff --git a/src/test/conf_examples/relay_08/error_no_dirauth_relay b/src/test/conf_examples/relay_08/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_08/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_08/expected b/src/test/conf_examples/relay_08/expected
new file mode 100644
index 0000000000..904c7339e0
--- /dev/null
+++ b/src/test/conf_examples/relay_08/expected
@@ -0,0 +1,3 @@
+BridgeRelay 1
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_08/expected_log b/src/test/conf_examples/relay_08/expected_log
new file mode 100644
index 0000000000..b0168c803d
--- /dev/null
+++ b/src/test/conf_examples/relay_08/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_08[./]*torrc
diff --git a/src/test/conf_examples/relay_08/torrc b/src/test/conf_examples/relay_08/torrc
new file mode 100644
index 0000000000..9e2ff9465c
--- /dev/null
+++ b/src/test/conf_examples/relay_08/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid config with all the bridge options
+ORPort 1
+BridgeRelay 1
+DirCache 1
diff --git a/src/test/conf_examples/relay_09/error_no_dirauth_relay b/src/test/conf_examples/relay_09/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_09/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_09/expected b/src/test/conf_examples/relay_09/expected
new file mode 100644
index 0000000000..ae58cee1af
--- /dev/null
+++ b/src/test/conf_examples/relay_09/expected
@@ -0,0 +1,3 @@
+DirPort 2
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_09/expected_log b/src/test/conf_examples/relay_09/expected_log
new file mode 100644
index 0000000000..d3ab4f6593
--- /dev/null
+++ b/src/test/conf_examples/relay_09/expected_log
@@ -0,0 +1 @@
+By default, Tor does not run as an exit relay
diff --git a/src/test/conf_examples/relay_09/torrc b/src/test/conf_examples/relay_09/torrc
new file mode 100644
index 0000000000..014eeca34b
--- /dev/null
+++ b/src/test/conf_examples/relay_09/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid config with all the non-bridge options
+ORPort 1
+DirPort 2
+DirCache 1
diff --git a/src/test/conf_examples/relay_10/error_no_dirauth_relay b/src/test/conf_examples/relay_10/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_10/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_10/expected b/src/test/conf_examples/relay_10/expected
new file mode 100644
index 0000000000..904c7339e0
--- /dev/null
+++ b/src/test/conf_examples/relay_10/expected
@@ -0,0 +1,3 @@
+BridgeRelay 1
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_10/expected_log b/src/test/conf_examples/relay_10/expected_log
new file mode 100644
index 0000000000..5b81a904e5
--- /dev/null
+++ b/src/test/conf_examples/relay_10/expected_log
@@ -0,0 +1 @@
+Can't set a DirPort on a bridge relay
diff --git a/src/test/conf_examples/relay_10/torrc b/src/test/conf_examples/relay_10/torrc
new file mode 100644
index 0000000000..4318ebb45b
--- /dev/null
+++ b/src/test/conf_examples/relay_10/torrc
@@ -0,0 +1,7 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Try a valid config, that has a warning: Bridge, warn and disable DirPort
+ORPort 1
+DirPort 2
+DirCache 1
+BridgeRelay 1
diff --git a/src/test/conf_examples/relay_11/error b/src/test/conf_examples/relay_11/error
new file mode 100644
index 0000000000..8ed5c31bc7
--- /dev/null
+++ b/src/test/conf_examples/relay_11/error
@@ -0,0 +1 @@
+We are advertising an ORPort, but not actually listening on one
diff --git a/src/test/conf_examples/relay_11/error_no_dirauth_relay b/src/test/conf_examples/relay_11/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_11/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_11/torrc b/src/test/conf_examples/relay_11/torrc
new file mode 100644
index 0000000000..a1e13eb3ce
--- /dev/null
+++ b/src/test/conf_examples/relay_11/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Advertising but not listening: ORPort
+ORPort 1 NoListen
diff --git a/src/test/conf_examples/relay_12/error b/src/test/conf_examples/relay_12/error
new file mode 100644
index 0000000000..57706d6a7a
--- /dev/null
+++ b/src/test/conf_examples/relay_12/error
@@ -0,0 +1 @@
+We are advertising a DirPort, but not actually listening on one
diff --git a/src/test/conf_examples/relay_12/error_no_dirauth_relay b/src/test/conf_examples/relay_12/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_12/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_12/torrc b/src/test/conf_examples/relay_12/torrc
new file mode 100644
index 0000000000..4a7d398112
--- /dev/null
+++ b/src/test/conf_examples/relay_12/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Advertising but not listening: DirPort
+DirPort 1 NoListen
diff --git a/src/test/conf_examples/relay_13/error b/src/test/conf_examples/relay_13/error
new file mode 100644
index 0000000000..cd74247ea8
--- /dev/null
+++ b/src/test/conf_examples/relay_13/error
@@ -0,0 +1 @@
+We are listening on an ORPort, but not advertising any ORPorts
diff --git a/src/test/conf_examples/relay_13/error_no_dirauth_relay b/src/test/conf_examples/relay_13/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_13/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_13/torrc b/src/test/conf_examples/relay_13/torrc
new file mode 100644
index 0000000000..b76b72c0cc
--- /dev/null
+++ b/src/test/conf_examples/relay_13/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Listening but not advertising: ORPort
+ORPort 1 NoAdvertise
diff --git a/src/test/conf_examples/relay_14/error_no_dirauth_relay b/src/test/conf_examples/relay_14/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_14/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_14/expected b/src/test/conf_examples/relay_14/expected
new file mode 100644
index 0000000000..31bb1c2507
--- /dev/null
+++ b/src/test/conf_examples/relay_14/expected
@@ -0,0 +1 @@
+DirPort 1 NoAdvertise
diff --git a/src/test/conf_examples/relay_14/expected_log b/src/test/conf_examples/relay_14/expected_log
new file mode 100644
index 0000000000..9b0c820211
--- /dev/null
+++ b/src/test/conf_examples/relay_14/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_14[./]*torrc
diff --git a/src/test/conf_examples/relay_14/torrc b/src/test/conf_examples/relay_14/torrc
new file mode 100644
index 0000000000..15c6496c7e
--- /dev/null
+++ b/src/test/conf_examples/relay_14/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Listening but not advertising: DirPort
+DirPort 1 NoAdvertise
diff --git a/src/test/conf_examples/relay_15/error b/src/test/conf_examples/relay_15/error
new file mode 100644
index 0000000000..da30f0cd14
--- /dev/null
+++ b/src/test/conf_examples/relay_15/error
@@ -0,0 +1 @@
+Can't advertise more than one DirPort
diff --git a/src/test/conf_examples/relay_15/error_no_dirauth_relay b/src/test/conf_examples/relay_15/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_15/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_15/torrc b/src/test/conf_examples/relay_15/torrc
new file mode 100644
index 0000000000..e1f78ee6a0
--- /dev/null
+++ b/src/test/conf_examples/relay_15/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Advertising more than one DirPort
+DirPort 1
+DirPort 2
diff --git a/src/test/conf_examples/relay_16/error b/src/test/conf_examples/relay_16/error
new file mode 100644
index 0000000000..37b89ee572
--- /dev/null
+++ b/src/test/conf_examples/relay_16/error
@@ -0,0 +1 @@
+Configured public relay to listen only on an IPv6 address. Tor needs to listen on an IPv4 address
diff --git a/src/test/conf_examples/relay_16/error_no_dirauth_relay b/src/test/conf_examples/relay_16/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_16/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_16/torrc b/src/test/conf_examples/relay_16/torrc
new file mode 100644
index 0000000000..e544cd87a4
--- /dev/null
+++ b/src/test/conf_examples/relay_16/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# IPv6 ORPort only
+ORPort [::1]:2
diff --git a/src/test/conf_examples/relay_17/error_no_dirauth_relay b/src/test/conf_examples/relay_17/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_17/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_17/expected b/src/test/conf_examples/relay_17/expected
new file mode 100644
index 0000000000..3fb0c9db92
--- /dev/null
+++ b/src/test/conf_examples/relay_17/expected
@@ -0,0 +1,4 @@
+AccountingMax 1
+KeepBindCapabilities 0
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_17/expected_log b/src/test/conf_examples/relay_17/expected_log
new file mode 100644
index 0000000000..7711f90178
--- /dev/null
+++ b/src/test/conf_examples/relay_17/expected_log
@@ -0,0 +1 @@
+You have set AccountingMax to use hibernation
diff --git a/src/test/conf_examples/relay_17/torrc b/src/test/conf_examples/relay_17/torrc
new file mode 100644
index 0000000000..f63f36815e
--- /dev/null
+++ b/src/test/conf_examples/relay_17/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Rebind warning
+ORPort 1
+AccountingMax 1
+KeepBindCapabilities 0
diff --git a/src/test/conf_examples/relay_18/error b/src/test/conf_examples/relay_18/error
new file mode 100644
index 0000000000..5b28d311b0
--- /dev/null
+++ b/src/test/conf_examples/relay_18/error
@@ -0,0 +1 @@
+Invalid DirPort configuration
diff --git a/src/test/conf_examples/relay_18/error_no_dirauth_relay b/src/test/conf_examples/relay_18/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_18/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_18/torrc b/src/test/conf_examples/relay_18/torrc
new file mode 100644
index 0000000000..67a0fd0dfb
--- /dev/null
+++ b/src/test/conf_examples/relay_18/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Bad DirPort
+DirPort illegal_hostname_chars$()^*%(%#%)#(%*
diff --git a/src/test/conf_examples/relay_19/error_no_dirauth_relay b/src/test/conf_examples/relay_19/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_19/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_19/expected b/src/test/conf_examples/relay_19/expected
new file mode 100644
index 0000000000..f077169c88
--- /dev/null
+++ b/src/test/conf_examples/relay_19/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 1
+PublishServerDescriptor
diff --git a/src/test/conf_examples/relay_19/expected_log b/src/test/conf_examples/relay_19/expected_log
new file mode 100644
index 0000000000..17656ba2cf
--- /dev/null
+++ b/src/test/conf_examples/relay_19/expected_log
@@ -0,0 +1 @@
+Read configuration file .*relay_19[./]*torrc
diff --git a/src/test/conf_examples/relay_19/torrc b/src/test/conf_examples/relay_19/torrc
new file mode 100644
index 0000000000..fd2cd91fa5
--- /dev/null
+++ b/src/test/conf_examples/relay_19/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Minimal PublishServerDescriptor
+ORPort 1
+PublishServerDescriptor
diff --git a/src/test/conf_examples/relay_20/error b/src/test/conf_examples/relay_20/error
new file mode 100644
index 0000000000..e5a81637f8
--- /dev/null
+++ b/src/test/conf_examples/relay_20/error
@@ -0,0 +1 @@
+Unrecognized value in PublishServerDescriptor
diff --git a/src/test/conf_examples/relay_20/error_no_dirauth_relay b/src/test/conf_examples/relay_20/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_20/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_20/torrc b/src/test/conf_examples/relay_20/torrc
new file mode 100644
index 0000000000..87dd74fdc1
--- /dev/null
+++ b/src/test/conf_examples/relay_20/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Invalid PublishServerDescriptor
+ORPort 1
+PublishServerDescriptor bad
diff --git a/src/test/conf_examples/relay_21/error_no_dirauth_relay b/src/test/conf_examples/relay_21/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_21/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_21/expected b/src/test/conf_examples/relay_21/expected
new file mode 100644
index 0000000000..9bcead1402
--- /dev/null
+++ b/src/test/conf_examples/relay_21/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 1
+PublishServerDescriptor v1,v2,hidserv
diff --git a/src/test/conf_examples/relay_21/expected_log b/src/test/conf_examples/relay_21/expected_log
new file mode 100644
index 0000000000..ba0d56fb1c
--- /dev/null
+++ b/src/test/conf_examples/relay_21/expected_log
@@ -0,0 +1 @@
+PublishServerDescriptor v1 has no effect
diff --git a/src/test/conf_examples/relay_21/torrc b/src/test/conf_examples/relay_21/torrc
new file mode 100644
index 0000000000..97f032f626
--- /dev/null
+++ b/src/test/conf_examples/relay_21/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Ignored PublishServerDescriptor values
+ORPort 1
+PublishServerDescriptor v1,v2,hidserv
diff --git a/src/test/conf_examples/relay_22/error b/src/test/conf_examples/relay_22/error
new file mode 100644
index 0000000000..c47dd8c4c6
--- /dev/null
+++ b/src/test/conf_examples/relay_22/error
@@ -0,0 +1 @@
+Invalid BridgeDistribution value
diff --git a/src/test/conf_examples/relay_22/error_no_dirauth_relay b/src/test/conf_examples/relay_22/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_22/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_22/torrc b/src/test/conf_examples/relay_22/torrc
new file mode 100644
index 0000000000..e83c83260e
--- /dev/null
+++ b/src/test/conf_examples/relay_22/torrc
@@ -0,0 +1,6 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Bad BridgeDistribution characters
+ORPort 1
+BridgeRelay 1
+BridgeDistribution *$%()@!
diff --git a/src/test/conf_examples/relay_23/error b/src/test/conf_examples/relay_23/error
new file mode 100644
index 0000000000..f76bbe77c4
--- /dev/null
+++ b/src/test/conf_examples/relay_23/error
@@ -0,0 +1 @@
+Relays must use 'auto' for the ConnectionPadding setting
diff --git a/src/test/conf_examples/relay_23/error_no_dirauth_relay b/src/test/conf_examples/relay_23/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_23/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_23/torrc b/src/test/conf_examples/relay_23/torrc
new file mode 100644
index 0000000000..3d28a1e27c
--- /dev/null
+++ b/src/test/conf_examples/relay_23/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ConnectionPadding
+ORPort 1
+ConnectionPadding 1
diff --git a/src/test/conf_examples/relay_24/error b/src/test/conf_examples/relay_24/error
new file mode 100644
index 0000000000..f76bbe77c4
--- /dev/null
+++ b/src/test/conf_examples/relay_24/error
@@ -0,0 +1 @@
+Relays must use 'auto' for the ConnectionPadding setting
diff --git a/src/test/conf_examples/relay_24/error_no_dirauth_relay b/src/test/conf_examples/relay_24/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_24/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_24/torrc b/src/test/conf_examples/relay_24/torrc
new file mode 100644
index 0000000000..1206e59e09
--- /dev/null
+++ b/src/test/conf_examples/relay_24/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ConnectionPadding
+ORPort 1
+ConnectionPadding 0
diff --git a/src/test/conf_examples/relay_25/error b/src/test/conf_examples/relay_25/error
new file mode 100644
index 0000000000..bac681e6cc
--- /dev/null
+++ b/src/test/conf_examples/relay_25/error
@@ -0,0 +1 @@
+Relays cannot set ReducedConnectionPadding
diff --git a/src/test/conf_examples/relay_25/error_no_dirauth_relay b/src/test/conf_examples/relay_25/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_25/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_25/torrc b/src/test/conf_examples/relay_25/torrc
new file mode 100644
index 0000000000..ab862a16f3
--- /dev/null
+++ b/src/test/conf_examples/relay_25/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ReducedConnectionPadding 1
+ORPort 1
+ReducedConnectionPadding 1
diff --git a/src/test/conf_examples/relay_26/error b/src/test/conf_examples/relay_26/error
new file mode 100644
index 0000000000..94334935e3
--- /dev/null
+++ b/src/test/conf_examples/relay_26/error
@@ -0,0 +1 @@
+Relays cannot set CircuitPadding to 0
diff --git a/src/test/conf_examples/relay_26/error_no_dirauth_relay b/src/test/conf_examples/relay_26/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_26/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_26/torrc b/src/test/conf_examples/relay_26/torrc
new file mode 100644
index 0000000000..5dd6d68dc4
--- /dev/null
+++ b/src/test/conf_examples/relay_26/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set CircuitPadding to 0
+ORPort 1
+CircuitPadding 0
diff --git a/src/test/conf_examples/relay_27/error b/src/test/conf_examples/relay_27/error
new file mode 100644
index 0000000000..e26ce46914
--- /dev/null
+++ b/src/test/conf_examples/relay_27/error
@@ -0,0 +1 @@
+Relays cannot set ReducedCircuitPadding
diff --git a/src/test/conf_examples/relay_27/error_no_dirauth_relay b/src/test/conf_examples/relay_27/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_27/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_27/torrc b/src/test/conf_examples/relay_27/torrc
new file mode 100644
index 0000000000..8556b2f351
--- /dev/null
+++ b/src/test/conf_examples/relay_27/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Relays can't set ReducedCircuitPadding 1
+ORPort 1
+ReducedCircuitPadding 1
diff --git a/src/test/conf_examples/relay_28/error b/src/test/conf_examples/relay_28/error
new file mode 100644
index 0000000000..3f14df975b
--- /dev/null
+++ b/src/test/conf_examples/relay_28/error
@@ -0,0 +1 @@
+SigningKeyLifetime is too short
diff --git a/src/test/conf_examples/relay_28/error_no_dirauth_relay b/src/test/conf_examples/relay_28/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_28/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_28/torrc b/src/test/conf_examples/relay_28/torrc
new file mode 100644
index 0000000000..3e2c895bb7
--- /dev/null
+++ b/src/test/conf_examples/relay_28/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# Short key lifetimes
+ORPort 1
+SigningKeyLifetime 1
diff --git a/src/test/conf_examples/relay_29/error_no_dirauth_relay b/src/test/conf_examples/relay_29/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_29/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_29/expected b/src/test/conf_examples/relay_29/expected
new file mode 100644
index 0000000000..1d25374ed1
--- /dev/null
+++ b/src/test/conf_examples/relay_29/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort 1
diff --git a/src/test/conf_examples/relay_29/expected_log b/src/test/conf_examples/relay_29/expected_log
new file mode 100644
index 0000000000..f46c609c0c
--- /dev/null
+++ b/src/test/conf_examples/relay_29/expected_log
@@ -0,0 +1 @@
+Linelist option 'MyFamily' has no value\. Skipping
diff --git a/src/test/conf_examples/relay_29/torrc b/src/test/conf_examples/relay_29/torrc
new file mode 100644
index 0000000000..4181d5acc2
--- /dev/null
+++ b/src/test/conf_examples/relay_29/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# Options from relay/relay_config.c
+# MyFamily normalisation: empty MyFamily
+ORPort 1
+MyFamily
diff --git a/src/test/conf_examples/relay_30/error_no_dirauth_relay b/src/test/conf_examples/relay_30/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_30/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_30/expected b/src/test/conf_examples/relay_30/expected
new file mode 100644
index 0000000000..3a4e9feb3f
--- /dev/null
+++ b/src/test/conf_examples/relay_30/expected
@@ -0,0 +1,2 @@
+Nickname Unnamed
+ORPort auto
diff --git a/src/test/conf_examples/relay_30/expected_log b/src/test/conf_examples/relay_30/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_30/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_30/torrc b/src/test/conf_examples/relay_30/torrc
new file mode 100644
index 0000000000..bf8487fe16
--- /dev/null
+++ b/src/test/conf_examples/relay_30/torrc
@@ -0,0 +1,3 @@
+# Relay tests
+# default (IPv4) ORPort auto
+ORPort auto
diff --git a/src/test/conf_examples/relay_31/error_no_dirauth_relay b/src/test/conf_examples/relay_31/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_31/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_31/expected b/src/test/conf_examples/relay_31/expected
new file mode 100644
index 0000000000..9a40cdd588
--- /dev/null
+++ b/src/test/conf_examples/relay_31/expected
@@ -0,0 +1,3 @@
+DirPort auto
+Nickname Unnamed
+ORPort auto
diff --git a/src/test/conf_examples/relay_31/expected_log b/src/test/conf_examples/relay_31/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_31/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_31/torrc b/src/test/conf_examples/relay_31/torrc
new file mode 100644
index 0000000000..e662bb71b3
--- /dev/null
+++ b/src/test/conf_examples/relay_31/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# default (IPv4) ORPort and DirPort auto
+ORPort auto
+DirPort auto
diff --git a/src/test/conf_examples/relay_32/error_no_dirauth_relay b/src/test/conf_examples/relay_32/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_32/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_32/expected b/src/test/conf_examples/relay_32/expected
new file mode 100644
index 0000000000..14b36c8259
--- /dev/null
+++ b/src/test/conf_examples/relay_32/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_32/expected_log b/src/test/conf_examples/relay_32/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_32/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_32/torrc b/src/test/conf_examples/relay_32/torrc
new file mode 100644
index 0000000000..95a66c4852
--- /dev/null
+++ b/src/test/conf_examples/relay_32/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# default (IPv4) ORPort auto and IPv6 ORPort auto
+ORPort auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_33/error_no_dirauth_relay b/src/test/conf_examples/relay_33/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_33/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_33/expected b/src/test/conf_examples/relay_33/expected
new file mode 100644
index 0000000000..22567cbe2e
--- /dev/null
+++ b/src/test/conf_examples/relay_33/expected
@@ -0,0 +1,3 @@
+Nickname Unnamed
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_33/expected_log b/src/test/conf_examples/relay_33/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_33/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_33/torrc b/src/test/conf_examples/relay_33/torrc
new file mode 100644
index 0000000000..44d16ad31a
--- /dev/null
+++ b/src/test/conf_examples/relay_33/torrc
@@ -0,0 +1,4 @@
+# Relay tests
+# explicit IPv4 ORPort auto and IPv6 ORPort auto
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_34/error_no_dirauth_relay b/src/test/conf_examples/relay_34/error_no_dirauth_relay
new file mode 100644
index 0000000000..9f9c0fd8f3
--- /dev/null
+++ b/src/test/conf_examples/relay_34/error_no_dirauth_relay
@@ -0,0 +1 @@
+This tor was built with relay mode disabled.
diff --git a/src/test/conf_examples/relay_34/expected b/src/test/conf_examples/relay_34/expected
new file mode 100644
index 0000000000..bccde684f3
--- /dev/null
+++ b/src/test/conf_examples/relay_34/expected
@@ -0,0 +1,4 @@
+DirPort 127.0.0.1:auto
+Nickname Unnamed
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
diff --git a/src/test/conf_examples/relay_34/expected_log b/src/test/conf_examples/relay_34/expected_log
new file mode 100644
index 0000000000..d5478c1a15
--- /dev/null
+++ b/src/test/conf_examples/relay_34/expected_log
@@ -0,0 +1 @@
+Your ContactInfo config option is not set
\ No newline at end of file
diff --git a/src/test/conf_examples/relay_34/torrc b/src/test/conf_examples/relay_34/torrc
new file mode 100644
index 0000000000..01010a5c38
--- /dev/null
+++ b/src/test/conf_examples/relay_34/torrc
@@ -0,0 +1,5 @@
+# Relay tests
+# explicit IPv4 ORPort and DirPort auto and IPv6 ORPort auto
+ORPort 127.0.0.1:auto
+ORPort [::1]:auto
+DirPort 127.0.0.1:auto
diff --git a/src/test/conf_examples/relpath_rad/error b/src/test/conf_examples/relpath_rad/error
new file mode 100644
index 0000000000..e131744475
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/error
@@ -0,0 +1 @@
+RunAsDaemon is not compatible with relative paths.
diff --git a/src/test/conf_examples/relpath_rad/torrc b/src/test/conf_examples/relpath_rad/torrc
new file mode 100644
index 0000000000..fe02441c3f
--- /dev/null
+++ b/src/test/conf_examples/relpath_rad/torrc
@@ -0,0 +1,4 @@
+
+# Relative-path data directories are incompatible with RunAsDaemon
+DataDirectory ./datadir
+RunAsDaemon 1
diff --git a/src/test/conf_failures/README b/src/test/conf_failures/README
new file mode 100644
index 0000000000..0da470eeb4
--- /dev/null
+++ b/src/test/conf_failures/README
@@ -0,0 +1,5 @@
+This directory contains typical test_parseconf.sh failure cases.
+
+If these directories are copied into conf_examples, test_parseconf.sh will
+fail. Use these failure cases to make sure test_parseconf.sh handles failures
+correctly, and produces useful output.
diff --git a/src/test/conf_failures/fail-error-success/error b/src/test/conf_failures/fail-error-success/error
new file mode 100644
index 0000000000..569a631e86
--- /dev/null
+++ b/src/test/conf_failures/fail-error-success/error
@@ -0,0 +1 @@
+Tor
diff --git a/src/test/conf_failures/fail-error-success/torrc b/src/test/conf_failures/fail-error-success/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-error-success/torrc
diff --git a/src/test/conf_failures/fail-error/error b/src/test/conf_failures/fail-error/error
new file mode 100644
index 0000000000..4c0be97832
--- /dev/null
+++ b/src/test/conf_failures/fail-error/error
@@ -0,0 +1 @@
+no match
diff --git a/src/test/conf_failures/fail-error/torrc b/src/test/conf_failures/fail-error/torrc
new file mode 100644
index 0000000000..bb6fe186a4
--- /dev/null
+++ b/src/test/conf_failures/fail-error/torrc
@@ -0,0 +1 @@
+bad bad bad
diff --git a/src/test/conf_failures/fail-expected-error/expected b/src/test/conf_failures/fail-expected-error/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-error/expected
diff --git a/src/test/conf_failures/fail-expected-error/torrc b/src/test/conf_failures/fail-expected-error/torrc
new file mode 100644
index 0000000000..bb6fe186a4
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-error/torrc
@@ -0,0 +1 @@
+bad bad bad
diff --git a/src/test/conf_failures/fail-expected-log/expected b/src/test/conf_failures/fail-expected-log/expected
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-log/expected
diff --git a/src/test/conf_failures/fail-expected-log/expected_log b/src/test/conf_failures/fail-expected-log/expected_log
new file mode 100644
index 0000000000..bb6fe186a4
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-log/expected_log
@@ -0,0 +1 @@
+bad bad bad
diff --git a/src/test/conf_failures/fail-expected-log/torrc b/src/test/conf_failures/fail-expected-log/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected-log/torrc
diff --git a/src/test/conf_failures/fail-expected/expected b/src/test/conf_failures/fail-expected/expected
new file mode 100644
index 0000000000..67be85f127
--- /dev/null
+++ b/src/test/conf_failures/fail-expected/expected
@@ -0,0 +1 @@
+bad
diff --git a/src/test/conf_failures/fail-expected/torrc b/src/test/conf_failures/fail-expected/torrc
new file mode 100644
index 0000000000..e69de29bb2
--- /dev/null
+++ b/src/test/conf_failures/fail-expected/torrc
diff --git a/src/test/ed25519_exts_ref.py b/src/test/ed25519_exts_ref.py
index 75562184b5..ae537ff15b 100644
--- a/src/test/ed25519_exts_ref.py
+++ b/src/test/ed25519_exts_ref.py
@@ -8,6 +8,11 @@
    Includes self-tester and test vector generator.
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import slow_ed25519
 from slow_ed25519 import *
 
@@ -147,7 +152,7 @@ class SelfTest(unittest.TestCase):
         # Check that identities match
         assert(identity == identity2)
         # Check that identity is the point (0,1)
-        assert(identity == [0L,1L])
+        assert(identity == [0,1])
 
         # Check identity element: a*E = E, where a is a random scalar
         scalar = random_scalar(os.urandom)
@@ -181,22 +186,22 @@ BLINDING_PARAMS = [
 PREFIX = "ED25519_"
 
 def writeArray(name, array):
-    print "static const char *{prefix}{name}[] = {{".format(
-        prefix=PREFIX,name=name)
+    print("static const char *{prefix}{name}[] = {{".format(
+        prefix=PREFIX,name=name))
     for a in array:
         h = binascii.b2a_hex(a)
         if len(h) > 70:
             h1 = h[:70]
             h2 = h[70:]
-            print '  "{0}"\n      "{1}",'.format(h1,h2)
+            print('  "{0}"\n      "{1}",'.format(h1,h2))
         else:
-            print '  "{0}",'.format(h)
-    print "};\n"
+            print('  "{0}",'.format(h))
+    print("};\n")
 
 def comment(text, initial="/**"):
-    print initial
-    print textwrap.fill(text,initial_indent=" * ",subsequent_indent=" * ")
-    print " */"
+    print(initial)
+    print(textwrap.fill(text,initial_indent=" * ",subsequent_indent=" * "))
+    print(" */")
 
 def makeTestVectors():
     comment("""Test vectors for our ed25519 implementation and related
@@ -252,7 +257,7 @@ def makeTestVectors():
 if __name__ == '__main__':
     import sys
     if len(sys.argv) == 1 or sys.argv[1] not in ("SelfTest", "MakeVectors"):
-        print "You should specify one of 'SelfTest' or 'MakeVectors'"
+        print("You should specify one of 'SelfTest' or 'MakeVectors'")
         sys.exit(1)
     if sys.argv[1] == 'SelfTest':
         unittest.main()
diff --git a/src/test/fakechans.h b/src/test/fakechans.h
index 4006e1bec4..c3accb1637 100644
--- a/src/test/fakechans.h
+++ b/src/test/fakechans.h
@@ -1,4 +1,4 @@
- /* Copyright (c) 2014-2019, The Tor Project, Inc. */
+ /* Copyright (c) 2014-2020, The Tor Project, Inc. */
  /* See LICENSE for licensing information */
 
 #ifndef TOR_FAKECHANS_H
diff --git a/src/test/fakecircs.c b/src/test/fakecircs.c
new file mode 100644
index 0000000000..4d5b97197e
--- /dev/null
+++ b/src/test/fakecircs.c
@@ -0,0 +1,91 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fakecircs.c
+ * \brief Fake circuits API for unit test.
+ **/
+
+#define CIRCUITBUILD_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define CRYPT_PATH_PRIVATE
+
+#include "core/or/or.h"
+
+#include "core/crypto/relay_crypto.h"
+#include "core/or/channel.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/crypt_path.h"
+#include "core/or/relay.h"
+#include "core/or/relay_crypto_st.h"
+
+#include "test/fakecircs.h"
+
+/** Return newly allocated OR circuit using the given nchan and pchan. It must
+ * be freed with the free_fake_orcirc(). */
+or_circuit_t *
+new_fake_orcirc(channel_t *nchan, channel_t *pchan)
+{
+  or_circuit_t *orcirc = NULL;
+  circuit_t *circ = NULL;
+  crypt_path_t tmp_cpath;
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+
+  orcirc = tor_malloc_zero(sizeof(*orcirc));
+  circ = &(orcirc->base_);
+  circ->magic = OR_CIRCUIT_MAGIC;
+
+  circuit_set_n_circid_chan(circ, get_unique_circ_id_by_chan(nchan), nchan);
+  cell_queue_init(&(circ->n_chan_cells));
+
+  circ->n_hop = NULL;
+  circ->streams_blocked_on_n_chan = 0;
+  circ->streams_blocked_on_p_chan = 0;
+  circ->n_delete_pending = 0;
+  circ->p_delete_pending = 0;
+  circ->received_destroy = 0;
+  circ->state = CIRCUIT_STATE_OPEN;
+  circ->purpose = CIRCUIT_PURPOSE_OR;
+  circ->package_window = CIRCWINDOW_START_MAX;
+  circ->deliver_window = CIRCWINDOW_START_MAX;
+  circ->n_chan_create_cell = NULL;
+
+  circuit_set_p_circid_chan(orcirc, get_unique_circ_id_by_chan(pchan), pchan);
+  cell_queue_init(&(orcirc->p_chan_cells));
+
+  memset(&tmp_cpath, 0, sizeof(tmp_cpath));
+  if (cpath_init_circuit_crypto(&tmp_cpath, whatevs_key,
+                                sizeof(whatevs_key), 0, 0)<0) {
+    log_warn(LD_BUG,"Circuit initialization failed");
+    return NULL;
+  }
+  orcirc->crypto = tmp_cpath.pvt_crypto;
+
+  return orcirc;
+}
+
+/** Free fake OR circuit which MUST be created by new_fake_orcirc(). */
+void
+free_fake_orcirc(or_circuit_t *orcirc)
+{
+  if (!orcirc) {
+    return;
+  }
+
+  circuit_t *circ = TO_CIRCUIT(orcirc);
+
+  relay_crypto_clear(&orcirc->crypto);
+
+  circpad_circuit_free_all_machineinfos(circ);
+
+  if (orcirc->p_chan && orcirc->p_chan->cmux) {
+    circuitmux_detach_circuit(orcirc->p_chan->cmux, circ);
+  }
+  if (circ->n_chan && circ->n_chan->cmux) {
+    circuitmux_detach_circuit(circ->n_chan->cmux, circ);
+  }
+
+  tor_free_(circ);
+}
diff --git a/src/test/fakecircs.h b/src/test/fakecircs.h
new file mode 100644
index 0000000000..ed8a150a3f
--- /dev/null
+++ b/src/test/fakecircs.h
@@ -0,0 +1,17 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fakecircs.h
+ * \brief Declarations for fake circuits for test suite use.
+ **/
+
+#ifndef TOR_FAKECIRCS_H
+#define TOR_FAKECIRCS_H
+
+#include "core/or/or_circuit_st.h"
+
+or_circuit_t *new_fake_orcirc(channel_t *nchan, channel_t *pchan);
+void free_fake_orcirc(or_circuit_t *orcirc);
+
+#endif /* !defined(TOR_FAKECIRCS_H) */
diff --git a/src/test/fuzz/.may_include b/src/test/fuzz/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/test/fuzz/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/test/fuzz/fixup_filenames.sh b/src/test/fuzz/fixup_filenames.sh
index 68efc1abc5..f730d532a5 100755
--- a/src/test/fuzz/fixup_filenames.sh
+++ b/src/test/fuzz/fixup_filenames.sh
@@ -8,9 +8,9 @@ if [ ! -d "$1" ] ; then
 fi
 
 for fn in "$1"/* ; do
-    prev=`basename "$fn"`
-    post=`sha256sum "$fn" | sed -e 's/ .*//;'`
-    if [ "$prev" == "$post" ] ; then
+    prev=$(basename "$fn")
+    post=$(sha256sum "$fn" | sed -e 's/ .*//;')
+    if [ "$prev" = "$post" ] ; then
       echo "OK $prev"
     else
       echo "mv $prev $post"
diff --git a/src/test/fuzz/fuzz_consensus.c b/src/test/fuzz/fuzz_consensus.c
index 5947a3f48c..f2bf29ea78 100644
--- a/src/test/fuzz/fuzz_consensus.c
+++ b/src/test/fuzz/fuzz_consensus.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define SIGCOMMON_PRIVATE
 #include "core/or/or.h"
@@ -61,13 +61,13 @@ int
 fuzz_main(const uint8_t *data, size_t sz)
 {
   networkstatus_t *ns;
-  char *str = tor_memdup_nulterm(data, sz);
   const char *eos = NULL;
   networkstatus_type_t tp = NS_TYPE_CONSENSUS;
   if (tor_memstr(data, MIN(sz, 1024), "tus vote"))
     tp = NS_TYPE_VOTE;
   const char *what = (tp == NS_TYPE_CONSENSUS) ? "consensus" : "vote";
-  ns = networkstatus_parse_vote_from_string(str,
+  ns = networkstatus_parse_vote_from_string((const char *)data,
+                                            sz,
                                             &eos,
                                             tp);
   if (ns) {
@@ -76,6 +76,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   } else {
     log_debug(LD_GENERAL, "Parsing as %s failed", what);
   }
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzz_descriptor.c b/src/test/fuzz/fuzz_descriptor.c
index 58ee3dbc35..eb4d4d507f 100644
--- a/src/test/fuzz/fuzz_descriptor.c
+++ b/src/test/fuzz/fuzz_descriptor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define SIGCOMMON_PRIVATE
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_diff.c b/src/test/fuzz/fuzz_diff.c
index 1bc60e50ee..9cd2116245 100644
--- a/src/test/fuzz/fuzz_diff.c
+++ b/src/test/fuzz/fuzz_diff.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFF_PRIVATE
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -42,28 +44,34 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_generate(c1, c2);
+  const char *cp = memchr(c1, 0, c1_len);
+  if (cp)
+    c1_len = cp - c1;
+
+  cp = memchr(c2, 0, c2_len);
+  if (cp)
+    c2_len = cp - c2;
+
+  char *c3 = consensus_diff_generate(c1, c1_len, c2, c2_len);
 
   if (c3) {
-    char *c4 = consensus_diff_apply(c1, c3);
+    char *c4 = consensus_diff_apply(c1, c1_len, c3, strlen(c3));
     tor_assert(c4);
-    if (strcmp(c2, c4)) {
-      printf("%s\n", escaped(c1));
-      printf("%s\n", escaped(c2));
+    int equal = (c2_len == strlen(c4)) && fast_memeq(c2, c4, c2_len);
+    if (! equal) {
+      //printf("%s\n", escaped(c1));
+      //printf("%s\n", escaped(c2));
       printf("%s\n", escaped(c3));
       printf("%s\n", escaped(c4));
     }
-    tor_assert(! strcmp(c2, c4));
+    tor_assert(equal);
     tor_free(c3);
     tor_free(c4);
   }
-  tor_free(c1);
-  tor_free(c2);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_diff_apply.c b/src/test/fuzz/fuzz_diff_apply.c
index 9bd3cb0bf8..a819c73338 100644
--- a/src/test/fuzz/fuzz_diff_apply.c
+++ b/src/test/fuzz/fuzz_diff_apply.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFF_PRIVATE
@@ -10,9 +10,11 @@
 #include "test/fuzz/fuzzing.h"
 
 static int
-mock_consensus_compute_digest_(const char *c, consensus_digest_t *d)
+mock_consensus_compute_digest_(const char *c, size_t len,
+                               consensus_digest_t *d)
 {
   (void)c;
+  (void)len;
   memset(d->sha3_256, 3, sizeof(d->sha3_256));
   return 0;
 }
@@ -50,16 +52,13 @@ fuzz_main(const uint8_t *stdin_buf, size_t data_size)
   if (! separator)
     return 0;
   size_t c1_len = separator - stdin_buf;
-  char *c1 = tor_memdup_nulterm(stdin_buf, c1_len);
+  const char *c1 = (const char *)stdin_buf;
   size_t c2_len = data_size - c1_len - SEPLEN;
-  char *c2 = tor_memdup_nulterm(separator + SEPLEN, c2_len);
+  const char *c2 = (const char *)separator + SEPLEN;
 
-  char *c3 = consensus_diff_apply(c1, c2);
+  char *c3 = consensus_diff_apply(c1, c1_len, c2, c2_len);
 
-  tor_free(c1);
-  tor_free(c2);
   tor_free(c3);
 
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_extrainfo.c b/src/test/fuzz/fuzz_extrainfo.c
index f18bd68d65..ad21254e3e 100644
--- a/src/test/fuzz/fuzz_extrainfo.c
+++ b/src/test/fuzz/fuzz_extrainfo.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define SIGCOMMON_PRIVATE
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_hsdescv2.c b/src/test/fuzz/fuzz_hsdescv2.c
index 34639b237c..81d9e5f00e 100644
--- a/src/test/fuzz/fuzz_hsdescv2.c
+++ b/src/test/fuzz/fuzz_hsdescv2.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #include "core/or/or.h"
 #include "feature/dirparse/unparseable.h"
diff --git a/src/test/fuzz/fuzz_hsdescv3.c b/src/test/fuzz/fuzz_hsdescv3.c
index 2cbd655898..8d7eab1a8d 100644
--- a/src/test/fuzz/fuzz_hsdescv3.c
+++ b/src/test/fuzz/fuzz_hsdescv3.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define HS_DESCRIPTOR_PRIVATE
@@ -35,16 +35,21 @@ mock_rsa_ed25519_crosscert_check(const uint8_t *crosscert,
 
 static size_t
 mock_decrypt_desc_layer(const hs_descriptor_t *desc,
-                        const uint8_t *encrypted_blob,
-                        size_t encrypted_blob_size,
                         const uint8_t *descriptor_cookie,
-                        int is_superencrypted_layer,
+                        bool is_superencrypted_layer,
                         char **decrypted_out)
 {
   (void)is_superencrypted_layer;
   (void)desc;
   (void)descriptor_cookie;
   const size_t overhead = HS_DESC_ENCRYPTED_SALT_LEN + DIGEST256_LEN;
+  const uint8_t *encrypted_blob = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob
+    : desc->superencrypted_data.encrypted_blob;
+  size_t encrypted_blob_size = (is_superencrypted_layer)
+    ? desc->plaintext_data.superencrypted_blob_size
+    : desc->superencrypted_data.encrypted_blob_size;
+
   if (encrypted_blob_size < overhead)
     return 0;
   *decrypted_out = tor_memdup_nulterm(
@@ -80,12 +85,12 @@ int
 fuzz_main(const uint8_t *data, size_t sz)
 {
   hs_descriptor_t *desc = NULL;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   char *fuzzing_data = tor_memdup_nulterm(data, sz);
-  memset(subcredential, 'A', sizeof(subcredential));
+  memset(&subcredential, 'A', sizeof(subcredential));
 
-  hs_desc_decode_descriptor(fuzzing_data, subcredential, NULL, &desc);
+  hs_desc_decode_descriptor(fuzzing_data, &subcredential, NULL, &desc);
   if (desc) {
     log_debug(LD_GENERAL, "Decoding okay");
     hs_descriptor_free(desc);
@@ -96,4 +101,3 @@ fuzz_main(const uint8_t *data, size_t sz)
   tor_free(fuzzing_data);
   return 0;
 }
-
diff --git a/src/test/fuzz/fuzz_http.c b/src/test/fuzz/fuzz_http.c
index 2798c47d23..a4fd182f1e 100644
--- a/src/test/fuzz/fuzz_http.c
+++ b/src/test/fuzz/fuzz_http.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/dircache/dircache.h"
diff --git a/src/test/fuzz/fuzz_http_connect.c b/src/test/fuzz/fuzz_http_connect.c
index a60fc36804..9e5a48ba4d 100644
--- a/src/test/fuzz/fuzz_http_connect.c
+++ b/src/test/fuzz/fuzz_http_connect.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
diff --git a/src/test/fuzz/fuzz_iptsv2.c b/src/test/fuzz/fuzz_iptsv2.c
index 76fa3c164e..bc51ffcdb8 100644
--- a/src/test/fuzz/fuzz_iptsv2.c
+++ b/src/test/fuzz/fuzz_iptsv2.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_microdesc.c b/src/test/fuzz/fuzz_microdesc.c
index 28fdc5e24d..3fc709183b 100644
--- a/src/test/fuzz/fuzz_microdesc.c
+++ b/src/test/fuzz/fuzz_microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/fuzz/fuzz_multi.sh b/src/test/fuzz/fuzz_multi.sh
index b4a17ed8cb..406ab498d9 100755
--- a/src/test/fuzz/fuzz_multi.sh
+++ b/src/test/fuzz/fuzz_multi.sh
@@ -1,3 +1,5 @@
+#!/bin/sh
+
 MEMLIMIT_BYTES=21990500990976
 
 N_CPUS=1
@@ -6,9 +8,9 @@ if [ $# -ge 1 ]; then
     shift
 fi
 
-FILTER=echo
+FILTER="echo"
 
-for i in `seq -w "$N_CPUS"`; do
+for i in $(seq -w "$N_CPUS"); do
     if [ "$i" -eq 1 ]; then
         if [ "$N_CPUS" -eq 1 ]; then
             INSTANCE=""
diff --git a/src/test/fuzz/fuzz_socks.c b/src/test/fuzz/fuzz_socks.c
index 06cb08391e..4e7cb4d48d 100644
--- a/src/test/fuzz/fuzz_socks.c
+++ b/src/test/fuzz/fuzz_socks.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/err/backtrace.h"
 #include "lib/log/log.h"
 #include "core/proto/proto_socks.h"
diff --git a/src/test/fuzz/fuzz_strops.c b/src/test/fuzz/fuzz_strops.c
new file mode 100644
index 0000000000..05d9101e72
--- /dev/null
+++ b/src/test/fuzz/fuzz_strops.c
@@ -0,0 +1,253 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file fuzz_strops.c
+ * \brief Fuzzers for various string encoding/decoding operations
+ **/
+
+#include "orconfig.h"
+
+#include "lib/cc/torint.h"
+#include "lib/ctime/di_ops.h"
+#include "lib/encoding/binascii.h"
+#include "lib/encoding/cstring.h"
+#include "lib/encoding/kvline.h"
+#include "lib/encoding/confline.h"
+#include "lib/malloc/malloc.h"
+#include "lib/log/escape.h"
+#include "lib/log/util_bug.h"
+#include "lib/intmath/muldiv.h"
+
+#include "test/fuzz/fuzzing.h"
+
+#include <stdio.h>
+#include <string.h>
+
+int
+fuzz_init(void)
+{
+  return 0;
+}
+
+int
+fuzz_cleanup(void)
+{
+  return 0;
+}
+
+typedef struct chunk_t {
+  uint8_t *buf;
+  size_t len;
+} chunk_t;
+
+#define chunk_free(ch)                          \
+  FREE_AND_NULL(chunk_t, chunk_free_, (ch))
+
+static chunk_t *
+chunk_new(size_t len)
+{
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = tor_malloc(len);
+  ch->len = len;
+  return ch;
+}
+static void
+chunk_free_(chunk_t *ch)
+{
+  if (!ch)
+    return;
+  tor_free(ch->buf);
+  tor_free(ch);
+}
+static bool
+chunk_eq(const chunk_t *a, const chunk_t *b)
+{
+  return a->len == b->len && fast_memeq(a->buf, b->buf, a->len);
+}
+
+static chunk_t *
+b16_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(CEIL_DIV(inp->len, 2));
+  int r = base16_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b16_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len * 2 + 1);
+  base16_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  return ch;
+}
+
+static chunk_t *
+b32_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX
+  int r = base32_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b32_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(base32_encoded_size(inp->len));
+  base32_encode((char *)ch->buf, ch->len, (char*)inp->buf, inp->len);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+b64_dec(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(inp->len);//XXXX This could be shorter.
+  int r = base64_decode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len);
+  if (r >= 0) {
+    ch->len = r;
+  } else {
+    chunk_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+b64_enc(const chunk_t *inp)
+{
+  chunk_t *ch = chunk_new(BASE64_BUFSIZE(inp->len));
+  base64_encode((char *)ch->buf, ch->len, (char *)inp->buf, inp->len, 0);
+  ch->len = strlen((char *) ch->buf);
+  return ch;
+}
+
+static chunk_t *
+c_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  char *r = NULL;
+  (void) unescape_string(s, &r, &ch->len);
+  tor_free(s);
+  ch->buf = (uint8_t*) r;
+  if (!ch->buf) {
+    tor_free(ch);
+  }
+  return ch;
+}
+static chunk_t *
+c_enc(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  chunk_t *ch = tor_malloc(sizeof(chunk_t));
+  ch->buf = (uint8_t*)esc_for_log(s);
+  tor_free(s);
+  ch->len = strlen((char*)ch->buf);
+  return ch;
+}
+
+static int kv_flags = 0;
+static config_line_t *
+kv_dec(const chunk_t *inp)
+{
+  char *s = tor_memdup_nulterm(inp->buf, inp->len);
+  config_line_t *res = kvline_parse(s, kv_flags);
+  tor_free(s);
+  return res;
+}
+static chunk_t *
+kv_enc(const config_line_t *inp)
+{
+  char *s = kvline_encode(inp, kv_flags);
+  if (!s)
+    return NULL;
+  chunk_t *res = tor_malloc(sizeof(chunk_t));
+  res->buf = (uint8_t*)s;
+  res->len = strlen(s);
+  return res;
+}
+
+/* Given an encoder function, a decoder function, and a function to free
+ * the decoded object, check whether any string that successfully decoded
+ * will then survive an encode-decode-encode round-trip unchanged.
+ */
+#define ENCODE_ROUNDTRIP(E,D,FREE)              \
+  STMT_BEGIN {                                  \
+    bool err = false;                           \
+    a = D(&inp);                                \
+    if (!a)                                     \
+      return 0;                                 \
+    b = E(a);                                   \
+    tor_assert(b);                              \
+    c = D(b);                                   \
+    tor_assert(c);                              \
+    d = E(c);                                   \
+    tor_assert(d);                              \
+    if (!chunk_eq(b,d)) {                       \
+      printf("Unequal chunks: %s\n",            \
+             hex_str((char*)b->buf, b->len));   \
+      printf("             vs %s\n",            \
+             hex_str((char*)d->buf, d->len));   \
+      err = true;                               \
+    }                                           \
+    FREE(a);                                    \
+    chunk_free(b);                              \
+    FREE(c);                                    \
+    chunk_free(d);                              \
+    tor_assert(!err);                           \
+  } STMT_END
+
+int
+fuzz_main(const uint8_t *stdin_buf, size_t data_size)
+{
+  if (!data_size)
+    return 0;
+
+  chunk_t inp = { (uint8_t*)stdin_buf, data_size };
+  chunk_t *b=NULL,*d=NULL;
+  void *a=NULL,*c=NULL;
+
+  switch (stdin_buf[0]) {
+    case 0:
+      ENCODE_ROUNDTRIP(b16_enc, b16_dec, chunk_free_);
+      break;
+    case 1:
+      ENCODE_ROUNDTRIP(b32_enc, b32_dec, chunk_free_);
+      break;
+    case 2:
+      ENCODE_ROUNDTRIP(b64_enc, b64_dec, chunk_free_);
+      break;
+    case 3:
+      ENCODE_ROUNDTRIP(c_enc, c_dec, chunk_free_);
+      break;
+    case 5:
+      kv_flags = KV_QUOTED|KV_OMIT_KEYS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 6:
+      kv_flags = 0;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 7:
+      kv_flags = KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 8:
+      kv_flags = KV_QUOTED;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    case 9:
+      kv_flags = KV_QUOTED|KV_OMIT_VALS;
+      ENCODE_ROUNDTRIP(kv_enc, kv_dec, config_free_lines_);
+      break;
+    }
+
+  return 0;
+}
diff --git a/src/test/fuzz/fuzz_vrs.c b/src/test/fuzz/fuzz_vrs.c
index 967397d1af..d6e88e59e7 100644
--- a/src/test/fuzz/fuzz_vrs.c
+++ b/src/test/fuzz/fuzz_vrs.c
@@ -1,8 +1,9 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define NS_PARSE_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #include "core/or/or.h"
+#include "feature/dirauth/dirvote.h"
 #include "feature/dirparse/ns_parse.h"
 #include "feature/dirparse/unparseable.h"
 #include "lib/memarea/memarea.h"
@@ -35,9 +36,12 @@ fuzz_init(void)
   dummy_vote = tor_malloc_zero(sizeof(*dummy_vote));
   dummy_vote->known_flags = smartlist_new();
   smartlist_split_string(dummy_vote->known_flags,
-                         "Authority BadExit Exit Fast Guard HSDir "
-                         "NoEdConsensus Running Stable V2Dir Valid",
+                         DIRVOTE_UNIVERSAL_FLAGS,
                          " ", 0, 0);
+  smartlist_split_string(dummy_vote->known_flags,
+                         DIRVOTE_OPTIONAL_FLAGS,
+                         " ", 0, 0);
+  smartlist_sort_strings(dummy_vote->known_flags);
   return 0;
 }
 
@@ -53,24 +57,24 @@ fuzz_cleanup(void)
 int
 fuzz_main(const uint8_t *data, size_t sz)
 {
-  char *str = tor_memdup_nulterm(data, sz);
   const char *s;
   routerstatus_t *rs_ns = NULL, *rs_md = NULL, *rs_vote = NULL;
   vote_routerstatus_t *vrs = tor_malloc_zero(sizeof(*vrs));
   smartlist_t *tokens = smartlist_new();
+  const char *eos = (const char *)data + sz;
 
-  s = str;
-  rs_ns = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_ns = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_md = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_md = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                                NULL, NULL, 26, FLAV_MICRODESC);
   tor_assert(smartlist_len(tokens) == 0);
 
-  s = str;
-  rs_vote = routerstatus_parse_entry_from_string(area, &s, tokens,
+  s = (const char *)data;
+  rs_vote = routerstatus_parse_entry_from_string(area, &s, eos, tokens,
                                               dummy_vote, vrs, 26, FLAV_NS);
   tor_assert(smartlist_len(tokens) == 0);
 
@@ -82,6 +86,6 @@ fuzz_main(const uint8_t *data, size_t sz)
   vote_routerstatus_free(vrs);
   memarea_clear(area);
   smartlist_free(tokens);
-  tor_free(str);
+
   return 0;
 }
diff --git a/src/test/fuzz/fuzzing.h b/src/test/fuzz/fuzzing.h
index 150ac4aa7d..6cbcdc41ad 100644
--- a/src/test/fuzz/fuzzing.h
+++ b/src/test/fuzz/fuzzing.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #ifndef FUZZING_H
 #define FUZZING_H
@@ -9,5 +9,5 @@ int fuzz_main(const uint8_t *data, size_t sz);
 
 void disable_signature_checking(void);
 
-#endif /* FUZZING_H */
+#endif /* !defined(FUZZING_H) */
 
diff --git a/src/test/fuzz/fuzzing_common.c b/src/test/fuzz/fuzzing_common.c
index 8ea4898522..d9719074ad 100644
--- a/src/test/fuzz/fuzzing_common.c
+++ b/src/test/fuzz/fuzzing_common.c
@@ -1,14 +1,17 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define CRYPTO_ED25519_PRIVATE
+#define CONFIG_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
+#include "app/main/subsysmgr.h"
 #include "lib/err/backtrace.h"
 #include "app/config/config.h"
 #include "test/fuzz/fuzzing.h"
 #include "lib/compress/compress.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
 
 static or_options_t *mock_options = NULL;
 static const or_options_t *
@@ -94,12 +97,10 @@ disable_signature_checking(void)
 static void
 global_init(void)
 {
-  tor_threads_init();
-  tor_compress_init();
+  subsystems_init_upto(SUBSYS_LEVEL_LIBS);
+  flush_log_messages_from_startup();
 
-  /* Initialise logging first */
-  init_logging(1);
-  configure_backtrace_handler(get_version());
+  tor_compress_init();
 
   if (crypto_global_init(0, NULL, NULL) < 0)
     abort();
@@ -111,7 +112,7 @@ global_init(void)
   }
 
   /* set up the options. */
-  mock_options = tor_malloc_zero(sizeof(or_options_t));
+  mock_options = options_new();
   MOCK(get_options, mock_get_options);
 
   /* Make BUG() and nonfatal asserts crash */
@@ -137,7 +138,7 @@ LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
   return fuzz_main(Data, Size);
 }
 
-#else /* Not LLVM_FUZZ, so AFL. */
+#else /* !defined(LLVM_FUZZ) */
 
 int
 main(int argc, char **argv)
@@ -166,7 +167,7 @@ main(int argc, char **argv)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
 
@@ -189,9 +190,9 @@ main(int argc, char **argv)
   if (fuzz_cleanup() < 0)
     abort();
 
-  tor_free(mock_options);
+  or_options_free(mock_options);
   UNMOCK(get_options);
   return 0;
 }
 
-#endif
+#endif /* defined(LLVM_FUZZ) */
diff --git a/src/test/fuzz/include.am b/src/test/fuzz/include.am
index 27eeced8c5..d0711f05d6 100644
--- a/src/test/fuzz/include.am
+++ b/src/test/fuzz/include.am
@@ -153,6 +153,16 @@ src_test_fuzz_fuzz_socks_LDADD = $(FUZZING_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_fuzz_strops_SOURCES = \
+	src/test/fuzz/fuzzing_common.c \
+	src/test/fuzz/fuzz_strops.c
+src_test_fuzz_fuzz_strops_CPPFLAGS = $(FUZZING_CPPFLAGS)
+src_test_fuzz_fuzz_strops_CFLAGS = $(FUZZING_CFLAGS)
+src_test_fuzz_fuzz_strops_LDFLAGS = $(FUZZING_LDFLAG)
+src_test_fuzz_fuzz_strops_LDADD = $(FUZZING_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_fuzz_vrs_SOURCES = \
 	src/test/fuzz/fuzzing_common.c \
 	src/test/fuzz/fuzz_vrs.c
@@ -176,6 +186,7 @@ FUZZERS = \
 	src/test/fuzz/fuzz-iptsv2 \
 	src/test/fuzz/fuzz-microdesc \
 	src/test/fuzz/fuzz-socks \
+	src/test/fuzz/fuzz-strops \
 	src/test/fuzz/fuzz-vrs
 endif
 
@@ -291,6 +302,15 @@ src_test_fuzz_lf_fuzz_socks_LDADD = $(LIBFUZZER_LIBS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_lf_fuzz_strops_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_lf_fuzz_strops_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
+src_test_fuzz_lf_fuzz_strops_CFLAGS = $(LIBFUZZER_CFLAGS)
+src_test_fuzz_lf_fuzz_strops_LDFLAGS = $(LIBFUZZER_LDFLAG)
+src_test_fuzz_lf_fuzz_strops_LDADD = $(LIBFUZZER_LIBS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_lf_fuzz_vrs_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_lf_fuzz_vrs_CPPFLAGS = $(LIBFUZZER_CPPFLAGS)
@@ -312,6 +332,7 @@ LIBFUZZER_FUZZERS = \
 	src/test/fuzz/lf-fuzz-iptsv2 \
 	src/test/fuzz/lf-fuzz-microdesc \
 	src/test/fuzz/lf-fuzz-socks \
+	src/test/fuzz/lf-fuzz-strops \
 	src/test/fuzz/lf-fuzz-vrs
 
 else
@@ -406,6 +427,13 @@ src_test_fuzz_liboss_fuzz_socks_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
 endif
 
 if UNITTESTS_ENABLED
+src_test_fuzz_liboss_fuzz_strops_a_SOURCES = \
+	$(src_test_fuzz_fuzz_strops_SOURCES)
+src_test_fuzz_liboss_fuzz_strops_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
+src_test_fuzz_liboss_fuzz_strops_a_CFLAGS = $(LIBOSS_FUZZ_CFLAGS)
+endif
+
+if UNITTESTS_ENABLED
 src_test_fuzz_liboss_fuzz_vrs_a_SOURCES = \
 	$(src_test_fuzz_fuzz_vrs_SOURCES)
 src_test_fuzz_liboss_fuzz_vrs_a_CPPFLAGS = $(LIBOSS_FUZZ_CPPFLAGS)
@@ -425,6 +453,7 @@ OSS_FUZZ_FUZZERS = \
 	src/test/fuzz/liboss-fuzz-iptsv2.a \
 	src/test/fuzz/liboss-fuzz-microdesc.a \
 	src/test/fuzz/liboss-fuzz-socks.a \
+	src/test/fuzz/liboss-fuzz-strops.a \
 	src/test/fuzz/liboss-fuzz-vrs.a
 
 else
diff --git a/src/test/fuzz/minimize.sh b/src/test/fuzz/minimize.sh
index 87d3dda13c..ce43812bb8 100755
--- a/src/test/fuzz/minimize.sh
+++ b/src/test/fuzz/minimize.sh
@@ -7,7 +7,7 @@ if [ ! -d "$1" ] ; then
     exit 1
 fi
 
-which=`basename "$1"`
+which=$(basename "$1")
 
 mkdir "$1.out"
 afl-cmin -i "$1" -o "$1.out" -m none "./src/test/fuzz/fuzz-${which}"
diff --git a/src/test/fuzz_static_testcases.sh b/src/test/fuzz_static_testcases.sh
index f7b3adffb1..b883352402 100755
--- a/src/test/fuzz_static_testcases.sh
+++ b/src/test/fuzz_static_testcases.sh
@@ -14,7 +14,7 @@ fi
 
 
 for fuzzer in "${builddir:-.}"/src/test/fuzz/fuzz-* ; do
-    f=`basename $fuzzer`
+    f=$(basename "$fuzzer")
     case="${f#fuzz-}"
     if [ -d "${TOR_FUZZ_CORPORA}/${case}" ]; then
         echo "Running tests for ${case}"
diff --git a/src/test/hs_build_address.py b/src/test/hs_build_address.py
index 7ff22c3a9a..91864eabcb 100644
--- a/src/test/hs_build_address.py
+++ b/src/test/hs_build_address.py
@@ -1,3 +1,8 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/hs_indexes.py b/src/test/hs_indexes.py
index af0b81f8de..5c6d893a66 100644
--- a/src/test/hs_indexes.py
+++ b/src/test/hs_indexes.py
@@ -7,6 +7,11 @@
 # store/fetch the descriptor on the hashring. (hs_build_hs_index()).
 #
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/hs_ntor_ref.py b/src/test/hs_ntor_ref.py
index d58ac3ca23..98025dd584 100644
--- a/src/test/hs_ntor_ref.py
+++ b/src/test/hs_ntor_ref.py
@@ -41,6 +41,11 @@ The whole logic and concept for this test suite was taken from ntor_ref.py.
                 *** DO NOT USE THIS IN PRODUCTION. ***
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import struct
 import os, sys
 import binascii
diff --git a/src/test/hs_test_helpers.c b/src/test/hs_test_helpers.c
index f2ae8398df..5116fc7169 100644
--- a/src/test/hs_test_helpers.c
+++ b/src/test/hs_test_helpers.c
@@ -1,17 +1,34 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define HS_CLIENT_PRIVATE
+
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "test/test.h"
 #include "feature/nodelist/torcert.h"
 
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_service.h"
 #include "test/hs_test_helpers.h"
 
+/**
+ * Create an introduction point taken straight out of an HSv3 descriptor.
+ *
+ * Use 'signing_kp' to sign the introduction point certificates.
+ *
+ * If 'intro_auth_kp' is provided use that as the introduction point
+ * authentication keypair, otherwise generate one on the fly.
+ *
+ * If 'intro_enc_kp' is provided use that as the introduction point encryption
+ * keypair, otherwise generate one on the fly.
+ */
 hs_desc_intro_point_t *
 hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
-                            const char *addr, int legacy)
+                            const char *addr, int legacy,
+                            const ed25519_keypair_t *intro_auth_kp,
+                            const curve25519_keypair_t *intro_enc_kp)
 {
   int ret;
   ed25519_keypair_t auth_kp;
@@ -21,30 +38,43 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
   /* For a usable intro point we need at least two link specifiers: One legacy
    * keyid and one ipv4 */
   {
-    hs_desc_link_specifier_t *ls_legacy = tor_malloc_zero(sizeof(*ls_legacy));
-    hs_desc_link_specifier_t *ls_v4 = tor_malloc_zero(sizeof(*ls_v4));
-    ls_legacy->type = LS_LEGACY_ID;
-    memcpy(ls_legacy->u.legacy_id, "0299F268FCA9D55CD157976D39AE92B4B455B3A8",
-           DIGEST_LEN);
-    ls_v4->u.ap.port = 9001;
-    int family = tor_addr_parse(&ls_v4->u.ap.addr, addr);
+    tor_addr_t a;
+    tor_addr_make_unspec(&a);
+    link_specifier_t *ls_legacy = link_specifier_new();
+    link_specifier_t *ls_ip = link_specifier_new();
+    link_specifier_set_ls_type(ls_legacy, LS_LEGACY_ID);
+    memset(link_specifier_getarray_un_legacy_id(ls_legacy), 'C',
+           link_specifier_getlen_un_legacy_id(ls_legacy));
+    int family = tor_addr_parse(&a, addr);
     switch (family) {
     case AF_INET:
-          ls_v4->type = LS_IPV4;
+          link_specifier_set_ls_type(ls_ip, LS_IPV4);
+          link_specifier_set_un_ipv4_addr(ls_ip, tor_addr_to_ipv4h(&a));
+          link_specifier_set_un_ipv4_port(ls_ip, 9001);
           break;
         case AF_INET6:
-          ls_v4->type = LS_IPV6;
+          link_specifier_set_ls_type(ls_ip, LS_IPV6);
+          memcpy(link_specifier_getarray_un_ipv6_addr(ls_ip),
+                 tor_addr_to_in6_addr8(&a),
+                 link_specifier_getlen_un_ipv6_addr(ls_ip));
+          link_specifier_set_un_ipv6_port(ls_ip, 9001);
           break;
         default:
-          /* Stop the test, not suppose to have an error. */
-          tt_int_op(family, OP_EQ, AF_INET);
+          /* Stop the test, not supposed to have an error.
+           * Compare with -1 to show the actual family.
+           */
+          tt_int_op(family, OP_EQ, -1);
     }
     smartlist_add(ip->link_specifiers, ls_legacy);
-    smartlist_add(ip->link_specifiers, ls_v4);
+    smartlist_add(ip->link_specifiers, ls_ip);
   }
 
-  ret = ed25519_keypair_generate(&auth_kp, 0);
-  tt_int_op(ret, ==, 0);
+  if (intro_auth_kp) {
+    memcpy(&auth_kp, intro_auth_kp, sizeof(ed25519_keypair_t));
+  } else {
+    ret = ed25519_keypair_generate(&auth_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+  }
   ip->auth_key_cert = tor_cert_create(signing_kp, CERT_TYPE_AUTH_HS_IP_KEY,
                                       &auth_kp.pubkey, now,
                                       HS_DESC_CERT_LIFETIME,
@@ -55,7 +85,7 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
     ip->legacy.key = crypto_pk_new();
     tt_assert(ip->legacy.key);
     ret = crypto_pk_generate_key(ip->legacy.key);
-    tt_int_op(ret, ==, 0);
+    tt_int_op(ret, OP_EQ, 0);
     ssize_t cert_len = tor_make_rsa_ed25519_crosscert(
                                     &signing_kp->pubkey, ip->legacy.key,
                                     now + HS_DESC_CERT_LIFETIME,
@@ -72,8 +102,12 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
     ed25519_keypair_t ed25519_kp;
     tor_cert_t *cross_cert;
 
-    ret = curve25519_keypair_generate(&curve25519_kp, 0);
-    tt_int_op(ret, ==, 0);
+    if (intro_enc_kp) {
+      memcpy(&curve25519_kp, intro_enc_kp, sizeof(curve25519_keypair_t));
+    } else {
+      ret = curve25519_keypair_generate(&curve25519_kp, 0);
+      tt_int_op(ret, OP_EQ, 0);
+    }
     ed25519_keypair_from_curve25519_keypair(&ed25519_kp, &signbit,
                                             &curve25519_kp);
     cross_cert = tor_cert_create(signing_kp, CERT_TYPE_CROSS_HS_IP_KEYS,
@@ -82,6 +116,8 @@ hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
                                  CERT_FLAG_INCLUDE_SIGNING_KEY);
     tt_assert(cross_cert);
     ip->enc_key_cert = cross_cert;
+    memcpy(ip->enc_key.public_key, curve25519_kp.pubkey.public_key,
+           CURVE25519_PUBKEY_LEN);
   }
 
   intro_point = ip;
@@ -127,11 +163,11 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip,
   desc->plaintext_data.lifetime_sec = 3 * 60 * 60;
 
   hs_get_subcredential(&signing_kp->pubkey, &blinded_kp.pubkey,
-                    desc->subcredential);
+                    &desc->subcredential);
 
   /* Setup superencrypted data section. */
   ret = curve25519_keypair_generate(&auth_ephemeral_kp, 0);
-  tt_int_op(ret, ==, 0);
+  tt_int_op(ret, OP_EQ, 0);
   memcpy(&desc->superencrypted_data.auth_ephemeral_pubkey,
          &auth_ephemeral_kp.pubkey,
          sizeof(curve25519_public_key_t));
@@ -152,13 +188,17 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip,
   if (!no_ip) {
     /* Add four intro points. */
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0));
+                  hs_helper_build_intro_point(signing_kp, now, "1.2.3.4", 0,
+                                              NULL, NULL));
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0));
+                  hs_helper_build_intro_point(signing_kp, now, "[2600::1]", 0,
+                                              NULL, NULL));
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1));
+                  hs_helper_build_intro_point(signing_kp, now, "3.2.1.4", 1,
+                                              NULL, NULL));
     smartlist_add(desc->encrypted_data.intro_points,
-              hs_helper_build_intro_point(signing_kp, now, "5.6.7.8", 1));
+                  hs_helper_build_intro_point(signing_kp, now, "5.6.7.8", 1,
+                                              NULL, NULL));
   }
 
   descp = desc;
@@ -173,7 +213,7 @@ hs_helper_build_hs_desc_impl(unsigned int no_ip,
  *  an HS. Used to decrypt descriptors in unittests. */
 void
 hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
-                                            uint8_t *subcred_out)
+                                            hs_subcredential_t *subcred_out)
 {
   ed25519_keypair_t blinded_kp;
   uint64_t current_time_period = hs_get_time_period_num(approx_time());
@@ -198,11 +238,39 @@ hs_helper_build_hs_desc_no_ip(const ed25519_keypair_t *signing_kp)
   return hs_helper_build_hs_desc_impl(1, signing_kp);
 }
 
+hs_descriptor_t *
+hs_helper_build_hs_desc_with_client_auth(
+                        const uint8_t *descriptor_cookie,
+                        const curve25519_public_key_t *client_pk,
+                        const ed25519_keypair_t *signing_kp)
+{
+  curve25519_keypair_t auth_ephemeral_kp;
+  hs_descriptor_t *desc = hs_helper_build_hs_desc_impl(0, signing_kp);
+  hs_desc_authorized_client_t *desc_client;
+
+  /* The number of client authorized auth has tobe a multiple of
+   * HS_DESC_AUTH_CLIENT_MULTIPLE so remove one that we'll replace. */
+  desc_client = smartlist_get(desc->superencrypted_data.clients, 0);
+  smartlist_remove(desc->superencrypted_data.clients, desc_client);
+  hs_desc_authorized_client_free(desc_client);
+
+  desc_client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
+
+  curve25519_keypair_generate(&auth_ephemeral_kp, 0);
+  memcpy(&desc->superencrypted_data.auth_ephemeral_pubkey,
+         &auth_ephemeral_kp.pubkey, sizeof(curve25519_public_key_t));
+
+  hs_desc_build_authorized_client(&desc->subcredential, client_pk,
+                                  &auth_ephemeral_kp.seckey,
+                                  descriptor_cookie, desc_client);
+  smartlist_add(desc->superencrypted_data.clients, desc_client);
+  return desc;
+}
+
 void
 hs_helper_desc_equal(const hs_descriptor_t *desc1,
                      const hs_descriptor_t *desc2)
 {
-  char *addr1 = NULL, *addr2 = NULL;
   /* Plaintext data section. */
   tt_int_op(desc1->plaintext_data.version, OP_EQ,
             desc2->plaintext_data.version);
@@ -216,7 +284,7 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   tt_mem_op(desc1->plaintext_data.blinded_pubkey.pubkey, OP_EQ,
             desc2->plaintext_data.blinded_pubkey.pubkey,
             ED25519_PUBKEY_LEN);
-  tt_u64_op(desc1->plaintext_data.revision_counter, ==,
+  tt_u64_op(desc1->plaintext_data.revision_counter, OP_EQ,
             desc2->plaintext_data.revision_counter);
 
   /* NOTE: We can't compare the encrypted blob because when encoding the
@@ -233,7 +301,7 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   {
     tt_assert(desc1->superencrypted_data.clients);
     tt_assert(desc2->superencrypted_data.clients);
-    tt_int_op(smartlist_len(desc1->superencrypted_data.clients), ==,
+    tt_int_op(smartlist_len(desc1->superencrypted_data.clients), OP_EQ,
               smartlist_len(desc2->superencrypted_data.clients));
     for (int i=0;
          i < smartlist_len(desc1->superencrypted_data.clients);
@@ -251,15 +319,15 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   }
 
   /* Encrypted data section. */
-  tt_uint_op(desc1->encrypted_data.create2_ntor, ==,
+  tt_uint_op(desc1->encrypted_data.create2_ntor, OP_EQ,
              desc2->encrypted_data.create2_ntor);
 
   /* Authentication type. */
-  tt_int_op(!!desc1->encrypted_data.intro_auth_types, ==,
+  tt_int_op(!!desc1->encrypted_data.intro_auth_types, OP_EQ,
             !!desc2->encrypted_data.intro_auth_types);
   if (desc1->encrypted_data.intro_auth_types &&
       desc2->encrypted_data.intro_auth_types) {
-    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), ==,
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_auth_types), OP_EQ,
               smartlist_len(desc2->encrypted_data.intro_auth_types));
     for (int i = 0;
          i < smartlist_len(desc1->encrypted_data.intro_auth_types);
@@ -273,7 +341,7 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
   {
     tt_assert(desc1->encrypted_data.intro_points);
     tt_assert(desc2->encrypted_data.intro_points);
-    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), ==,
+    tt_int_op(smartlist_len(desc1->encrypted_data.intro_points), OP_EQ,
               smartlist_len(desc2->encrypted_data.intro_points));
     for (int i=0; i < smartlist_len(desc1->encrypted_data.intro_points); i++) {
       hs_desc_intro_point_t *ip1 = smartlist_get(desc1->encrypted_data
@@ -288,38 +356,76 @@ hs_helper_desc_equal(const hs_descriptor_t *desc1,
         tt_mem_op(&ip1->enc_key, OP_EQ, &ip2->enc_key, CURVE25519_PUBKEY_LEN);
       }
 
-      tt_int_op(smartlist_len(ip1->link_specifiers), ==,
+      tt_int_op(smartlist_len(ip1->link_specifiers), OP_EQ,
                 smartlist_len(ip2->link_specifiers));
       for (int j = 0; j < smartlist_len(ip1->link_specifiers); j++) {
-        hs_desc_link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
-                                 *ls2 = smartlist_get(ip2->link_specifiers, j);
-        tt_int_op(ls1->type, ==, ls2->type);
-        switch (ls1->type) {
+        link_specifier_t *ls1 = smartlist_get(ip1->link_specifiers, j),
+                         *ls2 = smartlist_get(ip2->link_specifiers, j);
+        tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ,
+                  link_specifier_get_ls_type(ls2));
+        switch (link_specifier_get_ls_type(ls1)) {
           case LS_IPV4:
+            {
+              uint32_t addr1 = link_specifier_get_un_ipv4_addr(ls1);
+              uint32_t addr2 = link_specifier_get_un_ipv4_addr(ls2);
+              tt_int_op(addr1, OP_EQ, addr2);
+              uint16_t port1 = link_specifier_get_un_ipv4_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv4_port(ls2);
+              tt_int_op(port1, OP_EQ, port2);
+            }
+            break;
           case LS_IPV6:
             {
-              addr1 = tor_addr_to_str_dup(&ls1->u.ap.addr);
-              addr2 = tor_addr_to_str_dup(&ls2->u.ap.addr);
-              tt_str_op(addr1, OP_EQ, addr2);
-              tor_free(addr1);
-              tor_free(addr2);
-              tt_int_op(ls1->u.ap.port, ==, ls2->u.ap.port);
+              const uint8_t *addr1 =
+                link_specifier_getconstarray_un_ipv6_addr(ls1);
+              const uint8_t *addr2 =
+                link_specifier_getconstarray_un_ipv6_addr(ls2);
+              tt_int_op(link_specifier_getlen_un_ipv6_addr(ls1), OP_EQ,
+                        link_specifier_getlen_un_ipv6_addr(ls2));
+              tt_mem_op(addr1, OP_EQ, addr2,
+                        link_specifier_getlen_un_ipv6_addr(ls1));
+              uint16_t port1 = link_specifier_get_un_ipv6_port(ls1);
+              uint16_t port2 = link_specifier_get_un_ipv6_port(ls2);
+              tt_int_op(port1, OP_EQ, port2);
             }
             break;
           case LS_LEGACY_ID:
-            tt_mem_op(ls1->u.legacy_id, OP_EQ, ls2->u.legacy_id,
-                      sizeof(ls1->u.legacy_id));
+            {
+              const uint8_t *id1 =
+                link_specifier_getconstarray_un_legacy_id(ls1);
+              const uint8_t *id2 =
+                link_specifier_getconstarray_un_legacy_id(ls2);
+              tt_int_op(link_specifier_getlen_un_legacy_id(ls1), OP_EQ,
+                        link_specifier_getlen_un_legacy_id(ls2));
+              tt_mem_op(id1, OP_EQ, id2,
+                        link_specifier_getlen_un_legacy_id(ls1));
+            }
             break;
           default:
             /* Unknown type, caught it and print its value. */
-            tt_int_op(ls1->type, OP_EQ, -1);
+            tt_int_op(link_specifier_get_ls_type(ls1), OP_EQ, -1);
         }
       }
     }
   }
 
  done:
-  tor_free(addr1);
-  tor_free(addr2);
+  ;
 }
 
+void
+hs_helper_add_client_auth(const ed25519_public_key_t *service_pk,
+                          const curve25519_secret_key_t *client_sk)
+{
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  if (client_auths == NULL) {
+    client_auths = digest256map_new();
+    set_hs_client_auths_map(client_auths);
+  }
+
+  hs_client_service_authorization_t *auth =
+    tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  memcpy(&auth->enc_seckey, client_sk, sizeof(curve25519_secret_key_t));
+  hs_build_address(service_pk, HS_VERSION_THREE, auth->onion_address);
+  digest256map_set(client_auths, service_pk->pubkey, auth);
+}
diff --git a/src/test/hs_test_helpers.h b/src/test/hs_test_helpers.h
index 9662a83ba8..23d11f2a4a 100644
--- a/src/test/hs_test_helpers.h
+++ b/src/test/hs_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_HS_TEST_HELPERS_H
@@ -8,18 +8,26 @@
 #include "feature/hs/hs_descriptor.h"
 
 /* Set of functions to help build and test descriptors. */
-hs_desc_intro_point_t *hs_helper_build_intro_point(
-                          const ed25519_keypair_t *signing_kp, time_t now,
-                          const char *addr, int legacy);
+hs_desc_intro_point_t *
+hs_helper_build_intro_point(const ed25519_keypair_t *signing_kp, time_t now,
+                            const char *addr, int legacy,
+                            const ed25519_keypair_t *intro_auth_kp,
+                            const curve25519_keypair_t *intro_enc_kp);
 hs_descriptor_t *hs_helper_build_hs_desc_no_ip(
                                  const ed25519_keypair_t *signing_kp);
 hs_descriptor_t *hs_helper_build_hs_desc_with_ip(
                                  const ed25519_keypair_t *signing_kp);
+hs_descriptor_t *hs_helper_build_hs_desc_with_client_auth(
+                                 const uint8_t *descriptor_cookie,
+                                 const curve25519_public_key_t *client_pk,
+                                 const ed25519_keypair_t *signing_kp);
 void hs_helper_desc_equal(const hs_descriptor_t *desc1,
                           const hs_descriptor_t *desc2);
-void
-hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
-                                            uint8_t *subcred_out);
+struct hs_subcredential_t;
+void hs_helper_get_subcred_from_identity_keypair(ed25519_keypair_t *signing_kp,
+                                      struct hs_subcredential_t *subcred_out);
 
-#endif /* !defined(TOR_HS_TEST_HELPERS_H) */
+void hs_helper_add_client_auth(const ed25519_public_key_t *service_pk,
+                               const curve25519_secret_key_t *client_sk);
 
+#endif /* !defined(TOR_HS_TEST_HELPERS_H) */
diff --git a/src/test/include.am b/src/test/include.am
index 75861fb9ef..7814dbca89 100644
--- a/src/test/include.am
+++ b/src/test/include.am
@@ -24,6 +24,8 @@ TESTSCRIPTS = \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
 	src/test/test_switch_id.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh \
         src/test/unittest_part1.sh \
         src/test/unittest_part2.sh \
         src/test/unittest_part3.sh \
@@ -39,9 +41,26 @@ TESTSCRIPTS += \
 endif
 
 if USEPYTHON
-TESTSCRIPTS += src/test/test_ntor.sh src/test/test_hs_ntor.sh src/test/test_bt.sh
+TESTSCRIPTS += \
+	src/test/test_ntor.sh \
+	src/test/test_hs_ntor.sh \
+	src/test/test_bt.sh \
+	scripts/maint/practracker/test_practracker.sh \
+	scripts/maint/run_check_subsystem_order.sh
+
+if COVERAGE_ENABLED
+# ...
+else
+# Only do this when coverage is not on, since it invokes lots of code
+# in a kind of unpredictable way.
 TESTSCRIPTS += src/test/test_rebind.sh
 endif
+endif
+
+if USE_PERL
+TESTSCRIPTS += \
+        scripts/maint/checkSpaceTest.sh
+endif
 
 TESTS += src/test/test-slow src/test/test-memwipe \
 	src/test/test_workqueue \
@@ -51,15 +70,25 @@ TESTS += src/test/test-slow src/test/test-memwipe \
 	$(TESTSCRIPTS)
 
 # These flavors are run using automake's test-driver and test-network.sh
-TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v2-min hs-v3-min \
-	single-onion-v23
+
+# run a quick test or two
+# this test only uses IPv4
+TEST_CHUTNEY_FLAVOR_QUICK = bridges+hs-v23
 # only run if we can ping6 ::1 (localhost)
-# IPv6-only v3 single onion services don't work yet, so we don't test the
-# single-onion-v23-ipv6-md flavor
+TEST_CHUTNEY_FLAVOR_QUICK_IPV6 = single-onion-v23-ipv6-md
+
+# run a basic set of tests, which only use IPv4
+TEST_CHUTNEY_FLAVORS = basic-min bridges-min hs-v23-min single-onion-v23
+
+# only run if we can ping ::1 (localhost)
 TEST_CHUTNEY_FLAVORS_IPV6 = bridges+ipv6-min ipv6-exit-min hs-v23-ipv6-md \
-	single-onion-ipv6-md
+	single-onion-v23-ipv6-md
+
 # only run if we can find a stable (or simply another) version of tor
-TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v2
+TEST_CHUTNEY_FLAVORS_MIXED = mixed+hs-v23
+
+# only run if IPv6 and mixed networks are run
+TEST_CHUTNEY_FLAVORS_IPV6_MIXED = mixed+hs-v23-ipv6
 
 ### This is a lovely feature, but it requires automake >= 1.12, and Tor
 ###  doesn't require that yet.
@@ -73,10 +102,11 @@ noinst_PROGRAMS+= \
 	src/test/test \
 	src/test/test-slow \
 	src/test/test-memwipe \
-	src/test/test-child \
+	src/test/test-process \
 	src/test/test_workqueue \
 	src/test/test-switch-id \
-	src/test/test-timers
+	src/test/test-timers \
+	src/test/test-rng
 endif
 
 src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
@@ -92,31 +122,42 @@ src_test_AM_CPPFLAGS = -DSHARE_DATADIR="\"$(datadir)\"" \
 src_test_test_SOURCES =
 
 if UNITTESTS_ENABLED
+
+# ADD_C_FILE: INSERT SOURCES HERE.
 src_test_test_SOURCES += \
+	src/test/fakecircs.c \
 	src/test/log_test_helpers.c \
 	src/test/hs_test_helpers.c \
+	src/test/opts_test_helpers.c \
 	src/test/rend_test_helpers.c \
+	src/test/resolve_test_helpers.c \
+	src/test/rng_test_helpers.c \
 	src/test/test.c \
 	src/test/test_accounting.c \
 	src/test/test_addr.c \
 	src/test/test_address.c \
 	src/test/test_address_set.c \
 	src/test/test_bridges.c \
+	src/test/test_btrack.c \
 	src/test/test_buffers.c \
 	src/test/test_bwmgt.c \
 	src/test/test_cell_formats.c \
 	src/test/test_cell_queue.c \
 	src/test/test_channel.c \
 	src/test/test_channelpadding.c \
+	src/test/test_circuitpadding.c \
 	src/test/test_channeltls.c \
 	src/test/test_checkdir.c \
 	src/test/test_circuitlist.c \
 	src/test/test_circuitmux.c \
+	src/test/test_circuitmux_ewma.c \
 	src/test/test_circuitbuild.c \
 	src/test/test_circuituse.c \
 	src/test/test_circuitstats.c \
 	src/test/test_compat_libevent.c \
 	src/test/test_config.c \
+	src/test/test_confmgr.c \
+	src/test/test_confparse.c \
 	src/test/test_connection.c \
 	src/test/test_conscache.c \
 	src/test/test_consdiff.c \
@@ -126,10 +167,12 @@ src_test_test_SOURCES += \
 	src/test/test_controller_events.c \
 	src/test/test_crypto.c \
 	src/test/test_crypto_ope.c \
+	src/test/test_crypto_rng.c \
 	src/test/test_data.c \
 	src/test/test_dir.c \
 	src/test/test_dir_common.c \
 	src/test/test_dir_handle_get.c \
+	src/test/test_dispatch.c \
 	src/test/test_dos.c \
 	src/test/test_entryconn.c \
 	src/test/test_entrynodes.c \
@@ -145,27 +188,38 @@ src_test_test_SOURCES += \
 	src/test/test_hs_client.c  \
 	src/test/test_hs_intropoint.c \
 	src/test/test_hs_control.c \
+	src/test/test_hs_ob.c \
 	src/test/test_handles.c \
 	src/test/test_hs_cache.c \
 	src/test/test_hs_descriptor.c \
+	src/test/test_hs_dos.c \
 	src/test/test_introduce.c \
 	src/test/test_keypin.c \
 	src/test/test_link_handshake.c \
 	src/test/test_logging.c \
 	src/test/test_mainloop.c \
 	src/test/test_microdesc.c \
+	src/test/test_namemap.c \
+	src/test/test_netinfo.c \
 	src/test/test_nodelist.c \
 	src/test/test_oom.c \
 	src/test/test_oos.c \
 	src/test/test_options.c \
+	src/test/test_options_act.c \
 	src/test/test_pem.c \
 	src/test/test_periodic_event.c \
 	src/test/test_policy.c \
+	src/test/test_process.c \
+	src/test/test_process_descs.c \
+	src/test/test_prob_distr.c \
 	src/test/test_procmon.c \
+	src/test/test_proto_haproxy.c \
 	src/test/test_proto_http.c \
 	src/test/test_proto_misc.c \
 	src/test/test_protover.c \
 	src/test/test_pt.c \
+	src/test/test_pubsub_build.c \
+	src/test/test_pubsub_msg.c \
 	src/test/test_relay.c \
 	src/test/test_relaycell.c \
 	src/test/test_relaycrypt.c \
@@ -176,19 +230,24 @@ src_test_test_SOURCES += \
 	src/test/test_routerlist.c \
 	src/test/test_routerset.c \
 	src/test/test_scheduler.c \
+	src/test/test_sendme.c \
 	src/test/test_shared_random.c \
 	src/test/test_socks.c \
+	src/test/test_stats.c \
 	src/test/test_status.c \
 	src/test/test_storagedir.c \
 	src/test/test_threads.c \
+	src/test/test_token_bucket.c \
 	src/test/test_tortls.c \
 	src/test/test_util.c \
 	src/test/test_util_format.c \
 	src/test/test_util_process.c \
+	src/test/test_voting_flags.c \
 	src/test/test_voting_schedule.c \
 	src/test/test_x509.c \
 	src/test/test_helpers.c \
 	src/test/test_dns.c \
+	src/test/test_parsecommon.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -206,9 +265,13 @@ endif
 src_test_test_slow_SOURCES =
 if UNITTESTS_ENABLED
 src_test_test_slow_SOURCES += \
+	src/test/rng_test_helpers.c \
 	src/test/test_slow.c \
 	src/test/test_crypto_slow.c \
-	src/test/test_util_slow.c \
+	src/test/test_process_slow.c \
+	src/test/test_prob_distr.c \
+	src/test/ptr_helpers.c \
+	src/test/test_ptr_slow.c \
 	src/test/testing_common.c \
 	src/test/testing_rsakeys.c \
 	src/ext/tinytest.c
@@ -259,6 +322,12 @@ src_test_test_slow_CFLAGS = $(src_test_test_CFLAGS)
 src_test_test_slow_LDADD = $(src_test_test_LDADD)
 src_test_test_slow_LDFLAGS = $(src_test_test_LDFLAGS)
 
+src_test_test_rng_CPPFLAGS = $(src_test_test_CPPFLAGS)
+src_test_test_rng_CFLAGS = $(src_test_test_CFLAGS)
+src_test_test_rng_SOURCES = src/test/test_rng.c
+src_test_test_rng_LDFLAGS =  $(src_test_test_LDFLAGS)
+src_test_test_rng_LDADD = $(src_test_test_LDADD)
+
 src_test_test_memwipe_CPPFLAGS = $(src_test_test_CPPFLAGS)
 # Don't use bugtrap cflags here: memwipe tests require memory violations.
 src_test_test_memwipe_CFLAGS = $(TEST_CFLAGS)
@@ -300,12 +369,18 @@ src_test_test_timers_LDADD = \
 	@TOR_LZMA_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
 
+# ADD_C_FILE: INSERT HEADERS HERE.
 noinst_HEADERS+= \
 	src/test/fakechans.h \
+	src/test/fakecircs.h \
 	src/test/hs_test_helpers.h \
 	src/test/log_test_helpers.h \
+	src/test/opts_test_helpers.h \
 	src/test/rend_test_helpers.h \
+	src/test/resolve_test_helpers.h \
+	src/test/rng_test_helpers.h \
 	src/test/test.h \
+	src/test/ptr_helpers.h \
 	src/test/test_helpers.h \
 	src/test/test_dir_common.h \
 	src/test/test_connection.h \
@@ -364,6 +439,7 @@ EXTRA_DIST += \
 	src/test/test_rebind.sh \
 	src/test/test_rebind.py \
 	src/test/zero_length_keys.sh \
+	scripts/maint/run_check_subsystem_order.sh \
 	src/test/rust_supp.txt \
 	src/test/test_keygen.sh \
 	src/test/test_key_expiration.sh \
@@ -378,6 +454,8 @@ EXTRA_DIST += \
 	src/test/test_workqueue_pipe.sh \
 	src/test/test_workqueue_pipe2.sh \
 	src/test/test_workqueue_socketpair.sh \
+	src/test/test_cmdline.sh \
+	src/test/test_parseconf.sh \
         src/test/unittest_part1.sh \
         src/test/unittest_part2.sh \
         src/test/unittest_part3.sh \
diff --git a/src/test/log_test_helpers.c b/src/test/log_test_helpers.c
index 03c52dd6bd..5e60d6b282 100644
--- a/src/test/log_test_helpers.c
+++ b/src/test/log_test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 #define LOG_PRIVATE
 #include "lib/log/log.h"
diff --git a/src/test/log_test_helpers.h b/src/test/log_test_helpers.h
index 5d1c3c1914..c2d71c6bcd 100644
--- a/src/test/log_test_helpers.h
+++ b/src/test/log_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -78,7 +78,7 @@ void mock_dump_saved_logs(void);
                          mock_saved_log_n_entries() == 1,               \
                          ("expected log to contain exactly 1 message \"%s\"", \
                           str));                                        \
-  } while (0);
+  } while (0)
 
 #define expect_single_log_msg_containing(str) \
   do {                                                                  \
@@ -86,30 +86,30 @@ void mock_dump_saved_logs(void);
                          mock_saved_log_n_entries() == 1 ,              \
                     ("expected log to contain 1 message, containing \"%s\"",\
                      str));                                             \
-  } while (0);
+  } while (0)
 
 #define expect_no_log_msg(str) \
   assert_log_predicate(!mock_saved_log_has_message(str), \
-                       ("expected log to not contain \"%s\"",str));
+                       ("expected log to not contain \"%s\"",str))
 
 #define expect_no_log_msg_containing(str) \
   assert_log_predicate(!mock_saved_log_has_message_containing(str), \
-                       ("expected log to not contain \"%s\"", str));
+                       ("expected log to not contain \"%s\"", str))
 
 #define expect_log_severity(severity) \
   assert_log_predicate(mock_saved_log_has_severity(severity), \
-                       ("expected log to contain severity " # severity));
+                       ("expected log to contain severity " # severity))
 
 #define expect_no_log_severity(severity) \
   assert_log_predicate(!mock_saved_log_has_severity(severity), \
-                       ("expected log to not contain severity " # severity));
+                       ("expected log to not contain severity " # severity))
 
 #define expect_log_entry() \
   assert_log_predicate(mock_saved_log_has_entry(), \
-                       ("expected log to contain entries"));
+                       ("expected log to contain entries"))
 
 #define expect_no_log_entry() \
   assert_log_predicate(!mock_saved_log_has_entry(), \
-                       ("expected log to not contain entries"));
+                       ("expected log to not contain entries"))
 
 #endif /* !defined(TOR_LOG_TEST_HELPERS_H) */
diff --git a/src/test/ntor_ref.py b/src/test/ntor_ref.py
index 204f05e2ad..e3307430e1 100755
--- a/src/test/ntor_ref.py
+++ b/src/test/ntor_ref.py
@@ -27,6 +27,11 @@ commands:
 
 """
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import binascii
 try:
     import curve25519
@@ -99,7 +104,7 @@ else:
    def int2byte(i):
       return bytes([i])
 
-def  kdf_rfc5869(key, salt, info, n):
+def kdf_rfc5869(key, salt, info, n):
 
     prk = HMAC(key=salt, msg=key)
 
diff --git a/src/test/ope_ref.py b/src/test/ope_ref.py
index f9bd97c546..61a86b57bb 100644
--- a/src/test/ope_ref.py
+++ b/src/test/ope_ref.py
@@ -1,9 +1,14 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 # Copyright 2018-2019, The Tor Project, Inc. See LICENSE for licensing info.
 
 # Reference implementation for our rudimentary OPE code, used to
 # generate test vectors. See crypto_ope.c for more details.
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
 from cryptography.hazmat.backends import default_backend
diff --git a/src/test/opts_test_helpers.c b/src/test/opts_test_helpers.c
new file mode 100644
index 0000000000..619ca40733
--- /dev/null
+++ b/src/test/opts_test_helpers.c
@@ -0,0 +1,38 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file opts_testing_helpers.c
+ * @brief Helper functions to access module-specific config options.
+ **/
+
+#include "orconfig.h"
+#include "test/opts_test_helpers.h"
+
+#define CONFIG_PRIVATE
+#include "core/or/or.h"
+#include "lib/confmgt/confmgt.h"
+#include "app/main/subsysmgr.h"
+#include "app/config/config.h"
+
+#include "lib/crypt_ops/crypto_sys.h"
+#include "feature/dirauth/dirauth_sys.h"
+
+struct dirauth_options_t *
+get_dirauth_options(struct or_options_t *opt)
+{
+  int idx = subsystems_get_options_idx(&sys_dirauth);
+  tor_assert(idx >= 0);
+  return config_mgr_get_obj_mutable(get_options_mgr(), opt, idx);
+}
+
+struct crypto_options_t *
+get_crypto_options(struct or_options_t *opt)
+{
+  int idx = subsystems_get_options_idx(&sys_crypto);
+  tor_assert(idx >= 0);
+  return config_mgr_get_obj_mutable(get_options_mgr(), opt, idx);
+}
diff --git a/src/test/opts_test_helpers.h b/src/test/opts_test_helpers.h
new file mode 100644
index 0000000000..f925194e63
--- /dev/null
+++ b/src/test/opts_test_helpers.h
@@ -0,0 +1,22 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file opts_testing_helpers.h
+ * @brief Header for test/opts_test_helpers.c
+ **/
+
+#ifndef TOR_TEST_OPTS_TESTING_HELPERS_H
+#define TOR_TEST_OPTS_TESTING_HELPERS_H
+
+struct crypto_options_t;
+struct dirauth_options_t;
+struct or_options_t;
+
+struct crypto_options_t *get_crypto_options(struct or_options_t *opt);
+struct dirauth_options_t *get_dirauth_options(struct or_options_t *opt);
+
+#endif /* !defined(TOR_TEST_OPTS_TESTING_HELPERS_H) */
diff --git a/src/test/prob_distr_mpfr_ref.c b/src/test/prob_distr_mpfr_ref.c
new file mode 100644
index 0000000000..de4179c4e0
--- /dev/null
+++ b/src/test/prob_distr_mpfr_ref.c
@@ -0,0 +1,64 @@
+/* Copyright 2012-2020, The Tor Project, Inc
+ * See LICENSE for licensing information */
+
+/** prob_distr_mpfr_ref.c
+ *
+ * Example reference file for GNU MPFR vectors tested in test_prob_distr.c .
+ * Code by Riastradh.
+ */
+
+#include <complex.h>
+#include <float.h>
+#include <math.h>
+#include <stdio.h>
+
+/* Must come after <stdio.h> so we get mpfr_printf.  */
+#include <mpfr.h>
+
+/*  gcc -o mpfr prob_distr_mpfr_ref.c -lmpfr -lm */
+
+/* Computes logit(p) for p = .49999 */
+int
+main(void)
+{
+  mpfr_t p, q, r;
+  mpfr_init(p);
+  mpfr_set_prec(p, 200);
+  mpfr_init(q);
+  mpfr_set_prec(q, 200);
+  mpfr_init(r);
+  mpfr_set_prec(r, 200);
+  mpfr_set_d(p, .49999, MPFR_RNDN);
+  mpfr_set_d(q, 1, MPFR_RNDN);
+  /* r := q - p = 1 - p */
+  mpfr_sub(r, q, p, MPFR_RNDN);
+  /* q := p/r = p/(1 - p) */
+  mpfr_div(q, p, r, MPFR_RNDN);
+  /* r := log(q) = log(p/(1 - p)) */
+  mpfr_log(r, q, MPFR_RNDN);
+  mpfr_printf("mpfr 200-bit\t%.128Rg\n", r);
+
+  /*
+   * Print a double approximation to logit three different ways.  All
+   * three agree bit for bit on the libms I tried, with the nextafter
+   * adjustment (which is well within the 10 eps relative error bound
+   * advertised).  Apparently I must have used the Goldberg expression
+   * for what I wrote down in the test case.
+   */
+  printf("mpfr 53-bit\t%.17g\n", nextafter(mpfr_get_d(r, MPFR_RNDN), 0), 0);
+  volatile double p0 = .49999;
+  printf("log1p\t\t%.17g\n", nextafter(-log1p((1 - 2*p0)/p0), 0));
+  volatile double x = (1 - 2*p0)/p0;
+  volatile double xp1 = x + 1;
+  printf("Goldberg\t%.17g\n", -x*log(xp1)/(xp1 - 1));
+
+  /*
+   * Print a bad approximation, using the naive expression, to see a
+   * lot of wrong digits, far beyond the 10 eps relative error attained
+   * by -log1p((1 - 2*p)/p).
+   */
+  printf("naive\t\t%.17g\n", log(p0/(1 - p0)));
+
+  fflush(stdout);
+  return ferror(stdout);
+}
diff --git a/src/test/ptr_helpers.c b/src/test/ptr_helpers.c
new file mode 100644
index 0000000000..0e0995df7c
--- /dev/null
+++ b/src/test/ptr_helpers.c
@@ -0,0 +1,50 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/ptr_helpers.h"
+
+/**
+ * Cast <b> (inptr_t value) to a void pointer.
+ */
+void *
+cast_intptr_to_voidstar(intptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to inptr_t value.
+ */
+intptr_t
+cast_voidstar_to_intptr(void *x)
+{
+  intptr_t r = (intptr_t)x;
+
+  return r;
+}
+
+/**
+ * Cast x (uinptr_t value) to void pointer.
+ */
+void *
+cast_uintptr_to_voidstar(uintptr_t x)
+{
+  void *r = (void *)x;
+
+  return r;
+}
+
+/**
+ * Cast x (void pointer) to uinptr_t value.
+ */
+uintptr_t
+cast_voidstar_to_uintptr(void *x)
+{
+  uintptr_t r = (uintptr_t)x;
+
+  return r;
+}
diff --git a/src/test/ptr_helpers.h b/src/test/ptr_helpers.h
new file mode 100644
index 0000000000..0999fdf5d2
--- /dev/null
+++ b/src/test/ptr_helpers.h
@@ -0,0 +1,23 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_PTR_HELPERS_H
+#define TOR_PTR_HELPERS_H
+
+#include <stdint.h>
+
+void *
+cast_intptr_to_voidstar(intptr_t x);
+
+intptr_t
+cast_voidstar_to_intptr(void *x);
+
+void *
+cast_uintptr_to_voidstar(uintptr_t x);
+
+uintptr_t
+cast_voidstar_to_uintptr(void *x);
+
+#endif /* !defined(TOR_PTR_HELPERS_H) */
diff --git a/src/test/rend_test_helpers.c b/src/test/rend_test_helpers.c
index f12d193cc5..61bacb4d2e 100644
--- a/src/test/rend_test_helpers.c
+++ b/src/test/rend_test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/rend_test_helpers.h b/src/test/rend_test_helpers.h
index c10da52cd7..b1078ce866 100644
--- a/src/test/rend_test_helpers.h
+++ b/src/test/rend_test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/resolve_test_helpers.c b/src/test/resolve_test_helpers.c
new file mode 100644
index 0000000000..ed5853c359
--- /dev/null
+++ b/src/test/resolve_test_helpers.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.c
+ * @brief Helper functions for mocking libc's blocking hostname lookup
+ *   facilities.
+ **/
+
+#define RESOLVE_PRIVATE
+#include "orconfig.h"
+#include "test/resolve_test_helpers.h"
+#include "lib/net/address.h"
+#include "lib/net/resolve.h"
+#include "test/test.h"
+
+#include <stdio.h>
+#include <string.h>
+
+/**
+ * Mock replacement for our getaddrinfo/gethostbyname wrapper.
+ **/
+static int
+replacement_host_lookup(const char *name, uint16_t family, tor_addr_t *addr)
+{
+  static const struct lookup_table_ent {
+    const char *name;
+    const char *ipv4;
+    const char *ipv6;
+  } entries[] = {
+    { "localhost", "127.0.0.1", "::1" },
+    { "torproject.org", "198.51.100.6", "2001:DB8::700" },
+    { NULL, NULL, NULL },
+  };
+
+  int r = -1;
+
+  for (unsigned i = 0; entries[i].name != NULL; ++i) {
+    if (!strcasecmp(name, entries[i].name)) {
+      if (family == AF_INET6) {
+        int s = tor_addr_parse(addr, entries[i].ipv6);
+        tt_int_op(s, OP_EQ, AF_INET6);
+      } else {
+        int s = tor_addr_parse(addr, entries[i].ipv4);
+        tt_int_op(s, OP_EQ, AF_INET);
+      }
+      r = 0;
+      break;
+    }
+  }
+
+  log_debug(LD_GENERAL, "resolve(%s,%d) => %s",
+            name, family, r == 0 ? fmt_addr(addr) : "-1");
+
+  return r;
+ done:
+  return -1;
+}
+
+/**
+ * Set up a mock replacement for our wrapper on libc's resolver code.
+ *
+ * According to our replacement, only "localhost" and "torproject.org"
+ * are real addresses; everything else doesn't exist.
+ *
+ * Use this function to avoid using the DNS resolver during unit tests;
+ * call unmock_hostname_resolver() when you're done.
+ **/
+void
+mock_hostname_resolver(void)
+{
+  MOCK(tor_addr_lookup_host_impl, replacement_host_lookup);
+}
+
+/**
+ * Unmock our wrappers for libc's blocking hostname resolver code.
+ **/
+void
+unmock_hostname_resolver(void)
+{
+  UNMOCK(tor_addr_lookup_host_impl);
+}
diff --git a/src/test/resolve_test_helpers.h b/src/test/resolve_test_helpers.h
new file mode 100644
index 0000000000..ca642d6c63
--- /dev/null
+++ b/src/test/resolve_test_helpers.h
@@ -0,0 +1,18 @@
+/* Copyright (c) 2001 Matej Pfajfar.
+ * Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * @file resolve_test_helpers.h
+ * @brief Header for test/resolve_test_helpers.c
+ **/
+
+#ifndef TOR_TEST_RESOLVE_TEST_HELPERS_H
+#define TOR_TEST_RESOLVE_TEST_HELPERS_H
+
+void mock_hostname_resolver(void);
+void unmock_hostname_resolver(void);
+
+#endif /* !defined(TOR_TEST_RESOLVE_TEST_HELPERS_H) */
diff --git a/src/test/rng_test_helpers.c b/src/test/rng_test_helpers.c
new file mode 100644
index 0000000000..b7d7cb0dfa
--- /dev/null
+++ b/src/test/rng_test_helpers.c
@@ -0,0 +1,259 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file rng_test_helpers.c
+ * \brief Helpers for overriding PRNGs during unit tests.
+ *
+ * We define two PRNG overrides: a "reproducible PRNG" where the seed is
+ * chosen randomly but the stream can be replayed later on in case a bug is
+ * found, and a "deterministic PRNG" where the seed is fixed in the unit
+ * tests.
+ *
+ * Obviously, this code is testing-only.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+
+#include "lib/crypt_ops/crypto_rand.h"
+#include "ext/tinytest.h"
+
+#include "test/rng_test_helpers.h"
+
+#ifndef TOR_UNIT_TESTS
+#error "No. Never link this code into Tor proper."
+#endif
+
+/**
+ * True iff the RNG is currently replaced.  Prevents double-replacement.
+ **/
+static bool rng_is_replaced = false;
+
+/**
+ * Mutex to protect deterministic prng.
+ *
+ * Note that if you actually _use_ the prng from two threads at the same time,
+ * the results will probably be nondeterministic anyway.
+ */
+static tor_mutex_t *rng_mutex = NULL;
+
+/**
+ * Cached old value for the thread prng.
+ **/
+static crypto_fast_rng_t *stored_fast_rng = NULL;
+
+/** replacement for crypto_strongest_rand that delegates to crypto_rand. */
+static void
+mock_crypto_strongest_rand(uint8_t *out, size_t len)
+{
+  crypto_rand((char *)out, len);
+}
+
+/* This is the seed of the deterministic randomness. */
+static uint8_t rng_seed[16];
+static crypto_xof_t *rng_xof = NULL;
+
+/**
+ * Print the seed for our PRNG to stdout.  We use this when we're failed
+ * test that had a reproducible RNG set.
+ **/
+void
+testing_dump_reproducible_rng_seed(void)
+{
+  printf("\n"
+         "Seed: %s\n",
+         hex_str((const char*)rng_seed, sizeof(rng_seed)));
+}
+
+/** Produce deterministic randomness for the stochastic tests using the global
+ * rng_xof output.
+ *
+ * This function produces deterministic data over multiple calls iff it's
+ * called in the same call order with the same 'n' parameter.
+ * If not, outputs will deviate. */
+static void
+crypto_rand_deterministic(char *out, size_t n)
+{
+  tor_assert(rng_xof);
+  tor_mutex_acquire(rng_mutex);
+  crypto_xof_squeeze_bytes(rng_xof, (uint8_t*)out, n);
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Implementation helper: override our crypto_rand() PRNG with a given seed of
+ * length <b>seed_len</b>.  Overlong seeds are truncated; short ones are
+ * padded.
+ **/
+static void
+enable_deterministic_rng_impl(const uint8_t *seed, size_t seed_len)
+{
+  tor_assert(!rng_is_replaced);
+  tor_assert(crypto_rand == crypto_rand__real);
+
+  memset(rng_seed, 0, sizeof(rng_seed));
+  memcpy(rng_seed, seed, MIN(seed_len, sizeof(rng_seed)));
+
+  rng_mutex = tor_mutex_new();
+
+  crypto_xof_free(rng_xof);
+  rng_xof = crypto_xof_new();
+  crypto_xof_add_bytes(rng_xof, rng_seed, sizeof(rng_seed));
+  MOCK(crypto_rand, crypto_rand_deterministic);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+
+  uint8_t fast_rng_seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(fast_rng_seed, 0xff, sizeof(fast_rng_seed));
+  memcpy(fast_rng_seed, rng_seed, MIN(sizeof(rng_seed),
+                                      sizeof(fast_rng_seed)));
+  crypto_fast_rng_t *fast_rng = crypto_fast_rng_new_from_seed(fast_rng_seed);
+  crypto_fast_rng_disable_reseed(fast_rng);
+  stored_fast_rng = crypto_replace_thread_fast_rng(fast_rng);
+
+  rng_is_replaced = true;
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a randomly chosen seed.  In the event of an
+ * error, you can log the seed later on with
+ * testing_dump_reproducible_rng_seed.
+ **/
+void
+testing_enable_reproducible_rng(void)
+{
+  const char *provided_seed = getenv("TOR_TEST_RNG_SEED");
+  if (provided_seed) {
+    size_t hexlen = strlen(provided_seed);
+    size_t seedlen = hexlen / 2;
+    uint8_t *seed = tor_malloc(hexlen / 2);
+    if (base16_decode((char*)seed, seedlen, provided_seed, hexlen) < 0) {
+      puts("Cannot decode value in TOR_TEST_RNG_SEED");
+      exit(1);
+    }
+    enable_deterministic_rng_impl(seed, seedlen);
+    tor_free(seed);
+  } else {
+    uint8_t seed[16];
+    crypto_rand((char*)seed, sizeof(seed));
+    enable_deterministic_rng_impl(seed, sizeof(seed));
+  }
+}
+
+/**
+ * Replace our get_thread_fast_rng(), crypto_rand() and
+ * crypto_strongest_rand() prngs with a variant that generates all of its
+ * output deterministically from a fixed seed.  This variant is mainly useful
+ * for cases when we don't want coverage to change between runs.
+ *
+ * USAGE NOTE: Test correctness SHOULD NOT depend on the specific output of
+ * this "rng".  If you need a specific output, use
+ * testing_enable_prefilled_rng() instead.
+ **/
+void
+testing_enable_deterministic_rng(void)
+{
+  static const uint8_t quotation[] =
+    "What will it be? A tree? A weed? "
+    "Each one is started from a seed."; // -- Mary Ann Hoberman
+  enable_deterministic_rng_impl(quotation, sizeof(quotation));
+}
+
+static uint8_t *prefilled_rng_buffer = NULL;
+static size_t prefilled_rng_buflen;
+static size_t prefilled_rng_idx;
+
+/**
+ * crypto_rand() replacement that returns canned data.
+ **/
+static void
+crypto_rand_prefilled(char *out, size_t n)
+{
+  tor_mutex_acquire(rng_mutex);
+  while (n) {
+    size_t n_to_copy = MIN(prefilled_rng_buflen - prefilled_rng_idx, n);
+    memcpy(out, prefilled_rng_buffer + prefilled_rng_idx, n_to_copy);
+    out += n_to_copy;
+    n -= n_to_copy;
+    prefilled_rng_idx += n_to_copy;
+
+    if (prefilled_rng_idx == prefilled_rng_buflen) {
+      prefilled_rng_idx = 0;
+    }
+  }
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Replace our crypto_rand() and crypto_strongest_rand() prngs with a variant
+ * that yields output from a buffer.  If it reaches the end of the buffer, it
+ * starts over.
+ *
+ * Note: the get_thread_fast_rng() prng is not replaced by this; we'll need
+ * more code to support that.
+ **/
+void
+testing_enable_prefilled_rng(const void *buffer, size_t buflen)
+{
+  tor_assert(buflen > 0);
+  tor_assert(!rng_mutex);
+  rng_mutex = tor_mutex_new();
+
+  tor_mutex_acquire(rng_mutex);
+
+  prefilled_rng_buffer = tor_memdup(buffer, buflen);
+  prefilled_rng_buflen = buflen;
+  prefilled_rng_idx = 0;
+
+  tor_mutex_release(rng_mutex);
+
+  MOCK(crypto_rand, crypto_rand_prefilled);
+  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+}
+
+/**
+ * Reset the position in the prefilled RNG buffer to the start.
+ */
+void
+testing_prefilled_rng_reset(void)
+{
+  tor_mutex_acquire(rng_mutex);
+  prefilled_rng_idx = 0;
+  tor_mutex_release(rng_mutex);
+}
+
+/**
+ * Undo the overrides for our PRNG.  To be used at the end of testing.
+ *
+ * Note that this function should be safe to call even if the rng has not
+ * yet been replaced.
+ **/
+void
+testing_disable_rng_override(void)
+{
+  crypto_xof_free(rng_xof);
+  tor_free(prefilled_rng_buffer);
+  UNMOCK(crypto_rand);
+  UNMOCK(crypto_strongest_rand_);
+  tor_mutex_free(rng_mutex);
+
+  crypto_fast_rng_t *rng = crypto_replace_thread_fast_rng(stored_fast_rng);
+  crypto_fast_rng_free(rng);
+
+  rng_is_replaced = false;
+}
+
+/**
+ * As testing_disable_rng_override(), but dump the seed if the current
+ * test has failed.
+ */
+void
+testing_disable_reproducible_rng(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    testing_dump_reproducible_rng_seed();
+  }
+  testing_disable_rng_override();
+}
diff --git a/src/test/rng_test_helpers.h b/src/test/rng_test_helpers.h
new file mode 100644
index 0000000000..6fcdaa2653
--- /dev/null
+++ b/src/test/rng_test_helpers.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#ifndef TOR_RNG_TEST_HELPERS_H
+#define TOR_RNG_TEST_HELPERS_H
+
+#include "core/or/or.h"
+
+void testing_enable_deterministic_rng(void);
+void testing_enable_reproducible_rng(void);
+void testing_enable_prefilled_rng(const void *buffer, size_t buflen);
+
+void testing_prefilled_rng_reset(void);
+
+void testing_disable_rng_override(void);
+
+void testing_disable_reproducible_rng(void);
+#define testing_disable_deterministic_rng() \
+  testing_disable_rng_override()
+#define testing_disable_prefilled_rng() \
+  testing_disable_rng_override()
+
+void testing_dump_reproducible_rng_seed(void);
+
+#endif /* !defined(TOR_RNG_TEST_HELPERS_H) */
diff --git a/src/test/slow_ed25519.py b/src/test/slow_ed25519.py
index f44708b200..be4eeab857 100644
--- a/src/test/slow_ed25519.py
+++ b/src/test/slow_ed25519.py
@@ -1,5 +1,5 @@
 # This is the ed25519 implementation from
-#     http://ed25519.cr.yp.to/python/ed25519.py .
+#     https://ed25519.cr.yp.to/python/ed25519.py .
 # It is in the public domain.
 #
 # It isn't constant-time.  Don't use it except for testing.  Also, see
@@ -8,6 +8,11 @@
 #
 # Don't edit this file.  Mess with ed25519_ref.py
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import hashlib
 
 b = 256
diff --git a/src/test/slownacl_curve25519.py b/src/test/slownacl_curve25519.py
index 4dabab61b6..0cafe0e71f 100644
--- a/src/test/slownacl_curve25519.py
+++ b/src/test/slownacl_curve25519.py
@@ -6,10 +6,15 @@
 # Nick got the slownacl source from:
 #        https://github.com/mdempsky/dnscurve/tree/master/slownacl
 
-__all__ = ['smult_curve25519_base', 'smult_curve25519']
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 import sys
 
+__all__ = ['smult_curve25519_base', 'smult_curve25519']
+
 P = 2 ** 255 - 19
 A = 486662
 
diff --git a/src/test/sr_commit_calc_ref.py b/src/test/sr_commit_calc_ref.py
index 45e629cfb0..c4cb72d87f 100644
--- a/src/test/sr_commit_calc_ref.py
+++ b/src/test/sr_commit_calc_ref.py
@@ -12,6 +12,11 @@
 #      COMMIT = base64-encode( TIMESTAMP || H(REVEAL) )
 #
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/sr_srv_calc_ref.py b/src/test/sr_srv_calc_ref.py
index 492ca62b15..a3752b15cc 100644
--- a/src/test/sr_srv_calc_ref.py
+++ b/src/test/sr_srv_calc_ref.py
@@ -10,6 +10,11 @@
 #                     HASHED_REVEALS | previous_SRV)
 #
 
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
 import sys
 import hashlib
 import struct
diff --git a/src/test/test-child.c b/src/test/test-child.c
deleted file mode 100644
index 11a1695cad..0000000000
--- a/src/test/test-child.c
+++ /dev/null
@@ -1,61 +0,0 @@
-/* Copyright (c) 2011-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#include <stdio.h>
-#ifdef _WIN32
-#define WINDOWS_LEAN_AND_MEAN
-#include <windows.h>
-#else
-#include <unistd.h>
-#endif /* defined(_WIN32) */
-#include <string.h>
-
-#ifdef _WIN32
-#define SLEEP(sec) Sleep((sec)*1000)
-#else
-#define SLEEP(sec) sleep(sec)
-#endif
-
-/** Trivial test program which prints out its command line arguments so we can
- * check if tor_spawn_background() works */
-int
-main(int argc, char **argv)
-{
-  int i;
-  int delay = 1;
-  int fast = 0;
-
-  if (argc > 1)  {
-    if (!strcmp(argv[1], "--hang")) {
-      delay = 60;
-    } else if (!strcmp(argv[1], "--fast")) {
-      fast = 1;
-      delay = 0;
-    }
-  }
-
-  fprintf(stdout, "OUT\n");
-  fprintf(stderr, "ERR\n");
-  for (i = 1; i < argc; i++)
-    fprintf(stdout, "%s\n", argv[i]);
-  if (!fast)
-    fprintf(stdout, "SLEEPING\n");
-  /* We need to flush stdout so that test_util_spawn_background_partial_read()
-     succeed. Otherwise ReadFile() will get the entire output in one */
-  // XXX: Can we make stdio flush on newline?
-  fflush(stdout);
-  if (!fast)
-    SLEEP(1);
-  fprintf(stdout, "DONE\n");
-  fflush(stdout);
-  if (fast)
-    return 0;
-
-  while (--delay) {
-    SLEEP(1);
-  }
-
-  return 0;
-}
-
diff --git a/src/test/test-memwipe.c b/src/test/test-memwipe.c
index 43754ed1c2..4faf7bc5a1 100644
--- a/src/test/test-memwipe.c
+++ b/src/test/test-memwipe.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -49,7 +49,7 @@ const char *s = NULL;
  * us do bad things, such as access freed buffers, without crashing. */
 extern const char *malloc_options;
 const char *malloc_options = "sufjj";
-#endif
+#endif /* defined(OpenBSD) */
 
 static unsigned
 fill_a_buffer_memset(void)
diff --git a/src/test/test-network.sh b/src/test/test-network.sh
index b7a9f1b3c0..5ef995f1a4 100755
--- a/src/test/test-network.sh
+++ b/src/test/test-network.sh
@@ -5,7 +5,7 @@
 # If we already know CHUTNEY_PATH, don't bother with argument parsing
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     # we can't produce any output, because we might be --quiet
     # this preserves arguments with spaces correctly
     exec "$TEST_NETWORK" "$@"
@@ -16,34 +16,16 @@ fi
 # Do we output anything at all?
 ECHO="${ECHO:-echo}"
 # Output is prefixed with the name of the script
-myname=$(basename $0)
-
-# Save the arguments before we destroy them
-# This might not preserve arguments with spaces in them
-ORIGINAL_ARGS="$@"
+myname=$(basename "$0")
 
 # We need to find CHUTNEY_PATH, so that we can call the version of this script
 # in chutney/tools with the same arguments. We also need to respect --quiet.
-until [ -z "$1" ]
-do
-  case "$1" in
-    --chutney-path)
-      CHUTNEY_PATH="$2"
-      shift
-    ;;
-    --tor-path)
-      TOR_DIR="$2"
-      shift
-    ;;
-    --quiet)
-      ECHO=true
-    ;;
-    *)
-      # maybe chutney's test-network.sh can handle it
-    ;;
-  esac
-  shift
-done
+CHUTNEY_PATH=$(echo "$@" | awk -F '--chutney-path ' '{sub(" .*","",$2); print $2}')
+TOR_DIR=$(echo "$@" | awk -F '--tor-dir ' '{sub(" .*","",$2); print $2}')
+
+if echo "$@" | grep -e "--quiet" > /dev/null; then
+  ECHO=true
+fi
 
 # optional: $TOR_DIR is the tor build directory
 # it's used to find the location of tor binaries
@@ -52,12 +34,12 @@ done
 #  - if $PWD looks like a tor build directory, set it to $PWD, or
 #  - unset $TOR_DIR, and let chutney fall back to finding tor binaries in $PATH
 if [ ! -d "$TOR_DIR" ]; then
-    if [ -d "$BUILDDIR/src/core/or" -a -d "$BUILDDIR/src/tools" ]; then
+    if [ -d "$BUILDDIR/src/core/or" ] && [ -d "$BUILDDIR/src/tools" ]; then
         # Choose the build directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$BUILDDIR"
         TOR_DIR="$BUILDDIR"
-    elif [ -d "$PWD/src/core/or" -a -d "$PWD/src/tools" ]; then
+    elif [ -d "$PWD/src/core/or" ] && [ -d "$PWD/src/tools" ]; then
         # Guess the tor directory is the current directory
         # But only if it looks like one
         $ECHO "$myname: \$TOR_DIR not set, trying \$PWD"
@@ -73,12 +55,12 @@ fi
 #  - if $PWD looks like a chutney directory, set it to $PWD, or
 #  - set it based on $TOR_DIR, expecting chutney to be next to tor, or
 #  - fail and tell the user how to clone the chutney repository
-if [ ! -d "$CHUTNEY_PATH" -o ! -x "$CHUTNEY_PATH/chutney" ]; then
+if [ ! -d "$CHUTNEY_PATH" ] || [ ! -x "$CHUTNEY_PATH/chutney" ]; then
     if [ -x "$PWD/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$PWD"
         CHUTNEY_PATH="$PWD"
-    elif [ -d "$TOR_DIR" -a -d "$TOR_DIR/../chutney" -a \
-           -x "$TOR_DIR/../chutney/chutney" ]; then
+    elif [ -d "$TOR_DIR" ] && [ -d "$TOR_DIR/../chutney" ] && \
+          [ -x "$TOR_DIR/../chutney/chutney" ]; then
         $ECHO "$myname: \$CHUTNEY_PATH not valid, trying \$TOR_DIR/../chutney"
         CHUTNEY_PATH="$TOR_DIR/../chutney"
     else
@@ -94,12 +76,12 @@ fi
 
 TEST_NETWORK="$CHUTNEY_PATH/tools/test-network.sh"
 # Call the chutney version of this script, if it exists, and we can find it
-if [ -d "$CHUTNEY_PATH" -a -x "$TEST_NETWORK" ]; then
+if [ -d "$CHUTNEY_PATH" ] && [ -x "$TEST_NETWORK" ]; then
     $ECHO "$myname: Calling newer chutney script $TEST_NETWORK"
     # this may fail if some arguments have spaces in them
     # if so, set CHUTNEY_PATH before calling test-network.sh, and spaces
     # will be handled correctly
-    exec "$TEST_NETWORK" $ORIGINAL_ARGS
+    exec "$TEST_NETWORK" "$@"
 else
     $ECHO "$myname: Could not find tools/test-network.sh in CHUTNEY_PATH."
     $ECHO "$myname: Please update your chutney using 'git pull'."
diff --git a/src/test/test-process.c b/src/test/test-process.c
new file mode 100644
index 0000000000..f5a1f1a54e
--- /dev/null
+++ b/src/test/test-process.c
@@ -0,0 +1,85 @@
+/* Copyright (c) 2011-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include <stdio.h>
+#ifdef _WIN32
+#define WINDOWS_LEAN_AND_MEAN
+#include <windows.h>
+#else
+#include <unistd.h>
+#endif /* defined(_WIN32) */
+#include <string.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define SLEEP(sec) Sleep((sec)*1000)
+#else
+#define SLEEP(sec) sleep(sec)
+#endif
+
+/* Trivial test program to test process_t. */
+int
+main(int argc, char **argv)
+{
+  /* Does our process get the right arguments? */
+  for (int i = 0; i < argc; ++i) {
+    fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]);
+    fflush(stdout);
+  }
+
+  /* Make sure our process got our environment variable. */
+  fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n",
+          getenv("TOR_TEST_ENV"));
+  fflush(stdout);
+
+  /* Test line handling on stdout and stderr. */
+  fprintf(stdout, "Output on stdout\nThis is a new line\n");
+  fflush(stdout);
+
+  fprintf(stderr, "Output on stderr\nThis is a new line\n");
+  fflush(stderr);
+
+  fprintf(stdout, "Partial line on stdout ...");
+  fflush(stdout);
+
+  fprintf(stderr, "Partial line on stderr ...");
+  fflush(stderr);
+
+  SLEEP(2);
+
+  fprintf(stdout, "end of partial line on stdout\n");
+  fflush(stdout);
+  fprintf(stderr, "end of partial line on stderr\n");
+  fflush(stderr);
+
+  /* Echo input from stdin. */
+  char buffer[1024];
+
+  int count = 0;
+
+  while (fgets(buffer, sizeof(buffer), stdin)) {
+    /* Strip the newline. */
+    size_t size = strlen(buffer);
+
+    if (size >= 1 && buffer[size - 1] == '\n') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    if (size >= 1 && buffer[size - 1] == '\r') {
+      buffer[size - 1] = '\0';
+      --size;
+    }
+
+    fprintf(stdout, "Read line from stdin: '%s'\n", buffer);
+    fflush(stdout);
+
+    if (++count == 3)
+      break;
+  }
+
+  fprintf(stdout, "We are done for here, thank you!\n");
+
+  return 0;
+}
diff --git a/src/test/test-timers.c b/src/test/test-timers.c
index c80fb1e305..18e2191a09 100644
--- a/src/test/test-timers.c
+++ b/src/test/test-timers.c
@@ -1,4 +1,4 @@
-/* Copyright 2016-2019, The Tor Project, Inc. */
+/* Copyright 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -59,7 +59,7 @@ main(int argc, char **argv)
 {
   (void)argc;
   (void)argv;
-  tor_libevent_cfg cfg;
+  tor_libevent_cfg_t cfg;
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
   timers_initialize();
diff --git a/src/test/test.c b/src/test/test.c
index 58b468775c..4b6082ce4f 100644
--- a/src/test/test.c
+++ b/src/test/test.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "app/config/or_state_st.h"
+#include "test/rng_test_helpers.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -37,7 +38,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "lib/compress/compress.h"
@@ -54,7 +55,6 @@
 #include "core/crypto/onion_fast.h"
 #include "core/crypto/onion_tap.h"
 #include "core/or/policies.h"
-#include "feature/stats/rephist.h"
 #include "app/config/statefile.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
 
@@ -283,7 +283,7 @@ test_fast_handshake(void *arg)
   /* First, test an entire handshake. */
   memset(client_handshake, 0, sizeof(client_handshake));
   tt_int_op(0, OP_EQ, fast_onionskin_create(&state, client_handshake));
-  tt_assert(! tor_mem_is_zero((char*)client_handshake,
+  tt_assert(! fast_mem_is_zero((char*)client_handshake,
                               sizeof(client_handshake)));
 
   tt_int_op(0, OP_EQ,
@@ -354,18 +354,6 @@ test_onion_queues(void *arg)
   tor_free(onionskin);
 }
 
-static crypto_cipher_t *crypto_rand_aes_cipher = NULL;
-
-// Mock replacement for crypto_rand: Generates bytes from a provided AES_CTR
-// cipher in <b>crypto_rand_aes_cipher</b>.
-static void
-crypto_rand_deterministic_aes(char *out, size_t n)
-{
-  tor_assert(crypto_rand_aes_cipher);
-  memset(out, 0, n);
-  crypto_cipher_crypt_inplace(crypto_rand_aes_cipher, out, n);
-}
-
 static void
 test_circuit_timeout(void *arg)
 {
@@ -397,8 +385,7 @@ test_circuit_timeout(void *arg)
 
   // Use a deterministic RNG here, or else we'll get nondeterministic
   // coverage in some of the circuitstats functions.
-  MOCK(crypto_rand, crypto_rand_deterministic_aes);
-  crypto_rand_aes_cipher = crypto_cipher_new("xyzzyplughplover");
+  testing_enable_deterministic_rng();
 
   circuitbuild_running_unit_tests();
 #define timeout0 (build_time_t)(30*1000.0)
@@ -534,8 +521,8 @@ test_circuit_timeout(void *arg)
   circuit_build_times_free_timeouts(&final);
   or_state_free(state);
   teardown_periodic_events();
-  UNMOCK(crypto_rand);
-  crypto_cipher_free(crypto_rand_aes_cipher);
+
+  testing_disable_deterministic_rng();
 }
 
 /** Test encoding and parsing of rendezvous service descriptors. */
@@ -651,166 +638,6 @@ test_rend_fns(void *arg)
   tor_free(intro_points_encrypted);
 }
 
-/** Run unit tests for stats code. */
-static void
-test_stats(void *arg)
-{
-  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
-  char *s = NULL;
-  int i;
-
-  /* Start with testing exit port statistics; we shouldn't collect exit
-   * stats without initializing them. */
-  (void)arg;
-  rep_hist_note_exit_stream_opened(80);
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Initialize stats, note some streams and bytes, and generate history
-   * string. */
-  rep_hist_exit_stats_init(now);
-  rep_hist_note_exit_stream_opened(80);
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  rep_hist_note_exit_stream_opened(443);
-  rep_hist_note_exit_bytes(443, 100, 10000);
-  rep_hist_note_exit_bytes(443, 100, 10000);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "exit-kibibytes-written 80=1,443=1,other=0\n"
-             "exit-kibibytes-read 80=10,443=20,other=0\n"
-             "exit-streams-opened 80=4,443=4,other=0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Add a few bytes on 10 more ports and ensure that only the top 10
-   * ports are contained in the history string. */
-  for (i = 50; i < 60; i++) {
-    rep_hist_note_exit_bytes(i, i, i);
-    rep_hist_note_exit_stream_opened(i);
-  }
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
-             "59=1,80=1,443=1,other=1\n"
-             "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
-             "59=1,80=10,443=20,other=1\n"
-             "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4,"
-             "59=4,80=4,443=4,other=4\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add some bytes, and ensure we don't generate
-   * a history string. */
-  rep_hist_exit_stats_term();
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Re-start stats, add some bytes, reset stats, and see what history we
-   * get when observing no streams or bytes at all. */
-  rep_hist_exit_stats_init(now);
-  rep_hist_note_exit_stream_opened(80);
-  rep_hist_note_exit_bytes(80, 100, 10000);
-  rep_hist_reset_exit_stats(now);
-  s = rep_hist_format_exit_stats(now + 86400);
-  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "exit-kibibytes-written other=0\n"
-             "exit-kibibytes-read other=0\n"
-             "exit-streams-opened other=0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Continue with testing connection statistics; we shouldn't collect
-   * conn stats without initializing them. */
-  rep_hist_note_or_conn_bytes(1, 20, 400, now);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Initialize stats, note bytes, and generate history string. */
-  rep_hist_conn_stats_init(now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add some bytes, and ensure we don't generate
-   * a history string. */
-  rep_hist_conn_stats_term();
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Re-start stats, add some bytes, reset stats, and see what history we
-   * get when observing no bytes at all. */
-  rep_hist_conn_stats_init(now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
-  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
-  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
-  rep_hist_reset_conn_stats(now);
-  s = rep_hist_format_conn_stats(now + 86400);
-  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Continue with testing buffer statistics; we shouldn't collect buffer
-   * stats without initializing them. */
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Initialize stats, add statistics for a single circuit, and generate
-   * the history string. */
-  rep_hist_buffer_stats_init(now);
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "cell-processed-cells 20,0,0,0,0,0,0,0,0,0\n"
-             "cell-queued-cells 2.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
-                               "0.00,0.00\n"
-             "cell-time-in-queue 2,0,0,0,0,0,0,0,0,0\n"
-             "cell-circuits-per-decile 1\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Add nineteen more circuit statistics to the one that's already in the
-   * history to see that the math works correctly. */
-  for (i = 21; i < 30; i++)
-    rep_hist_add_buffer_stats(2.0, 2.0, i);
-  for (i = 20; i < 30; i++)
-    rep_hist_add_buffer_stats(3.5, 3.5, i);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "cell-processed-cells 29,28,27,26,25,24,23,22,21,20\n"
-             "cell-queued-cells 2.75,2.75,2.75,2.75,2.75,2.75,2.75,2.75,"
-                               "2.75,2.75\n"
-             "cell-time-in-queue 3,3,3,3,3,3,3,3,3,3\n"
-             "cell-circuits-per-decile 2\n",OP_EQ, s);
-  tor_free(s);
-
-  /* Stop collecting stats, add statistics for one circuit, and ensure we
-   * don't generate a history string. */
-  rep_hist_buffer_stats_term();
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_ptr_op(s, OP_EQ, NULL);
-
-  /* Re-start stats, add statistics for one circuit, reset stats, and make
-   * sure that the history has all zeros. */
-  rep_hist_buffer_stats_init(now);
-  rep_hist_add_buffer_stats(2.0, 2.0, 20);
-  rep_hist_reset_buffer_stats(now);
-  s = rep_hist_format_buffer_stats(now + 86400);
-  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
-             "cell-processed-cells 0,0,0,0,0,0,0,0,0,0\n"
-             "cell-queued-cells 0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
-                               "0.00,0.00\n"
-             "cell-time-in-queue 0,0,0,0,0,0,0,0,0,0\n"
-             "cell-circuits-per-decile 0\n",OP_EQ, s);
-
- done:
-  tor_free(s);
-}
-
 #define ENT(name)                                                       \
   { #name, test_ ## name , 0, NULL, NULL }
 #define FORK(name)                                                      \
@@ -824,7 +651,6 @@ static struct testcase_t test_array[] = {
   { "fast_handshake", test_fast_handshake, 0, NULL, NULL },
   FORK(circuit_timeout),
   FORK(rend_fns),
-  FORK(stats),
 
   END_OF_TESTCASES
 };
@@ -845,18 +671,24 @@ struct testgroup_t testgroups[] = {
   { "channeltls/", channeltls_tests },
   { "checkdir/", checkdir_tests },
   { "circuitbuild/", circuitbuild_tests },
+  { "circuitpadding/", circuitpadding_tests },
   { "circuitlist/", circuitlist_tests },
   { "circuitmux/", circuitmux_tests },
-  { "circuituse/", circuituse_tests },
+  { "circuitmux_ewma/", circuitmux_ewma_tests },
   { "circuitstats/", circuitstats_tests },
+  { "circuituse/", circuituse_tests },
   { "compat/libevent/", compat_libevent_tests },
   { "config/", config_tests },
+  { "config/mgr/", confmgr_tests },
+  { "config/parse/", confparse_tests },
   { "connection/", connection_tests },
   { "conscache/", conscache_tests },
   { "consdiff/", consdiff_tests },
   { "consdiffmgr/", consdiffmgr_tests },
   { "container/", container_tests },
+  { "container/namemap/", namemap_tests },
   { "control/", controller_tests },
+  { "control/btrack/", btrack_tests },
   { "control/event/", controller_event_tests },
   { "crypto/", crypto_tests },
   { "crypto/ope/", crypto_ope_tests },
@@ -864,42 +696,57 @@ struct testgroup_t testgroups[] = {
   { "crypto/openssl/", crypto_openssl_tests },
 #endif
   { "crypto/pem/", pem_tests },
+  { "crypto/rng/", crypto_rng_tests },
   { "dir/", dir_tests },
-  { "dir_handle_get/", dir_handle_get_tests },
+  { "dir/auth/process_descs/", process_descs_tests },
   { "dir/md/", microdesc_tests },
-  { "dir/voting-schedule/", voting_schedule_tests },
+  { "dir/voting/flags/", voting_flags_tests },
+  { "dir/voting/schedule/", voting_schedule_tests },
+  { "dir_handle_get/", dir_handle_get_tests },
+  { "dispatch/", dispatch_tests, },
+  { "dns/", dns_tests },
   { "dos/", dos_tests },
   { "entryconn/", entryconn_tests },
   { "entrynodes/", entrynodes_tests },
-  { "guardfraction/", guardfraction_tests },
   { "extorport/", extorport_tests },
   { "geoip/", geoip_tests },
-  { "legacy_hs/", hs_tests },
+  { "guardfraction/", guardfraction_tests },
   { "hs_cache/", hs_cache },
   { "hs_cell/", hs_cell_tests },
+  { "hs_client/", hs_client_tests },
   { "hs_common/", hs_common_tests },
   { "hs_config/", hs_config_tests },
   { "hs_control/", hs_control_tests },
   { "hs_descriptor/", hs_descriptor },
+  { "hs_dos/", hs_dos_tests },
+  { "hs_intropoint/", hs_intropoint_tests },
   { "hs_ntor/", hs_ntor_tests },
+  { "hs_ob/", hs_ob_tests },
   { "hs_service/", hs_service_tests },
-  { "hs_client/", hs_client_tests },
-  { "hs_intropoint/", hs_intropoint_tests },
   { "introduce/", introduce_tests },
   { "keypin/", keypin_tests },
+  { "legacy_hs/", hs_tests },
   { "link-handshake/", link_handshake_tests },
   { "mainloop/", mainloop_tests },
+  { "netinfo/", netinfo_tests },
   { "nodelist/", nodelist_tests },
   { "oom/", oom_tests },
   { "oos/", oos_tests },
   { "options/", options_tests },
+  { "options/act/", options_act_tests },
+  { "parsecommon/", parsecommon_tests },
   { "periodic-event/" , periodic_event_tests },
   { "policy/" , policy_tests },
+  { "prob_distr/", prob_distr_tests },
   { "procmon/", procmon_tests },
+  { "process/", process_tests },
+  { "proto/haproxy/", proto_haproxy_tests },
   { "proto/http/", proto_http_tests },
   { "proto/misc/", proto_misc_tests },
   { "protover/", protover_tests },
   { "pt/", pt_tests },
+  { "pubsub/build/", pubsub_build_tests },
+  { "pubsub/msg/", pubsub_msg_tests },
   { "relay/" , relay_tests },
   { "relaycell/", relaycell_tests },
   { "relaycrypt/", relaycrypt_tests },
@@ -910,10 +757,13 @@ struct testgroup_t testgroups[] = {
   { "routerlist/", routerlist_tests },
   { "routerset/" , routerset_tests },
   { "scheduler/", scheduler_tests },
-  { "socks/", socks_tests },
+  { "sendme/", sendme_tests },
   { "shared-random/", sr_tests },
+  { "socks/", socks_tests },
+  { "stats/", stats_tests },
   { "status/" , status_tests },
   { "storagedir/", storagedir_tests },
+  { "token_bucket/", token_bucket_tests },
   { "tortls/", tortls_tests },
 #ifndef ENABLE_NSS
   { "tortls/openssl/", tortls_openssl_tests },
@@ -921,10 +771,9 @@ struct testgroup_t testgroups[] = {
   { "tortls/x509/", x509_tests },
   { "util/", util_tests },
   { "util/format/", util_format_tests },
+  { "util/handle/", handle_tests },
   { "util/logging/", logging_tests },
   { "util/process/", util_process_tests },
   { "util/thread/", thread_tests },
-  { "util/handle/", handle_tests },
-  { "dns/", dns_tests },
   END_OF_GROUPS
 };
diff --git a/src/test/test.h b/src/test/test.h
index aacc9dba87..18987719d0 100644
--- a/src/test/test.h
+++ b/src/test/test.h
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2003, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_H
@@ -37,6 +37,7 @@
 
 #define test_memeq_hex(expr1, hex) test_mem_op_hex(expr1, OP_EQ, hex)
 
+#ifndef COCCI
 #define tt_double_op(a,op,b)                                            \
   tt_assert_test_type(a,b,#a" "#op" "#b,double,(val1_ op val2_),"%g",   \
                       TT_EXIT_TEST_FUNCTION)
@@ -64,6 +65,7 @@
   tt_assert_test_fmt_type(a,b,#a" "#op" "#b,int64_t,(val1_ op val2_),  \
     int64_t, "%"PRId64,                                                \
     {print_ = (int64_t) value_;}, {}, TT_EXIT_TEST_FUNCTION)
+#endif /* !defined(COCCI) */
 
 /**
  * Declare that the test is done, even though no tt___op() calls were made.
@@ -79,169 +81,98 @@ struct crypto_pk_t *pk_generate(int idx);
 void init_pregenerated_keys(void);
 void free_pregenerated_keys(void);
 
-#define US2_CONCAT_2__(a, b) a ## __ ## b
-#define US_CONCAT_2__(a, b) a ## _ ## b
-#define US_CONCAT_3__(a, b, c) a ## _ ## b ## _ ## c
-#define US_CONCAT_2_(a, b) US_CONCAT_2__(a, b)
-#define US_CONCAT_3_(a, b, c) US_CONCAT_3__(a, b, c)
-
-/*
- * These macros are helpful for streamlining the authorship of several test
- * cases that use mocks.
- *
- * The pattern is as follows.
- * * Declare a top level namespace:
- *       #define NS_MODULE foo
- *
- * * For each test case you want to write, create a new submodule in the
- *   namespace. All mocks and other information should belong to a single
- *   submodule to avoid interference with other test cases.
- *   You can simply name the submodule after the function in the module you
- *   are testing:
- *       #define NS_SUBMODULE some_function
- *   or, if you're wanting to write several tests against the same function,
- *   ie., you are testing an aspect of that function, you can use:
- *       #define NS_SUBMODULE ASPECT(some_function, behavior)
- *
- * * Declare all the mocks you will use. The NS_DECL macro serves to declare
- *   the mock in the current namespace (defined by NS_MODULE and NS_SUBMODULE).
- *   It behaves like MOCK_DECL:
- *       NS_DECL(int, dependent_function, (void *));
- *   Here, dependent_function must be declared and implemented with the
- *   MOCK_DECL and MOCK_IMPL macros. The NS_DECL macro also defines an integer
- *   global for use for tracking how many times a mock was called, and can be
- *   accessed by CALLED(mock_name). For example, you might put
- *       CALLED(dependent_function)++;
- *   in your mock body.
- *
- * * Define a function called NS(main) that will contain the body of the
- *   test case. The NS macro can be used to reference a name in the current
- *   namespace.
- *
- * * In NS(main), indicate that a mock function in the current namespace,
- *   declared with NS_DECL is to override that in the global namespace,
- *   with the NS_MOCK macro:
- *       NS_MOCK(dependent_function)
- *   Unmock with:
- *       NS_UNMOCK(dependent_function)
- *
- * * Define the mocks with the NS macro, eg.,
- *       int
- *       NS(dependent_function)(void *)
- *       {
- *           CALLED(dependent_function)++;
- *       }
- *
- * * In the struct testcase_t array, you can use the TEST_CASE and
- *   TEST_CASE_ASPECT macros to define the cases without having to do so
- *   explicitly nor without having to reset NS_SUBMODULE, eg.,
- *       struct testcase_t foo_tests[] = {
- *         TEST_CASE_ASPECT(some_function, behavior),
- *         ...
- *         END_OF_TESTCASES
- *   which will define a test case named "some_function__behavior".
- */
-
-#define NAME_TEST_(name) #name
-#define NAME_TEST(name) NAME_TEST_(name)
-#define ASPECT(test_module, test_name) US2_CONCAT_2__(test_module, test_name)
-#define TEST_CASE(function)  \
-  {  \
-      NAME_TEST(function),  \
-      NS_FULL(NS_MODULE, function, test_main),  \
-      TT_FORK,  \
-      NULL,  \
-      NULL,  \
-  }
-#define TEST_CASE_ASPECT(function, aspect)  \
-  {  \
-      NAME_TEST(ASPECT(function, aspect)),  \
-      NS_FULL(NS_MODULE, ASPECT(function, aspect), test_main),  \
-      TT_FORK,  \
-      NULL,  \
-      NULL,  \
-  }
-
-#define NS(name) US_CONCAT_3_(NS_MODULE, NS_SUBMODULE, name)
-#define NS_FULL(module, submodule, name) US_CONCAT_3_(module, submodule, name)
-
-#define CALLED(mock_name) US_CONCAT_2_(NS(mock_name), called)
-#define NS_DECL(retval, mock_fn, args) \
-    extern int CALLED(mock_fn);        \
-    static retval NS(mock_fn) args; int CALLED(mock_fn) = 0
-#define NS_MOCK(name) MOCK(name, NS(name))
-#define NS_UNMOCK(name) UNMOCK(name)
-
 extern const struct testcase_setup_t passthrough_setup;
 extern const struct testcase_setup_t ed25519_test_setup;
 
 extern struct testcase_t accounting_tests[];
 extern struct testcase_t addr_tests[];
-extern struct testcase_t address_tests[];
 extern struct testcase_t address_set_tests[];
+extern struct testcase_t address_tests[];
 extern struct testcase_t bridges_tests[];
-extern struct testcase_t bwmgt_tests[];
+extern struct testcase_t btrack_tests[];
 extern struct testcase_t buffer_tests[];
+extern struct testcase_t bwmgt_tests[];
 extern struct testcase_t cell_format_tests[];
 extern struct testcase_t cell_queue_tests[];
 extern struct testcase_t channel_tests[];
 extern struct testcase_t channelpadding_tests[];
+extern struct testcase_t circuitpadding_tests[];
 extern struct testcase_t channeltls_tests[];
 extern struct testcase_t checkdir_tests[];
 extern struct testcase_t circuitbuild_tests[];
 extern struct testcase_t circuitlist_tests[];
 extern struct testcase_t circuitmux_tests[];
-extern struct testcase_t circuituse_tests[];
+extern struct testcase_t circuitmux_ewma_tests[];
 extern struct testcase_t circuitstats_tests[];
+extern struct testcase_t circuituse_tests[];
 extern struct testcase_t compat_libevent_tests[];
 extern struct testcase_t config_tests[];
+extern struct testcase_t confmgr_tests[];
+extern struct testcase_t confparse_tests[];
 extern struct testcase_t connection_tests[];
 extern struct testcase_t conscache_tests[];
 extern struct testcase_t consdiff_tests[];
 extern struct testcase_t consdiffmgr_tests[];
 extern struct testcase_t container_tests[];
-extern struct testcase_t controller_tests[];
 extern struct testcase_t controller_event_tests[];
-extern struct testcase_t crypto_tests[];
+extern struct testcase_t controller_tests[];
 extern struct testcase_t crypto_ope_tests[];
 extern struct testcase_t crypto_openssl_tests[];
-extern struct testcase_t dir_tests[];
+extern struct testcase_t crypto_rng_tests[];
+extern struct testcase_t crypto_tests[];
 extern struct testcase_t dir_handle_get_tests[];
+extern struct testcase_t dir_tests[];
+extern struct testcase_t dispatch_tests[];
+extern struct testcase_t dns_tests[];
 extern struct testcase_t dos_tests[];
 extern struct testcase_t entryconn_tests[];
 extern struct testcase_t entrynodes_tests[];
-extern struct testcase_t guardfraction_tests[];
 extern struct testcase_t extorport_tests[];
 extern struct testcase_t geoip_tests[];
-extern struct testcase_t hs_tests[];
+extern struct testcase_t guardfraction_tests[];
+extern struct testcase_t handle_tests[];
 extern struct testcase_t hs_cache[];
 extern struct testcase_t hs_cell_tests[];
+extern struct testcase_t hs_client_tests[];
 extern struct testcase_t hs_common_tests[];
 extern struct testcase_t hs_config_tests[];
 extern struct testcase_t hs_control_tests[];
 extern struct testcase_t hs_descriptor[];
+extern struct testcase_t hs_dos_tests[];
+extern struct testcase_t hs_intropoint_tests[];
 extern struct testcase_t hs_ntor_tests[];
+extern struct testcase_t hs_ob_tests[];
 extern struct testcase_t hs_service_tests[];
-extern struct testcase_t hs_client_tests[];
-extern struct testcase_t hs_intropoint_tests[];
+extern struct testcase_t hs_tests[];
 extern struct testcase_t introduce_tests[];
 extern struct testcase_t keypin_tests[];
 extern struct testcase_t link_handshake_tests[];
 extern struct testcase_t logging_tests[];
 extern struct testcase_t mainloop_tests[];
 extern struct testcase_t microdesc_tests[];
+extern struct testcase_t namemap_tests[];
+extern struct testcase_t netinfo_tests[];
 extern struct testcase_t nodelist_tests[];
 extern struct testcase_t oom_tests[];
 extern struct testcase_t oos_tests[];
 extern struct testcase_t options_tests[];
+extern struct testcase_t options_act_tests[];
+extern struct testcase_t parsecommon_tests[];
 extern struct testcase_t pem_tests[];
 extern struct testcase_t periodic_event_tests[];
 extern struct testcase_t policy_tests[];
+extern struct testcase_t prob_distr_tests[];
+extern struct testcase_t slow_stochastic_prob_distr_tests[];
 extern struct testcase_t procmon_tests[];
+extern struct testcase_t process_tests[];
+extern struct testcase_t process_descs_tests[];
+extern struct testcase_t proto_haproxy_tests[];
 extern struct testcase_t proto_http_tests[];
 extern struct testcase_t proto_misc_tests[];
 extern struct testcase_t protover_tests[];
 extern struct testcase_t pt_tests[];
+extern struct testcase_t pubsub_build_tests[];
+extern struct testcase_t pubsub_msg_tests[];
 extern struct testcase_t relay_tests[];
 extern struct testcase_t relaycell_tests[];
 extern struct testcase_t relaycrypt_tests[];
@@ -252,23 +183,26 @@ extern struct testcase_t routerkeys_tests[];
 extern struct testcase_t routerlist_tests[];
 extern struct testcase_t routerset_tests[];
 extern struct testcase_t scheduler_tests[];
-extern struct testcase_t storagedir_tests[];
+extern struct testcase_t sendme_tests[];
 extern struct testcase_t socks_tests[];
+extern struct testcase_t sr_tests[];
+extern struct testcase_t stats_tests[];
 extern struct testcase_t status_tests[];
+extern struct testcase_t storagedir_tests[];
 extern struct testcase_t thread_tests[];
-extern struct testcase_t tortls_tests[];
+extern struct testcase_t token_bucket_tests[];
 extern struct testcase_t tortls_openssl_tests[];
-extern struct testcase_t util_tests[];
+extern struct testcase_t tortls_tests[];
 extern struct testcase_t util_format_tests[];
 extern struct testcase_t util_process_tests[];
+extern struct testcase_t util_tests[];
+extern struct testcase_t voting_flags_tests[];
 extern struct testcase_t voting_schedule_tests[];
-extern struct testcase_t dns_tests[];
-extern struct testcase_t handle_tests[];
-extern struct testcase_t sr_tests[];
 extern struct testcase_t x509_tests[];
 
 extern struct testcase_t slow_crypto_tests[];
-extern struct testcase_t slow_util_tests[];
+extern struct testcase_t slow_process_tests[];
+extern struct testcase_t slow_ptr_tests[];
 
 extern struct testgroup_t testgroups[];
 
diff --git a/src/test/test_accounting.c b/src/test/test_accounting.c
index 8ae8fe4343..7933df5e35 100644
--- a/src/test/test_accounting.c
+++ b/src/test/test_accounting.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -11,19 +11,16 @@
 
 #include "app/config/or_state_st.h"
 
-#define NS_MODULE accounting
-
-#define NS_SUBMODULE limits
-
 /*
  * Test to make sure accounting triggers hibernation
  * correctly with both sum or max rules set
  */
 
 static or_state_t *or_state;
-NS_DECL(or_state_t *, get_or_state, (void));
+static or_state_t * acct_limits_get_or_state(void);
+ATTR_UNUSED static int acct_limits_get_or_state_called = 0;
 static or_state_t *
-NS(get_or_state)(void)
+acct_limits_get_or_state(void)
 {
   return or_state;
 }
@@ -35,7 +32,8 @@ test_accounting_limits(void *arg)
   time_t fake_time = time(NULL);
   (void) arg;
 
-  NS_MOCK(get_or_state);
+  MOCK(get_or_state,
+       acct_limits_get_or_state);
   or_state = or_state_new();
 
   options->AccountingMax = 100;
@@ -94,12 +92,10 @@ test_accounting_limits(void *arg)
 
   goto done;
  done:
-  NS_UNMOCK(get_or_state);
+  UNMOCK(get_or_state);
   or_state_free(or_state);
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t accounting_tests[] = {
   { "bwlimits", test_accounting_limits, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
diff --git a/src/test/test_addr.c b/src/test/test_addr.c
index 8868edce25..cf5aad7e71 100644
--- a/src/test/test_addr.c
+++ b/src/test/test_addr.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESSMAP_PRIVATE
@@ -11,6 +11,8 @@
 #include "feature/client/addressmap.h"
 #include "test/log_test_helpers.h"
 #include "lib/net/resolve.h"
+#include "test/rng_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #ifdef HAVE_SYS_UN_H
 #include <sys/un.h>
@@ -50,6 +52,7 @@ test_addr_basic(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define test_op_ip6_(a,op,b,e1,e2)                               \
   STMT_BEGIN                                                     \
   tt_assert_test_fmt_type(a,b,e1" "#op" "e2,struct in6_addr*,    \
@@ -67,6 +70,7 @@ test_addr_basic(void *arg)
     TT_EXIT_TEST_FUNCTION                                        \
   );                                                             \
   STMT_END
+#endif /* !defined(COCCI) */
 
 /** Helper: Assert that two strings both decode as IPv6 addresses with
  * tor_inet_pton(), and both decode to the same address. */
@@ -106,9 +110,10 @@ test_addr_basic(void *arg)
     tt_int_op(tor_inet_pton(AF_INET6, a, &t1.addr.in6_addr), OP_EQ, 1); \
     t1.family = AF_INET6;                                      \
     if (tor_addr_is_internal(&t1, for_listening))              \
-      TT_DIE(("%s was not internal", a));                      \
+      TT_DIE(("%s was internal", a));                      \
   STMT_END
 
+#ifndef COCCI
 /** Helper: Assert that <b>a</b> and <b>b</b>, when parsed by
  * tor_inet_pton(), give addresses that compare in the order defined by
  * <b>op</b> with tor_addr_compare(). */
@@ -133,6 +138,7 @@ test_addr_basic(void *arg)
       TT_DIE(("Failed: tor_addr_compare_masked(%s,%s,%d) %s 0", \
               a, b, m, #op));                                   \
   STMT_END
+#endif /* !defined(COCCI) */
 
 /** Helper: assert that <b>xx</b> is parseable as a masked IPv6 address with
  * ports by tor_parse_mask_addr_ports(), with family <b>f</b>, IP address
@@ -239,7 +245,7 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(0,OP_EQ, tor_addr_lookup("9000::5", AF_UNSPEC, &t1));
   tt_int_op(AF_INET6,OP_EQ, tor_addr_family(&t1));
   tt_int_op(0x90,OP_EQ, tor_addr_to_in6_addr8(&t1)[0]);
-  tt_assert(tor_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
+  tt_assert(fast_mem_is_zero((char*)tor_addr_to_in6_addr8(&t1)+1, 14));
   tt_int_op(0x05,OP_EQ, tor_addr_to_in6_addr8(&t1)[15]);
 
   /* === Test pton: valid af_inet6 */
@@ -653,12 +659,7 @@ test_addr_ip6_helpers(void *arg)
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
   tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x01010202);
   r=tor_addr_parse_mask_ports("3.4.16.032:1-2",0,&t1, &mask, &port1, &port2);
-  tt_int_op(r, OP_EQ, AF_INET);
-  tt_int_op(mask,OP_EQ,32);
-  tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET);
-  tt_int_op(tor_addr_to_ipv4h(&t1),OP_EQ,0x03041020);
-  tt_uint_op(port1, OP_EQ, 1);
-  tt_uint_op(port2, OP_EQ, 2);
+  tt_int_op(r, OP_EQ, -1);
   r=tor_addr_parse_mask_ports("1.1.2.3/255.255.128.0",0,&t1, &mask,NULL,NULL);
   tt_int_op(r, OP_EQ, AF_INET);
   tt_int_op(mask,OP_EQ,17);
@@ -696,7 +697,7 @@ test_addr_ip6_helpers(void *arg)
                               &t1,&mask,&port1,&port2);
   tt_int_op(r,OP_EQ,AF_INET6);
   tt_int_op(tor_addr_family(&t1),OP_EQ,AF_INET6);
-  tt_assert(tor_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
+  tt_assert(fast_mem_is_zero((const char*)tor_addr_to_in6_addr32(&t1), 16));
   tt_int_op(mask,OP_EQ,0);
   tt_int_op(port1,OP_EQ,1);
   tt_int_op(port2,OP_EQ,65535);
@@ -723,104 +724,559 @@ test_addr_ip6_helpers(void *arg)
   ;
 }
 
-/** Test tor_addr_port_parse(). */
+/* Test that addr_str successfully parses, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_PARSE_FMT(addr_str, expect_family, fmt_decorated, \
+                            expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that addr_str fails to parse, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_PARSE_XFAIL(addr_str) \
+  STMT_BEGIN \
+    r = tor_addr_parse(&addr, addr_str); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str and default_port successfully parse, and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_PARSE_FMT(addr_port_str, default_port, expect_family, \
+                                 fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that addr_port_str and default_port fail to parse, and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_PARSE_XFAIL(addr_port_str, default_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_parse(LOG_DEBUG, addr_port_str, &addr, &port, \
+                            default_port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as an IPv4 address using
+ * tor_lookup_hostname(), and:
+ *  - the fmt_addr32() of the result is expect_str.
+ */
+#define TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, expect_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(addr_str, &addr32h); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_str_op(fmt_addr32(addr32h), OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse using tor_lookup_hostname(), with a
+ * permanent failure, and:
+ *  - the returned address is 0.
+ */
+#define TEST_ADDR_V4_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(bad_str, &addr32h); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_str as an IPv4 address using tor_lookup_hostname()
+ * does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is 0.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_V4_LOOKUP(host_str) \
+  STMT_BEGIN \
+    r = tor_lookup_hostname(host_str, &addr32h); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_int_op(addr32h, OP_EQ, 0); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a require_family IP address using
+ * tor_addr_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str.
+ */
+#define TEST_ADDR_LOOKUP_FMT(addr_str, require_family, expect_family, \
+                             fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(addr_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+  STMT_END
+
+/* Test that bad_str fails to parse as a require_family IP address using
+ * tor_addr_lookup(), with a permanent failure, and:
+ *  - the returned address is null.
+ */
+#define TEST_ADDR_LOOKUP_XFAIL(bad_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(bad_str, require_family, &addr); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that looking up host_string as a require_family IP address using
+ * tor_addr_lookup(),  does something sensible:
+ *  - the result is -1, 0, or 1.
+ *  - if the result is a failure, the returned address is null.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_LOOKUP(host_str, require_family) \
+  STMT_BEGIN \
+    r = tor_addr_lookup(host_str, require_family, &addr); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 1); \
+    if (r != 0) \
+      tt_assert(tor_addr_is_null(&addr)); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the address has family expect_family,
+ *  - the fmt_decorated result of tor_addr_to_str() is expect_str,
+ *  - the port is expect_port.
+ */
+#define TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, expect_family, \
+                                  fmt_decorated, expect_str, expect_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(addr_port_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, 0); \
+    tt_int_op(tor_addr_family(&addr), OP_EQ, expect_family); \
+    sv = tor_addr_to_str(buf, &addr, sizeof(buf), fmt_decorated); \
+    tt_str_op(sv, OP_EQ, buf); \
+    tt_str_op(buf, OP_EQ, expect_str); \
+    tt_int_op(port, OP_EQ, expect_port); \
+  STMT_END
+
+/* Test that bad_str fails to parse as an IP address and port
+ * using tor_addr_port_lookup(), and:
+ *  - the returned address is null,
+ *  - the returned port is 0.
+ */
+#define TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(bad_str, &addr, &port); \
+    tt_int_op(r, OP_EQ, -1); \
+    tt_assert(tor_addr_is_null(&addr)); \
+    tt_int_op(port, OP_EQ, 0); \
+  STMT_END
+
+/* Test that looking up host_port_str as an IP address using
+ * tor_addr_port_lookup(),  does something sensible:
+ *  - the result is -1 or 0.
+ *  - if the result is a failure, the returned address is null, and the
+ *    returned port is zero,
+ *  - if the result is a success, the returned port is expect_success_port,
+ *    and the returned family is AF_INET or AF_INET6.
+ * We can't rely on the result of this function, because it depends on the
+ * network.
+ */
+#define TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    r = tor_addr_port_lookup(host_port_str, &addr, &port); \
+    tt_int_op(r, OP_GE, -1); \
+    tt_int_op(r, OP_LE, 0); \
+    if (r == -1) { \
+      tt_assert(tor_addr_is_null(&addr)); \
+      tt_int_op(port, OP_EQ, 0); \
+    } else { \
+      tt_assert(tor_addr_family(&addr) == AF_INET || \
+                tor_addr_family(&addr) == AF_INET6); \
+      tt_int_op(port, OP_EQ, expect_success_port); \
+    } \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical IPv4 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup() with AF_INET,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_port_lookup(), because there is no port.
+ */
+#define TEST_ADDR_V4_PARSE_CANONICAL(addr_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 111, AF_INET, 0, \
+                             addr_str, 111); \
+    TEST_ADDR_V4_LOOKUP_HOSTNAME(addr_str, addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET, 0, addr_str, 0); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET, AF_INET, 0, addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET, 0, addr_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_str successfully parses as a canonical fmt_decorated
+ * IPv6 address.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port,
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE_CANONICAL(addr_str, fmt_decorated) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, addr_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 222, AF_INET6, fmt_decorated, \
+                             addr_str, 222); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         addr_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, addr_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_str successfully parses, and the fmt_decorated canonical
+ * IPv6 string is expect_str.
+ * Check for successful parsing using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_lookup() with AF_INET6,
+ *  - tor_addr_lookup() with AF_UNSPEC,
+ *  - tor_addr_port_lookup(), with a zero port.
+ * Check for failures using:
+ *  - tor_addr_port_parse() without a default port, because there is no port.
+ *  - tor_lookup_hostname(), because it only supports IPv4,
+ *  - tor_addr_lookup() with AF_INET.
+ */
+#define TEST_ADDR_V6_PARSE(addr_str, fmt_decorated, expect_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_FMT(addr_str, AF_INET6, fmt_decorated, expect_str); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_str, 333, AF_INET6, fmt_decorated, \
+                             expect_str, 333); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_INET6, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_LOOKUP_FMT(addr_str, AF_UNSPEC, AF_INET6, fmt_decorated, \
+                         expect_str); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_str, AF_INET6, fmt_decorated, expect_str, \
+                              0); \
+    TEST_ADDR_PORT_PARSE_XFAIL(addr_str, -1); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_str, AF_INET); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical IPv4 address
+ * string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port.
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V4_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 444, AF_INET, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+  STMT_END
+
+/* Test that addr_port_str successfully parses to the canonical undecorated
+ * IPv6 address string expect_str, and port expect_port.
+ * Check for successful parsing using:
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_addr_port_lookup().
+ * Check for failures using:
+ *  - tor_addr_parse(), because there is a port,
+ *  - tor_lookup_hostname(), because there is a port, and because it only
+ *    supports IPv4,
+ *  - tor_addr_lookup(), regardless of the address family, because there is a
+ *    port.
+ */
+#define TEST_ADDR_V6_PORT_PARSE(addr_port_str, expect_str, expect_port) \
+  STMT_BEGIN \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str,  -1, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_PARSE_FMT(addr_port_str, 555, AF_INET6, 0, expect_str, \
+                             expect_port); \
+    TEST_ADDR_PORT_LOOKUP_FMT(addr_port_str, AF_INET6, 0, expect_str, \
+                              expect_port); \
+    TEST_ADDR_PARSE_XFAIL(addr_port_str); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(addr_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET6); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(addr_port_str, AF_INET); \
+  STMT_END
+
+/* Test that bad_str fails to parse due to a bad address or port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup().
+ */
+#define TEST_ADDR_PARSE_XFAIL_MALFORMED(bad_str) \
+  STMT_BEGIN \
+    TEST_ADDR_PARSE_XFAIL(bad_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(bad_str, 666); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(bad_str); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(bad_str, AF_INET6); \
+    TEST_ADDR_PORT_LOOKUP_XFAIL(bad_str); \
+  STMT_END
+
+/* Test that host_str is treated as a hostname, and not an address.
+ * Check for success or failure using the network-dependent functions:
+ *  - tor_lookup_hostname(),
+ *  - tor_addr_lookup(), regardless of the address family,
+ *  - tor_addr_port_lookup(), expecting a zero port.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port.
+ */
+#define TEST_HOSTNAME(host_str) \
+  STMT_BEGIN \
+    TEST_HOST_V4_LOOKUP(host_str); \
+    TEST_HOST_LOOKUP(host_str, AF_UNSPEC); \
+    TEST_HOST_LOOKUP(host_str, AF_INET); \
+    TEST_HOST_LOOKUP(host_str, AF_INET6); \
+    TEST_HOST_PORT_LOOKUP(host_str, 0); \
+    TEST_ADDR_PARSE_XFAIL(host_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_str, 777); \
+  STMT_END
+
+/* Test that host_port_str is treated as a hostname and port, and not a
+ * hostname or an address.
+ * Check for success or failure using the network-dependent function:
+ *  - tor_addr_port_lookup(), expecting expect_success_port if the lookup is
+ *    successful.
+ * Check for failures using:
+ *  - tor_addr_parse(),
+ *  - tor_addr_port_parse() without a default port,
+ *  - tor_addr_port_parse() with a default port,
+ *  - tor_lookup_hostname(), because it doesn't support ports,
+ *  - tor_addr_lookup(), regardless of the address family, because it doesn't
+ *    support ports.
+ */
+#define TEST_HOSTNAME_PORT(host_port_str, expect_success_port) \
+  STMT_BEGIN \
+    TEST_HOST_PORT_LOOKUP(host_port_str, expect_success_port); \
+    TEST_ADDR_PARSE_XFAIL(host_port_str); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str,  -1); \
+    TEST_ADDR_PORT_PARSE_XFAIL(host_port_str, 888); \
+    TEST_ADDR_V4_LOOKUP_XFAIL(host_port_str); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_UNSPEC); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET); \
+    TEST_ADDR_LOOKUP_XFAIL(host_port_str, AF_INET6); \
+  STMT_END
+
+static void
+test_addr_parse_canonical(void *arg)
+{
+  int r;
+  tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
+  char buf[TOR_ADDR_BUF_LEN];
+
+  (void)arg;
+
+  /* Correct calls. */
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.1");
+  TEST_ADDR_V4_PARSE_CANONICAL("192.0.2.2");
+
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::33:44]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::1]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[2::]", 1);
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22:33:44:55:66:77:88]", 1);
+
+  /* Allow IPv6 without square brackets, when there is no port, but only if
+   * there is a default port */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::33:44", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::1", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("2::", 0);
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22:33:44:55:66:77:88", 0);
+ done:
+  ;
+}
+
+/** Test tor_addr_parse() and tor_addr_port_parse(). */
 static void
 test_addr_parse(void *arg)
 {
+
   int r;
   tor_addr_t addr;
+  uint16_t port;
+  const char *sv;
+  uint32_t addr32h;
   char buf[TOR_ADDR_BUF_LEN];
-  uint16_t port = 0;
 
-  /* Correct call. */
   (void)arg;
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.1:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "192.0.2.1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, 0);
-  tor_addr_to_str(buf, &addr, sizeof(buf), 0);
-  tt_str_op(buf,OP_EQ, "::1");
-  tt_int_op(port,OP_EQ, 1234);
-
-  /* Domain name. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
 
-  /* Only IP. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  mock_hostname_resolver();
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,200);
+  /* IPv6-mapped IPv4 addresses. Tor doesn't really use these. */
+  TEST_ADDR_V6_PARSE("11:22:33:44:55:66:1.2.3.4", 0,
+                     "11:22:33:44:55:66:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_ADDR_V6_PARSE("11:22::33:44:1.2.3.4", 0,
+                     "11:22::33:44:102:304");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]",
-                         &addr, &port, 400);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,400);
+  /* Ports. */
+  TEST_ADDR_V4_PORT_PARSE("192.0.2.1:1234", "192.0.2.1", 1234);
+  TEST_ADDR_V6_PORT_PARSE("[::1]:1234", "::1", 1234);
 
-  /* Bad port. */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:66666",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  /* Host names. */
+  TEST_HOSTNAME("localhost");
+  TEST_HOSTNAME_PORT("localhost:1234", 1234);
+  TEST_HOSTNAME_PORT("localhost:0", 0);
 
-  /* Only domain name */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "torproject.org",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("torproject.org");
+  TEST_HOSTNAME_PORT("torproject.org:56", 56);
 
-  /* Bad IP address */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2:1234",
-                         &addr, &port, -1);
-  tt_int_op(r, OP_EQ, -1);
+  TEST_HOSTNAME("probably-not-a-valid-dns.name-tld");
+  TEST_HOSTNAME_PORT("probably-not-a-valid-dns.name-tld:789", 789);
+
+  /* Malformed addresses. */
+  /* Empty string. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("");
+
+  /* Square brackets around IPv4 address. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.1]");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[192.0.2.3]:12345");
+
+  /* Only left square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[11:22::33:44");
+
+  /* Only right square bracket. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44]");
 
-  /* Make sure that the default port has lower priority than the real
-     one */
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "192.0.2.2:1337",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1337);
+  /* Leading colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED(":11:22::33:44");
 
-  r= tor_addr_port_parse(LOG_DEBUG,
-                         "[::1]:1369",
-                         &addr, &port, 200);
-  tt_int_op(r, OP_EQ, 0);
-  tt_int_op(port,OP_EQ,1369);
+  /* Trailing colon. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("11:22::33:44:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:");
+
+  /* Bad port. */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:66666");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:77777");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:88888");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:99999");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:-2");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:-3");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:-4");
+
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:1 bad");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("192.0.2.2:bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("[::1]:bad-port-1");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("::1:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("localhost:1-bad-port-1");
+
+  /* Bad hostname */
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid");
+  TEST_ADDR_PARSE_XFAIL_MALFORMED("definitely invalid:22222");
+
+  /* Ambiguous cases */
+  /* Too many hex words in IPv4-mapped IPv6 address.
+   * But some OS host lookup routines accept it as a hostname, or
+   * as an IP address?? (I assume they discard unused characters). */
+  TEST_HOSTNAME("11:22:33:44:55:66:77:88:1.2.3.4");
+
+  /* IPv6 address with port and no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address:port ? */
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+  /* Is it a port, or are there too many hex words?
+   * We reject it either way, but some OS host lookup routines accept it as an
+   * IPv6 address:port */
+  TEST_HOSTNAME_PORT("11:22:33:44:55:66:77:88:99", 99);
+  /* But we accept it if it has square brackets. */
+  TEST_ADDR_V6_PORT_PARSE("[11:22:33:44:55:66:77:88]:99",
+                           "11:22:33:44:55:66:77:88",99);
+
+  /* Bad IPv4 address
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv4 address[:port], with a zero last octet */
+  TEST_HOSTNAME("192.0.1");
+  TEST_HOSTNAME_PORT("192.0.2:1234", 1234);
+
+  /* More bad IPv6 addresses and ports: no brackets
+   * We reject it, but some OS host lookup routines accept it as an
+   * IPv6 address[:port] */
+  TEST_HOSTNAME_PORT("::1:12345", 12345);
+  TEST_HOSTNAME_PORT("11:22::33:44:12345", 12345);
+
+  /* And this is an ambiguous case, which is interpreted as an IPv6 address. */
+  TEST_ADDR_V6_PARSE_CANONICAL("11:22::88:99", 0);
+  /* Use square brackets to resolve the ambiguity */
+  TEST_ADDR_V6_PARSE_CANONICAL("[11:22::88:99]", 1);
+  TEST_ADDR_V6_PORT_PARSE("[11:22::88]:99",
+                           "11:22::88",99);
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -891,27 +1347,6 @@ test_virtaddrmap(void *data)
   ;
 }
 
-static const char *canned_data = NULL;
-static size_t canned_data_len = 0;
-
-/* Mock replacement for crypto_rand() that returns canned data from
- * canned_data above. */
-static void
-crypto_canned(char *ptr, size_t n)
-{
-  if (canned_data_len) {
-    size_t to_copy = MIN(n, canned_data_len);
-    memcpy(ptr, canned_data, to_copy);
-    canned_data += to_copy;
-    canned_data_len -= to_copy;
-    n -= to_copy;
-    ptr += to_copy;
-  }
-  if (n) {
-    crypto_rand_unmocked(ptr, n);
-  }
-}
-
 static void
 test_virtaddrmap_persist(void *data)
 {
@@ -919,6 +1354,8 @@ test_virtaddrmap_persist(void *data)
   const char *a, *b, *c;
   tor_addr_t addr;
   char *ones = NULL;
+  const char *canned_data;
+  size_t canned_data_len;
 
   addressmap_init();
 
@@ -937,7 +1374,7 @@ test_virtaddrmap_persist(void *data)
                 "1234567890" // the second call returns this.
                 "abcdefghij"; // the third call returns this.
   canned_data_len = 30;
-  MOCK(crypto_rand, crypto_canned);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
 
   a = addressmap_register_virtual_address(RESOLVED_TYPE_HOSTNAME,
                                           tor_strdup("quuxit.baz"));
@@ -947,9 +1384,9 @@ test_virtaddrmap_persist(void *data)
   tt_assert(b);
   tt_str_op(a, OP_EQ, "gezdgnbvgy3tqojq.virtual");
   tt_str_op(b, OP_EQ, "mfrggzdfmztwq2lk.virtual");
+  testing_disable_prefilled_rng();
 
   // Now try something to get us an ipv4 address
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("192.168.0.0/16",
                                                 AF_INET, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -966,22 +1403,23 @@ test_virtaddrmap_persist(void *data)
 
   // Try some canned entropy and verify all the we discard duplicates,
   // addresses that end with 0, and addresses that end with 255.
-  MOCK(crypto_rand, crypto_canned);
   canned_data = "\x01\x02\x03\x04" // okay
                 "\x01\x02\x03\x04" // duplicate
                 "\x03\x04\x00\x00" // bad ending 1
                 "\x05\x05\x00\xff" // bad ending 2
                 "\x05\x06\x07\xf0"; // okay
   canned_data_len = 20;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumble.onion"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
                                           tor_strdup("wumpus.onion"));
   tt_str_op(a, OP_EQ, "192.168.3.4");
   tt_str_op(b, OP_EQ, "192.168.7.240");
+  testing_disable_prefilled_rng();
 
   // Now try IPv6!
-  UNMOCK(crypto_rand);
   tt_int_op(0,OP_EQ, parse_virtual_addr_network("1010:F000::/20",
                                                 AF_INET6, 0, NULL));
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -997,7 +1435,7 @@ test_virtaddrmap_persist(void *data)
   tt_assert(!strcmpstart(b, "[1010:f"));
 
   // Try IPv6 with canned entropy, to make sure we detect duplicates.
-  MOCK(crypto_rand, crypto_canned);
+
   canned_data = "acanthopterygian" // okay
                 "cinematographist" // okay
                 "acanthopterygian" // duplicate
@@ -1006,6 +1444,8 @@ test_virtaddrmap_persist(void *data)
                 "cinematographist" // duplicate
                 "coadministration"; // okay
   canned_data_len = 16 * 7;
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
+
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
                                           tor_strdup("wuffle.baz"));
   b = addressmap_register_virtual_address(RESOLVED_TYPE_IPV6,
@@ -1018,9 +1458,11 @@ test_virtaddrmap_persist(void *data)
 
   // Try address exhaustion: make sure we can actually fail if we
   // get too many already-existing addresses.
+  testing_disable_prefilled_rng();
   canned_data_len = 128*1024;
   canned_data = ones = tor_malloc(canned_data_len);
   memset(ones, 1, canned_data_len);
+  testing_enable_prefilled_rng(canned_data, canned_data_len);
   // There is some chance this one will fail if a previous random
   // allocation gave out the address already.
   a = addressmap_register_virtual_address(RESOLVED_TYPE_IPV4,
@@ -1037,7 +1479,7 @@ test_virtaddrmap_persist(void *data)
   expect_single_log_msg_containing("Ran out of virtual addresses!");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(ones);
   addressmap_free_all();
   teardown_capture_of_logs();
@@ -1206,13 +1648,169 @@ test_addr_rfc6598(void *arg)
   ;
 }
 
+#define TEST_ADDR_ATON(a, rv) STMT_BEGIN \
+    struct in_addr addr; \
+    tt_int_op(tor_inet_aton(a, &addr), OP_EQ, rv); \
+  STMT_END;
+
+static void
+test_addr_octal(void *arg)
+{
+  (void)arg;
+
+  /* Test non-octal IP addresses. */
+  TEST_ADDR_ATON("0.1.2.3", 1);
+  TEST_ADDR_ATON("1.0.2.3", 1);
+  TEST_ADDR_ATON("1.2.3.0", 1);
+
+  /* Test octal IP addresses. */
+  TEST_ADDR_ATON("01.1.2.3", 0);
+  TEST_ADDR_ATON("1.02.3.4", 0);
+  TEST_ADDR_ATON("1.2.3.04", 0);
+ done:
+  ;
+}
+
+#define get_ipv4(test_addr, str, iprv) STMT_BEGIN               \
+    test_addr = tor_malloc(sizeof(tor_addr_t));                 \
+    test_addr->family = AF_INET;                                \
+    iprv = tor_inet_aton(str, &test_addr->addr.in_addr);        \
+    tor_assert(iprv);                                           \
+  STMT_END;
+
+#define get_ipv6(test_addr, str, iprv) STMT_BEGIN                       \
+    test_addr = tor_malloc(sizeof(tor_addr_t));                         \
+    test_addr->family = AF_INET6;                                       \
+    iprv = tor_inet_pton(AF_INET6, str, &test_addr->addr.in6_addr);     \
+    tor_assert(iprv);                                                   \
+  STMT_END;
+
+#define get_af_unix(test_addr) STMT_BEGIN                       \
+    test_addr = tor_malloc_zero(sizeof(tor_addr_t));            \
+    test_addr->family = AF_UNIX;                                \
+  STMT_END;
+
+#define get_af_unspec(test_addr) STMT_BEGIN                     \
+    test_addr = tor_malloc_zero(sizeof(tor_addr_t));            \
+    test_addr->family = AF_UNSPEC;                              \
+  STMT_END;
+
+#define TEST_ADDR_VALIDITY(a, lis, rv) STMT_BEGIN               \
+    tor_assert(a);                                              \
+    tt_int_op(tor_addr_is_valid(a, lis), OP_EQ, rv);            \
+  STMT_END;
+
+/* Here we can change the addresses we are testing for. */
+#define IP4_TEST_ADDR "123.98.45.1"
+#define IP6_TEST_ADDR "2001:0DB8:AC10:FE01::"
+
+static void
+test_addr_is_valid(void *arg)
+{
+  (void)arg;
+  tor_addr_t *test_addr;
+  int iprv;
+
+  /* Tests for IPv4 addresses. */
+
+  /* Test for null IPv4 address. */
+  get_ipv4(test_addr, "0.0.0.0", iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv4 address. */
+  get_ipv4(test_addr, IP4_TEST_ADDR, iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 1);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Tests for IPv6 addresses. */
+
+  /* Test for null IPv6 address. */
+  get_ipv6(test_addr, "::", iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv6 address. */
+  get_ipv6(test_addr, IP6_TEST_ADDR, iprv);
+  TEST_ADDR_VALIDITY(test_addr, 0, 1);
+  TEST_ADDR_VALIDITY(test_addr, 1, 1);
+  tor_free(test_addr);
+
+  /* Test for address of type AF_UNIX. */
+
+  get_af_unix(test_addr);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 0);
+  tor_free(test_addr);
+
+  /* Test for address of type AF_UNSPEC. */
+
+  get_af_unspec(test_addr);
+  TEST_ADDR_VALIDITY(test_addr, 0, 0);
+  TEST_ADDR_VALIDITY(test_addr, 1, 0);
+
+ done:
+  tor_free(test_addr);
+}
+
+#define TEST_ADDR_IS_NULL(a, rv) STMT_BEGIN                 \
+    tor_assert(a);                                          \
+    tt_int_op(tor_addr_is_null(a), OP_EQ, rv);              \
+  STMT_END;
+
+static void
+test_addr_is_null(void *arg)
+{
+  (void)arg;
+  tor_addr_t *test_addr;
+  int iprv;
+
+  /* Test for null IPv4. */
+  get_ipv4(test_addr, "0.0.0.0", iprv);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv4. */
+  get_ipv4(test_addr, IP4_TEST_ADDR, iprv);
+  TEST_ADDR_IS_NULL(test_addr, 0);
+  tor_free(test_addr);
+
+  /* Test for null IPv6. */
+  get_ipv6(test_addr, "::", iprv);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+  tor_free(test_addr);
+
+  /* Test for non-null IPv6. */
+  get_ipv6(test_addr, IP6_TEST_ADDR, iprv);
+  TEST_ADDR_IS_NULL(test_addr, 0);
+  tor_free(test_addr);
+
+  /* Test for address family AF_UNIX. */
+  get_af_unix(test_addr);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+  tor_free(test_addr);
+
+  /* Test for address family AF_UNSPEC. */
+  get_af_unspec(test_addr);
+  TEST_ADDR_IS_NULL(test_addr, 1);
+
+ done:
+  tor_free(test_addr);
+}
+
+#ifndef COCCI
 #define ADDR_LEGACY(name)                                               \
   { #name, test_addr_ ## name , 0, NULL, NULL }
+#endif
 
 struct testcase_t addr_tests[] = {
   ADDR_LEGACY(basic),
   ADDR_LEGACY(ip6_helpers),
   ADDR_LEGACY(parse),
+  ADDR_LEGACY(parse_canonical),
   { "virtaddr", test_virtaddrmap, 0, NULL, NULL },
   { "virtaddr_persist", test_virtaddrmap_persist, TT_FORK, NULL, NULL },
   { "localname", test_addr_localname, 0, NULL, NULL },
@@ -1221,5 +1819,8 @@ struct testcase_t addr_tests[] = {
   { "is_loopback", test_addr_is_loopback, 0, NULL, NULL },
   { "make_null", test_addr_make_null, 0, NULL, NULL },
   { "rfc6598", test_addr_rfc6598, 0, NULL, NULL },
+  { "octal", test_addr_octal, 0, NULL, NULL },
+  { "address_validity", test_addr_is_valid, 0, NULL, NULL },
+  { "address_is_null", test_addr_is_null, 0, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_address.c b/src/test/test_address.c
index c33c30aee5..4cedbda347 100644
--- a/src/test/test_address.c
+++ b/src/test/test_address.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ADDRESS_PRIVATE
@@ -24,6 +24,10 @@
 #endif /* defined(HAVE_IFCONF_TO_SMARTLIST) */
 
 #include "core/or/or.h"
+#include "app/config/config.h"
+#include "feature/dirauth/process_descs.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/node_st.h"
 #include "feature/nodelist/nodelist.h"
 #include "lib/net/address.h"
 #include "test/test.h"
@@ -456,7 +460,7 @@ test_address_ifreq_to_smartlist(void *arg)
   ifc->ifc_len = sizeof(struct ifreq);
   ifc->ifc_ifcu.ifcu_req = ifr;
 
-  results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
+  results = ifreq_to_smartlist((const uint8_t *)ifc->ifc_buf,ifc->ifc_len);
   tt_int_op(smartlist_len(results),OP_EQ,1);
 
   tor_addr = smartlist_get(results, 0);
@@ -479,7 +483,7 @@ test_address_ifreq_to_smartlist(void *arg)
   SMARTLIST_FOREACH(results, tor_addr_t *, t, tor_free(t));
   smartlist_free(results);
 
-  results = ifreq_to_smartlist(ifc->ifc_buf,ifc->ifc_len);
+  results = ifreq_to_smartlist((const uint8_t *)ifc->ifc_buf,ifc->ifc_len);
   tt_int_op(smartlist_len(results),OP_EQ,2);
 
   tor_addr = smartlist_get(results, 0);
@@ -1170,8 +1174,167 @@ test_address_tor_addr_in_same_network_family(void *ignored)
   return;
 }
 
+static node_t *
+helper_create_mock_node(char id_char)
+{
+  node_t *node = tor_malloc_zero(sizeof(node_t));
+  routerinfo_t *ri = tor_malloc_zero(sizeof(routerinfo_t));
+  tor_addr_make_null(&ri->ipv6_addr, AF_INET6);
+  node->ri = ri;
+  memset(node->identity, id_char, sizeof(node->identity));
+  return node;
+}
+
+static void
+helper_free_mock_node(node_t *node)
+{
+  if (!node)
+    return;
+  tor_free(node->ri);
+  tor_free(node);
+}
+
+#define NODE_SET_IPV4(node, ipv4_addr, ipv4_port) { \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, ipv4_addr); \
+    node->ri->addr = tor_addr_to_ipv4h(&addr); \
+    node->ri->or_port = ipv4_port; \
+  }
+
+#define NODE_CLEAR_IPV4(node) { \
+    node->ri->addr = 0; \
+    node->ri->or_port = 0; \
+  }
+
+#define NODE_SET_IPV6(node, ipv6_addr_str, ipv6_port) { \
+    tor_addr_parse(&node->ri->ipv6_addr, ipv6_addr_str); \
+    node->ri->ipv6_orport = ipv6_port; \
+  }
+
+static void
+test_address_tor_node_in_same_network_family(void *ignored)
+{
+  (void)ignored;
+  node_t *node_a = helper_create_mock_node('a');
+  node_t *node_b = helper_create_mock_node('b');
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "8.8.4.4", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+  NODE_SET_IPV4(node_a, "8.8.8.8", 1);
+  NODE_SET_IPV4(node_b, "1.1.1.1", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_a);
+  NODE_SET_IPV6(node_a, "2001:470:20::2", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_CLEAR_IPV4(node_b);
+  NODE_SET_IPV6(node_b, "2606:4700:4700::1111", 1);
+
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 0);
+
+  NODE_SET_IPV6(node_a, "2606:4700:4700::1001", 1);
+  tt_int_op(nodes_in_same_family(node_a, node_b), OP_EQ, 1);
+
+ done:
+  helper_free_mock_node(node_a);
+  helper_free_mock_node(node_b);
+}
+
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+/* Test dirserv_router_has_valid_address() on a stub routerinfo, with only its
+ * address fields set. Use IPv4 ipv4_addr_str and IPv6 ipv6_addr_str.
+ * Fail if it does not return rv. */
+#define TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, ipv6_addr_str, rv) \
+  STMT_BEGIN \
+    ri = tor_malloc_zero(sizeof(routerinfo_t)); \
+    tor_addr_t addr; \
+    tor_addr_parse(&addr, (ipv4_addr_str));   \
+    ri->addr = tor_addr_to_ipv4h(&addr); \
+    tor_addr_parse(&ri->ipv6_addr, (ipv6_addr_str)); \
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, (rv)); \
+    tor_free(ri); \
+  STMT_END
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a null
+ * IPv6 address. */
+#define CHECK_RI_ADDR(ipv4_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER(ipv4_addr_str, "::", rv)
+
+/* Like TEST_ROUTER_VALID_ADDRESS_HELPER(), but always passes a non-internal
+ * IPv4 address, so that the IPv6 check is reached. */
+#define CHECK_RI_ADDR6(ipv6_addr_str, rv) \
+  TEST_ROUTER_VALID_ADDRESS_HELPER("1.0.0.1", ipv6_addr_str, rv)
+
+static void
+test_address_dirserv_router_addr_private(void *opt_dir_allow_private)
+{
+  /* A stub routerinfo structure, with only its address fields set. */
+  routerinfo_t *ri = NULL;
+  /* The expected return value for private addresses.
+   * Modified if DirAllowPrivateAddresses is 1. */
+  int private_rv = -1;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(get_options, mock_get_options);
+
+  if (opt_dir_allow_private) {
+    mock_options.DirAllowPrivateAddresses = 1;
+    private_rv = 0;
+  }
+
+  CHECK_RI_ADDR("1.0.0.1", 0);
+  CHECK_RI_ADDR("10.0.0.1", private_rv);
+
+  CHECK_RI_ADDR6("2600::1", 0);
+  CHECK_RI_ADDR6("fe80::1", private_rv);
+
+  /* Null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  CHECK_RI_ADDR("0.0.0.0", private_rv);
+  TEST_ROUTER_VALID_ADDRESS_HELPER("0.0.0.0", "::", private_rv);
+
+  /* IPv6 null succeeds, because IPv4 is not null */
+  CHECK_RI_ADDR6("::", 0);
+
+  /* Byte-zeroed null addresses */
+  /* IPv4 null fails, regardless of IPv6 */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, private_rv);
+    tor_free(ri);
+  }
+
+  /* IPv6 null succeeds, because IPv4 is not internal */
+  {
+    ri = tor_malloc_zero(sizeof(routerinfo_t));
+    ri->addr = 16777217; /* 1.0.0.1 */
+    tt_int_op(dirserv_router_has_valid_address(ri), OP_EQ, 0);
+    tor_free(ri);
+  }
+
+ done:
+  tor_free(ri);
+  UNMOCK(get_options);
+}
+
 #define ADDRESS_TEST(name, flags) \
   { #name, test_address_ ## name, flags, NULL, NULL }
+#define ADDRESS_TEST_STR_ARG(name, flags, str_arg) \
+  { #name "/" str_arg, test_address_ ## name, flags, &passthrough_setup, \
+    (void *)(str_arg) }
 
 struct testcase_t address_tests[] = {
   ADDRESS_TEST(udp_socket_trick_whitebox, TT_FORK),
@@ -1202,5 +1365,8 @@ struct testcase_t address_tests[] = {
   ADDRESS_TEST(tor_addr_to_mapped_ipv4h, 0),
   ADDRESS_TEST(tor_addr_eq_ipv4h, 0),
   ADDRESS_TEST(tor_addr_in_same_network_family, 0),
+  ADDRESS_TEST(tor_node_in_same_network_family, 0),
+  ADDRESS_TEST(dirserv_router_addr_private, 0),
+  ADDRESS_TEST_STR_ARG(dirserv_router_addr_private, 0, "allow_private"),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_address_set.c b/src/test/test_address_set.c
index 4e55d71ff4..c2fe718935 100644
--- a/src/test/test_address_set.c
+++ b/src/test/test_address_set.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -105,6 +105,8 @@ test_nodelist(void *arg)
        mock_networkstatus_get_latest_consensus_by_flavor);
   MOCK(get_estimated_address_per_node,
        mock_get_estimated_address_per_node);
+  MOCK(dirlist_add_trusted_dir_addresses,
+       mock_dirlist_add_trusted_dir_addresses);
 
   dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
   dummy_ns->flavor = FLAV_MICRODESC;
@@ -120,7 +122,10 @@ test_nodelist(void *arg)
    * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
   addr_per_node = 1024;
 
-  /* No node no nothing. The lookups should be empty. */
+  /* No node no nothing. The lookups should be empty. We've mocked the
+   * dirlist_add_trusted_dir_addresses in order for _no_ authorities to be
+   * added to the filter else it makes this test to trigger many false
+   * positive. */
   nodelist_set_consensus(dummy_ns);
 
   /* The address set should be empty. */
@@ -174,6 +179,7 @@ test_nodelist(void *arg)
   UNMOCK(networkstatus_get_latest_consensus);
   UNMOCK(networkstatus_get_latest_consensus_by_flavor);
   UNMOCK(get_estimated_address_per_node);
+  UNMOCK(dirlist_add_trusted_dir_addresses);
 }
 
 /** Test that the no-reentry exit filter works as intended */
diff --git a/src/test/test_bridges.c b/src/test/test_bridges.c
index 879ae6636b..f1624a529d 100644
--- a/src/test/test_bridges.c
+++ b/src/test/test_bridges.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information     */
 
 /**
@@ -550,8 +550,6 @@ test_bridges_get_transport_by_bridge_addrport_no_ptlist(void *arg)
   sweep_bridge_list();
 }
 
-#define PT_PRIVATE
-
 /**
  * Calling get_transport_by_bridge_addrport() with the address and port of a
  * configured bridge which uses a pluggable transport should return 0 and set
diff --git a/src/test/test_bt.sh b/src/test/test_bt.sh
index df8bcb8eda..312905a4e2 100755
--- a/src/test/test_bt.sh
+++ b/src/test/test_bt.sh
@@ -3,8 +3,6 @@
 
 exitcode=0
 
-ulimit -c 0
-
 export ASAN_OPTIONS="handle_segv=0:allow_user_segv_handler=1"
 "${builddir:-.}/src/test/test-bt-cl" backtraces || exit $?
 "${builddir:-.}/src/test/test-bt-cl" assert 2>&1 | "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/bt_test.py" || exitcode="$?"
diff --git a/src/test/test_bt_cl.c b/src/test/test_bt_cl.c
index 0c15a02ee4..5f9a88705c 100644
--- a/src/test/test_bt_cl.c
+++ b/src/test/test_bt_cl.c
@@ -1,9 +1,12 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #include <stdio.h>
 #include <stdlib.h>
+#ifdef HAVE_SYS_RESOURCE_H
+#include <sys/resource.h>
+#endif
 
 /* To prevent 'assert' from going away. */
 #undef TOR_COVERAGE
@@ -30,7 +33,7 @@ int a_tangled_web(int x) NOINLINE;
 int we_weave(int x) NOINLINE;
 
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
-DISABLE_GCC_WARNING(null-dereference)
+DISABLE_GCC_WARNING("-Wnull-dereference")
 #endif
 int
 crash(int x)
@@ -43,7 +46,7 @@ crash(int x)
     *(volatile int *)0 = 0;
 #endif /* defined(__clang_analyzer__) || defined(__COVERITY__) */
   } else if (crashtype == 1) {
-    tor_assert(1 == 0);
+    tor_assertf(1 == 0, "%d != %d", 1, 0);
   } else if (crashtype == -1) {
     ;
   }
@@ -52,7 +55,7 @@ crash(int x)
   return crashtype;
 }
 #ifdef HAVE_CFLAG_WNULL_DEREFERENCE
-ENABLE_GCC_WARNING(null-dereference)
+ENABLE_GCC_WARNING("-Wnull-dereference")
 #endif
 
 int
@@ -88,6 +91,11 @@ main(int argc, char **argv)
     return 1;
   }
 
+#ifdef HAVE_SYS_RESOURCE_H
+  struct rlimit rlim = { .rlim_cur = 0, .rlim_max = 0 };
+  setrlimit(RLIMIT_CORE, &rlim);
+#endif
+
 #if !(defined(HAVE_EXECINFO_H) && defined(HAVE_BACKTRACE) && \
    defined(HAVE_BACKTRACE_SYMBOLS_FD) && defined(HAVE_SIGACTION))
     puts("Backtrace reporting is not supported on this platform");
diff --git a/src/test/test_btrack.c b/src/test/test_btrack.c
new file mode 100644
index 0000000000..2b2f34fc23
--- /dev/null
+++ b/src/test/test_btrack.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
+
+static void
+send_state(const orconn_state_msg_t *msg_in)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_state_publish(msg);
+}
+
+static void
+send_status(const orconn_status_msg_t *msg_in)
+{
+  orconn_status_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  orconn_status_publish(msg);
+}
+
+static void
+send_chan(const ocirc_chan_msg_t *msg_in)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_btrack_launch(void *arg)
+{
+  orconn_state_msg_t conn;
+  ocirc_chan_msg_t circ;
+  memset(&conn, 0, sizeof(conn));
+  memset(&circ, 0, sizeof(circ));
+
+  (void)arg;
+  conn.gid = 1;
+  conn.chan = 1;
+  conn.proxy_type = PROXY_NONE;
+  conn.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 1;
+  circ.onehop = true;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=1 onehop=1");
+  expect_log_msg_containing("ORCONN BEST_ANY state -1->1 gid=1");
+  teardown_capture_of_logs();
+
+  conn.gid = 2;
+  conn.chan = 2;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&conn);
+  expect_log_msg_containing("ORCONN gid=2 chan=2 proxy_type=0 state=1");
+  expect_no_log_msg_containing("ORCONN BEST_");
+  teardown_capture_of_logs();
+
+  circ.chan = 2;
+  circ.onehop = false;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_chan(&circ);
+  expect_log_msg_containing("ORCONN LAUNCH chan=2 onehop=0");
+  expect_log_msg_containing("ORCONN BEST_AP state -1->1 gid=2");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+static void
+test_btrack_delete(void *arg)
+{
+  orconn_state_msg_t state;
+  orconn_status_msg_t status;
+  memset(&state, 0, sizeof(state));
+  memset(&status, 0, sizeof(status));
+
+  (void)arg;
+  state.gid = 1;
+  state.chan = 1;
+  state.proxy_type = PROXY_NONE;
+  state.state = OR_CONN_STATE_CONNECTING;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_state(&state);
+  expect_log_msg_containing("ORCONN gid=1 chan=1 proxy_type=0");
+  teardown_capture_of_logs();
+
+  status.gid = 1;
+  status.status = OR_CONN_EVENT_CLOSED;
+  status.reason = 0;
+
+  setup_full_capture_of_logs(LOG_DEBUG);
+  send_status(&status);
+  expect_log_msg_containing("ORCONN DELETE gid=1 status=3 reason=0");
+  teardown_capture_of_logs();
+
+ done:
+  ;
+}
+
+struct testcase_t btrack_tests[] = {
+  { "launch", test_btrack_launch, TT_FORK, &helper_pubsub_setup, NULL },
+  { "delete", test_btrack_delete, TT_FORK, &helper_pubsub_setup, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_buffers.c b/src/test/test_buffers.c
index 3e7364a5c8..fbaa628fd7 100644
--- a/src/test/test_buffers.c
+++ b/src/test/test_buffers.c
@@ -1,12 +1,12 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define BUFFERS_PRIVATE
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/tls/tortls.h"
 #include "lib/compress/compress.h"
diff --git a/src/test/test_bwmgt.c b/src/test/test_bwmgt.c
index 5a013aa268..117783cafc 100644
--- a/src/test/test_bwmgt.c
+++ b/src/test/test_bwmgt.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,18 +6,70 @@
  * \brief tests for bandwidth management / token bucket functions
  */
 
+#define CONFIG_PRIVATE
+#define CONNECTION_PRIVATE
+#define DIRAUTH_SYS_PRIVATE
 #define TOKEN_BUCKET_PRIVATE
 
 #include "core/or/or.h"
-#include "test/test.h"
 
+#include "app/config/config.h"
+#include "core/mainloop/connection.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/dircommon/directory.h"
+#include "feature/nodelist/microdesc.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/nodelist/routerlist.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "lib/evloop/token_bucket.h"
+#include "test/test.h"
+#include "test/test_helpers.h"
+
+#include "app/config/or_options_st.h"
+#include "core/or/connection_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
+#include "feature/nodelist/microdesc_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+#include "feature/nodelist/routerstatus_st.h"
 
 // an imaginary time, in timestamp units. Chosen so it will roll over.
 static const uint32_t START_TS = UINT32_MAX-10;
 static const int32_t KB = 1024;
 static const uint32_t GB = (UINT64_C(1) << 30);
 
+static or_options_t mock_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mock_options;
+}
+
+static networkstatus_t *dummy_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus(void)
+{
+  return dummy_ns;
+}
+
+static networkstatus_t *
+mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
+{
+  tor_assert(f == FLAV_MICRODESC);
+  return dummy_ns;
+}
+
+/* Number of address a single node_t can have. Default to the production
+ * value. This is to control the size of the bloom filter. */
+static int addr_per_node = 2;
+static int
+mock_get_estimated_address_per_node(void)
+{
+  return addr_per_node;
+}
+
 static void
 test_bwmgt_token_buf_init(void *arg)
 {
@@ -220,8 +272,167 @@ test_bwmgt_token_buf_helpers(void *arg)
   ;
 }
 
+static void
+test_bwmgt_dir_conn_global_write_low(void *arg)
+{
+  bool ret;
+  int addr_family;
+  connection_t *conn = NULL;
+  routerstatus_t *rs = NULL; microdesc_t *md = NULL; routerinfo_t *ri = NULL;
+  tor_addr_t relay_addr;
+  dirauth_options_t *dirauth_opts = NULL;
+
+  (void) arg;
+
+  memset(&mock_options, 0, sizeof(or_options_t));
+  MOCK(networkstatus_get_latest_consensus,
+       mock_networkstatus_get_latest_consensus);
+  MOCK(networkstatus_get_latest_consensus_by_flavor,
+       mock_networkstatus_get_latest_consensus_by_flavor);
+  MOCK(get_estimated_address_per_node,
+       mock_get_estimated_address_per_node);
+
+  /*
+   * The following is rather complex but that is what it takes to add a dummy
+   * consensus with a valid routerlist which will populate our node address
+   * set that we need to lookup to test the known relay code path.
+   *
+   * We MUST do that before we MOCK(get_options) else it is another world of
+   * complexity.
+   */
+
+  /* This will be the address of our relay. */
+  tor_addr_parse(&relay_addr, "1.2.3.4");
+
+  /* We'll now add a relay into our routerlist and see if we let it. */
+  dummy_ns = tor_malloc_zero(sizeof(*dummy_ns));
+  dummy_ns->flavor = FLAV_MICRODESC;
+  dummy_ns->routerstatus_list = smartlist_new();
+
+  md = tor_malloc_zero(sizeof(*md));
+  ri = tor_malloc_zero(sizeof(*ri));
+  rs = tor_malloc_zero(sizeof(*rs));
+  crypto_rand(rs->identity_digest, sizeof(rs->identity_digest));
+  crypto_rand(md->digest, sizeof(md->digest));
+  memcpy(rs->descriptor_digest, md->digest, DIGEST256_LEN);
+
+  /* Set IP address. */
+  rs->addr = tor_addr_to_ipv4h(&relay_addr);
+  ri->addr = rs->addr;
+  /* Add the rs to the consensus becoming a node_t. */
+  smartlist_add(dummy_ns->routerstatus_list, rs);
+
+  /* Add all configured authorities (hardcoded) before we set the consensus so
+   * the address set exists. */
+  ret = consider_adding_dir_servers(&mock_options, &mock_options);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* This will make the nodelist bloom filter very large
+   * (the_nodelist->node_addrs) so we will fail the contain test rarely. */
+  addr_per_node = 1024;
+
+  nodelist_set_consensus(dummy_ns);
+
+  dirauth_opts = tor_malloc_zero(sizeof(dirauth_options_t));
+  dirauth_opts->AuthDirRejectRequestsUnderLoad = 0;
+  dirauth_set_options(dirauth_opts);
+
+  /* Ok, now time to control which options we use. */
+  MOCK(get_options, mock_get_options);
+
+  /* Set ourselves as an authoritative dir. */
+  mock_options.AuthoritativeDir = 1;
+  mock_options.V3AuthoritativeDir = 1;
+  mock_options.UseDefaultFallbackDirs = 0;
+
+  /* This will set our global bucket to 1 byte and thus we will hit the
+   * banwdith limit in our test. */
+  mock_options.BandwidthRate = 1;
+  mock_options.BandwidthBurst = 1;
+
+  /* Else an IPv4 address screams. */
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Initialize the global buckets. */
+  connection_bucket_init();
+
+  /* The address "127.0.0.1" is set with this helper. */
+  conn = test_conn_get_connection(DIR_CONN_STATE_MIN_, CONN_TYPE_DIR,
+                                  DIR_PURPOSE_MIN_);
+  tt_assert(conn);
+
+  /* First try a non authority non relay IP thus a client but we are not
+   * configured to reject requests under load so we should get a false value
+   * that our limit is _not_ low. */
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Now, we will reject requests under load so try again a non authority non
+   * relay IP thus a client. We should get a warning that our limit is too
+   * low. */
+  dirauth_opts->AuthDirRejectRequestsUnderLoad = 1;
+
+  addr_family = tor_addr_parse(&conn->addr, "1.1.1.1");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Now, lets try with a connection address from moria1. It should always
+   * pass even though our limit is too low. */
+  addr_family = tor_addr_parse(&conn->addr, "128.31.0.39");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* IPv6 testing of gabelmoo. */
+  addr_family = tor_addr_parse(&conn->addr, "[2001:638:a000:4140::ffff:189]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a known relay address. It should pass. Possible due to
+   * our consensus setting above. */
+  memcpy(&conn->addr, &relay_addr, sizeof(tor_addr_t));
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Lets retry with a random IP that is not an authority nor a relay. */
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Finally, just make sure it still denies an IP if we are _not_ a v3
+   * directory authority. */
+  mock_options.V3AuthoritativeDir = 0;
+  addr_family = tor_addr_parse(&conn->addr, "1.2.3.4");
+  tt_int_op(addr_family, OP_EQ, AF_INET);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+  /* Random IPv6 should not be allowed. */
+  addr_family = tor_addr_parse(&conn->addr, "[CAFE::ACAB]");
+  tt_int_op(addr_family, OP_EQ, AF_INET6);
+  ret = connection_dir_is_global_write_low(conn, INT_MAX);
+  tt_int_op(ret, OP_EQ, 1);
+
+ done:
+  connection_free_minimal(conn);
+  routerstatus_free(rs); routerinfo_free(ri); microdesc_free(md);
+  smartlist_clear(dummy_ns->routerstatus_list);
+  networkstatus_vote_free(dummy_ns);
+
+  UNMOCK(get_estimated_address_per_node);
+  UNMOCK(networkstatus_get_latest_consensus);
+  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
+  UNMOCK(get_options);
+}
+
 #define BWMGT(name)                                          \
-  { #name, test_bwmgt_ ## name , 0, NULL, NULL }
+  { #name, test_bwmgt_ ## name , TT_FORK, NULL, NULL }
 
 struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_init),
@@ -229,5 +440,7 @@ struct testcase_t bwmgt_tests[] = {
   BWMGT(token_buf_dec),
   BWMGT(token_buf_refill),
   BWMGT(token_buf_helpers),
+
+  BWMGT(dir_conn_global_write_low),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_cell_formats.c b/src/test/test_cell_formats.c
index fc5367557d..f9ff101c98 100644
--- a/src/test/test_cell_formats.c
+++ b/src/test/test_cell_formats.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -713,16 +713,20 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op(cc->onionskin,OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
   tt_int_op(p2_cmd, OP_EQ, RELAY_COMMAND_EXTEND2);
-  /* We'll generate it minus the IPv6 address and minus the konami code */
-  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  /* We'll generate it minus the konami code */
+  tt_int_op(p2_len, OP_EQ, 89+99-34);
   test_memeq_hex(p2,
-                 /* Two items: one that same darn IP address. */
-                 "02000612F40001F0F1"
-                 /* The next is a digest : anthropomorphization */
-                 "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 /* Three items */
+                 "03"
+                 /* IPv4 address */
+                 "0006" "12F40001" "F0F1"
+                 /* The next is an RSA digest: anthropomorphization */
+                 "0214" "616e7468726f706f6d6f727068697a6174696f6e"
+                 /*IPv6 address */
+                 "0112" "20020000000000000000000000f0c51e" "1112"
                  /* Now the handshake prologue */
                  "01050063");
-  tt_mem_op(p2+1+8+22+4,OP_EQ, b, 99+20);
+  tt_mem_op(p2+1+8+22+20+4, OP_EQ, b, 99+20);
   tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
 
   /* Now let's add an ed25519 key to that extend2 cell. */
@@ -732,22 +736,31 @@ test_cfmt_extend_cells(void *arg)
   /* As before, since we aren't extending by ed25519. */
   get_options_mutable()->ExtendByEd25519ID = 0;
   tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
-  tt_int_op(p2_len, OP_EQ, 89+99-34-20);
+  tt_int_op(p2_len, OP_EQ, 89+99-34);
   test_memeq_hex(p2,
-                 "02000612F40001F0F1"
+                 "03"
+                 "000612F40001F0F1"
                  "0214616e7468726f706f6d6f727068697a6174696f6e"
+                 "011220020000000000000000000000f0c51e1112"
                  "01050063");
 
   /* Now try with the ed25519 ID. */
   get_options_mutable()->ExtendByEd25519ID = 1;
   tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
-  tt_int_op(p2_len, OP_EQ, 89+99-34-20 + 34);
+  tt_int_op(p2_len, OP_EQ, 89+99);
   test_memeq_hex(p2,
-                 "03000612F40001F0F1"
+                 /* Four items */
+                 "04"
+                 /* IPv4 address */
+                 "0006" "12F40001" "F0F1"
+                 /* The next is an RSA digest: anthropomorphization */
                  "0214616e7468726f706f6d6f727068697a6174696f6e"
-                 // ed digest follows:
+                 /* Then an ed public key: brownshoesdontmakeit/brownshoesd */
                  "0320" "62726f776e73686f6573646f6e746d616b656"
                         "9742f62726f776e73686f657364"
+                 /*IPv6 address */
+                 "0112" "20020000000000000000000000f0c51e" "1112"
+                 /* Now the handshake prologue */
                  "01050063");
   /* Can we parse that? Did the key come through right? */
   memset(&ec, 0, sizeof(ec));
@@ -756,6 +769,40 @@ test_cfmt_extend_cells(void *arg)
   tt_mem_op("brownshoesdontmakeit/brownshoesd", OP_EQ,
             ec.ed_pubkey.pubkey, 32);
 
+  /* Now try IPv6 without IPv4 */
+  memset(p, 0, sizeof(p));
+  memcpy(p, "\x02", 1);
+  memcpy(p+1, "\x02\x14" "anthropomorphization", 22);
+  memcpy(p+23, "\x01\x12" "xxxxxxxxxxxxxxxxYY", 20);
+  memcpy(p+43, "\xff\xff\x00\x20", 4);
+  tt_int_op(0, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
+                                         p, sizeof(p)));
+  tt_int_op(RELAY_COMMAND_EXTEND2, OP_EQ, ec.cell_type);
+  tt_assert(fast_mem_is_zero((const char *)&ec.orport_ipv4.addr,
+                             sizeof(tor_addr_t)));
+  tt_int_op(0, OP_EQ, ec.orport_ipv4.port);
+  tt_str_op("7878:7878:7878:7878:7878:7878:7878:7878",
+            OP_EQ, fmt_addr(&ec.orport_ipv6.addr));
+  tt_int_op(22873, OP_EQ, ec.orport_ipv6.port);
+  tt_assert(ed25519_public_key_is_zero(&ec.ed_pubkey));
+  tt_mem_op(ec.node_id,OP_EQ, "anthropomorphization", 20);
+  tt_int_op(cc->cell_type, OP_EQ, CELL_CREATE2);
+  tt_int_op(cc->handshake_type, OP_EQ, 0xffff);
+  tt_int_op(cc->handshake_len, OP_EQ, 32);
+  tt_int_op(0, OP_EQ, extend_cell_format(&p2_cmd, &p2_len, p2, &ec));
+  tt_int_op(p2_cmd, OP_EQ, RELAY_COMMAND_EXTEND2);
+  tt_int_op(p2_len, OP_EQ, 47+32);
+  test_memeq_hex(p2,
+                 /* Two items */
+                 "02"
+                 /* The next is an RSA digest: anthropomorphization */
+                 "0214" "616e7468726f706f6d6f727068697a6174696f6e"
+                 /*IPv6 address */
+                 "0112" "78787878787878787878787878787878" "5959"
+                 /* Now the handshake prologue */
+                 "ffff0020");
+  tt_int_op(0, OP_EQ, create_cell_format_relayed(&cell, cc));
+
   /* == Now try parsing some junk */
 
   /* Try a too-long handshake */
@@ -811,13 +858,6 @@ test_cfmt_extend_cells(void *arg)
   memcpy(p+48, "\xff\xff\x00\x20", 4);
   tt_int_op(-1, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
                                       p, sizeof(p)));
-  memset(p, 0, sizeof(p));
-  memcpy(p, "\x02", 1);
-  memcpy(p+1, "\x02\x14" "anarchoindividualist", 22);
-  memcpy(p+23, "\x01\x12" "xxxxxxxxxxxxxxxxYY", 18);
-  memcpy(p+41, "\xff\xff\x00\x20", 4);
-  tt_int_op(-1, OP_EQ, extend_cell_parse(&ec, RELAY_COMMAND_EXTEND2,
-                                      p, sizeof(p)));
 
   /* Running out of space in specifiers  */
   memset(p,0,sizeof(p));
diff --git a/src/test/test_cell_queue.c b/src/test/test_cell_queue.c
index 8fc1da031e..b778c07802 100644
--- a/src/test/test_cell_queue.c
+++ b/src/test/test_cell_queue.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
diff --git a/src/test/test_channel.c b/src/test/test_channel.c
index afb7db813c..83b69cc80b 100644
--- a/src/test/test_channel.c
+++ b/src/test/test_channel.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
-#define CHANNEL_PRIVATE_
+#define CHANNEL_OBJECT_PRIVATE
+#define CHANNEL_FILE_PRIVATE
 #include "core/or/or.h"
 #include "core/or/channel.h"
 /* For channel_note_destroy_not_pending */
@@ -34,8 +34,6 @@
 static int test_chan_accept_cells = 0;
 static int test_chan_fixed_cells_recved = 0;
 static cell_t * test_chan_last_seen_fixed_cell_ptr = NULL;
-static int test_chan_var_cells_recved = 0;
-static var_cell_t * test_chan_last_seen_var_cell_ptr = NULL;
 static int test_cells_written = 0;
 static int test_doesnt_want_writes_count = 0;
 static int test_dumpstats_calls = 0;
@@ -112,24 +110,6 @@ chan_test_dumpstats(channel_t *ch, int severity)
   return;
 }
 
-/*
- * Handle an incoming variable-size cell for unit tests
- */
-
-static void
-chan_test_var_cell_handler(channel_t *ch,
-                           var_cell_t *var_cell)
-{
-  tt_assert(ch);
-  tt_assert(var_cell);
-
-  test_chan_last_seen_var_cell_ptr = var_cell;
-  ++test_chan_var_cells_recved;
-
- done:
-  return;
-}
-
 static void
 chan_test_close(channel_t *ch)
 {
@@ -487,11 +467,8 @@ test_channel_dumpstats(void *arg)
 
   /* Receive path */
   channel_set_cell_handlers(ch,
-                            chan_test_cell_handler,
-                            chan_test_var_cell_handler);
+                            chan_test_cell_handler);
   tt_ptr_op(channel_get_cell_handler(ch), OP_EQ, chan_test_cell_handler);
-  tt_ptr_op(channel_get_var_cell_handler(ch), OP_EQ,
-            chan_test_var_cell_handler);
   cell = tor_malloc_zero(sizeof(*cell));
   old_count = test_chan_fixed_cells_recved;
   channel_process_cell(ch, cell);
@@ -593,7 +570,6 @@ test_channel_outbound_cell(void *arg)
   circuit_set_n_circid_chan(TO_CIRCUIT(circ), 42, chan);
   tt_int_op(channel_num_circuits(chan), OP_EQ, 1);
   /* Test the cmux state. */
-  tt_ptr_op(TO_CIRCUIT(circ)->n_mux, OP_EQ, chan->cmux);
   tt_int_op(circuitmux_is_circuit_attached(chan->cmux, TO_CIRCUIT(circ)),
             OP_EQ, 1);
 
@@ -718,7 +694,7 @@ test_channel_inbound_cell(void *arg)
 
   /* Setup incoming cell handlers. We don't care about var cell, the channel
    * layers is not handling those. */
-  channel_set_cell_handlers(chan, chan_test_cell_handler, NULL);
+  channel_set_cell_handlers(chan, chan_test_cell_handler);
   tt_ptr_op(chan->cell_handler, OP_EQ, chan_test_cell_handler);
   /* Now process the cell, we should see it. */
   old_count = test_chan_fixed_cells_recved;
@@ -1345,7 +1321,7 @@ test_channel_for_extend(void *arg)
   channel_t *ret_chan = NULL;
   char digest[DIGEST_LEN];
   ed25519_public_key_t ed_id;
-  tor_addr_t addr;
+  tor_addr_t ipv4_addr, ipv6_addr;
   const char *msg;
   int launch;
   time_t now = time(NULL);
@@ -1355,6 +1331,9 @@ test_channel_for_extend(void *arg)
   memset(digest, 'A', sizeof(digest));
   memset(&ed_id, 'B', sizeof(ed_id));
 
+  tor_addr_make_null(&ipv4_addr, AF_INET);
+  tor_addr_make_null(&ipv6_addr, AF_INET6);
+
   chan1 = new_fake_channel();
   tt_assert(chan1);
   /* Need to be registered to get added to the id map. */
@@ -1388,7 +1367,8 @@ test_channel_for_extend(void *arg)
   tt_ptr_op(channel_find_by_remote_identity(digest, &ed_id), OP_EQ, chan1);
 
   /* The expected result is chan2 because it is older than chan1. */
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1396,16 +1376,18 @@ test_channel_for_extend(void *arg)
 
   /* Switch that around from previous test. */
   chan2->timestamp_created = chan1->timestamp_created + 1;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan1);
   tt_int_op(launch, OP_EQ, 0);
   tt_str_op(msg, OP_EQ, "Connection is fine; using it.");
 
   /* Same creation time, num circuits will be used and they both have 0 so the
-   * channel 2 should be picked due to how channel_is_better() work. */
+   * channel 2 should be picked due to how channel_is_better() works. */
   chan2->timestamp_created = chan1->timestamp_created;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan1);
   tt_int_op(launch, OP_EQ, 0);
@@ -1416,7 +1398,8 @@ test_channel_for_extend(void *arg)
 
   /* Condemned the older channel. */
   chan1->state = CHANNEL_STATE_CLOSING;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1425,7 +1408,8 @@ test_channel_for_extend(void *arg)
 
   /* Make the older channel a client one. */
   channel_mark_client(chan1);
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1435,8 +1419,9 @@ test_channel_for_extend(void *arg)
   /* Non matching ed identity with valid digest. */
   ed25519_public_key_t dumb_ed_id;
   memset(&dumb_ed_id, 0, sizeof(dumb_ed_id));
-  ret_chan = channel_get_for_extend(digest, &dumb_ed_id, &addr, &msg,
-                                    &launch);
+  ret_chan = channel_get_for_extend(digest, &dumb_ed_id,
+                                    &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Not connected. Connecting.");
   tt_int_op(launch, OP_EQ, 1);
@@ -1445,7 +1430,8 @@ test_channel_for_extend(void *arg)
   test_chan_should_match_target = 1;
   chan1->state = CHANNEL_STATE_OPENING;
   chan2->state = CHANNEL_STATE_OPENING;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Connection in progress; waiting.");
   tt_int_op(launch, OP_EQ, 0);
@@ -1454,7 +1440,8 @@ test_channel_for_extend(void *arg)
 
   /* Mark channel 1 as bad for circuits. */
   channel_mark_bad_for_new_circs(chan1);
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(ret_chan);
   tt_ptr_op(ret_chan, OP_EQ, chan2);
   tt_int_op(launch, OP_EQ, 0);
@@ -1464,7 +1451,8 @@ test_channel_for_extend(void *arg)
   /* Mark both channels as unusable. */
   channel_mark_bad_for_new_circs(chan1);
   channel_mark_bad_for_new_circs(chan2);
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
                         " Launching a new one.");
@@ -1475,7 +1463,8 @@ test_channel_for_extend(void *arg)
   /* Non canonical channels. */
   test_chan_should_be_canonical = 0;
   test_chan_should_match_target = 0;
-  ret_chan = channel_get_for_extend(digest, &ed_id, &addr, &msg, &launch);
+  ret_chan = channel_get_for_extend(digest, &ed_id, &ipv4_addr, &ipv6_addr,
+                                    &msg, &launch);
   tt_assert(!ret_chan);
   tt_str_op(msg, OP_EQ, "Connections all too old, or too non-canonical. "
                         " Launching a new one.");
@@ -1539,6 +1528,10 @@ test_channel_listener(void *arg)
   channel_listener_dump_statistics(chan, LOG_INFO);
 
  done:
+  if (chan) {
+    channel_listener_unregister(chan);
+    tor_free(chan);
+  }
   channel_free_all();
 }
 
diff --git a/src/test/test_channelpadding.c b/src/test/test_channelpadding.c
index 5da2d81377..63a591583d 100644
--- a/src/test/test_channelpadding.c
+++ b/src/test/test_channelpadding.c
@@ -1,7 +1,7 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define MAINLOOP_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define TOR_TIMERS_PRIVATE
@@ -21,7 +21,7 @@
 #include "test/log_test_helpers.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/timers.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "core/or/cell_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -289,8 +289,6 @@ test_channelpadding_timers(void *arg)
   channel_t *chans[CHANNELS_TO_TEST];
   (void)arg;
 
-  tor_libevent_postfork();
-
   if (!connection_array)
     connection_array = smartlist_new();
 
@@ -393,7 +391,6 @@ test_channelpadding_killonehop(void *arg)
   channelpadding_decision_t decision;
   int64_t new_time;
   (void)arg;
-  tor_libevent_postfork();
 
   routerstatus_t *relay = tor_malloc_zero(sizeof(routerstatus_t));
   monotime_init();
@@ -502,8 +499,6 @@ test_channelpadding_consensus(void *arg)
   int64_t new_time;
   (void)arg;
 
-  tor_libevent_postfork();
-
   /*
    * Params tested:
    *   nf_pad_before_usage
@@ -898,8 +893,6 @@ test_channelpadding_decide_to_pad_channel(void *arg)
     connection_array = smartlist_new();
   (void)arg;
 
-  tor_libevent_postfork();
-
   monotime_init();
   monotime_enable_test_mocking();
   monotime_set_mock_time_nsec(1);
diff --git a/src/test/test_channeltls.c b/src/test/test_channeltls.c
index 10513e451d..f4f5cb447e 100644
--- a/src/test/test_channeltls.c
+++ b/src/test/test_channeltls.c
@@ -1,19 +1,20 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #include <math.h>
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #include "core/or/or.h"
 #include "lib/net/address.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
+#include "app/config/resolve_addr.h"
 /* For init/free stuff */
 #include "core/or/scheduler.h"
 #include "lib/tls/tortls.h"
diff --git a/src/test/test_checkdir.c b/src/test/test_checkdir.c
index 1df74c390a..186a55cc8c 100644
--- a/src/test/test_checkdir.c
+++ b/src/test/test_checkdir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_circuitbuild.c b/src/test/test_circuitbuild.c
index 538f20781f..03fd176ead 100644
--- a/src/test/test_circuitbuild.c
+++ b/src/test/test_circuitbuild.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITBUILD_PRIVATE
@@ -8,18 +8,32 @@
 #define ENTRYNODES_PRIVATE
 
 #include "core/or/or.h"
+
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+
+#define CONFIG_PRIVATE
 #include "app/config/config.h"
+
+#include "core/or/channel.h"
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
+#include "core/or/onion.h"
 
+#include "core/or/cell_st.h"
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/extend_info_st.h"
 #include "core/or/origin_circuit_st.h"
+#include "core/or/or_circuit_st.h"
 
 #include "feature/client/entrynodes.h"
+#include "feature/nodelist/nodelist.h"
+#include "feature/relay/circuitbuild_relay.h"
+#include "feature/relay/router.h"
+#include "feature/relay/routermode.h"
+
+#include "feature/nodelist/node_st.h"
 
 /* Dummy nodes smartlist for testing */
 static smartlist_t dummy_nodes;
@@ -27,11 +41,11 @@ static smartlist_t dummy_nodes;
 static extend_info_t dummy_ei;
 
 static int
-mock_count_acceptable_nodes(smartlist_t *nodes)
+mock_count_acceptable_nodes(const smartlist_t *nodes, int direct)
 {
   (void)nodes;
 
-  return DEFAULT_ROUTE_LEN + 1;
+  return direct ? 1 : DEFAULT_ROUTE_LEN + 1;
 }
 
 /* Test route lengths when the caller of new_route_len() doesn't
@@ -114,6 +128,14 @@ test_new_route_len_unhandled_exit(void *arg)
   int r;
 
   (void)arg;
+#ifdef ALL_BUGS_ARE_FATAL
+  /* Coverity (and maybe clang analyser) complain that the code following
+   * tt_skip() is unconditionally unreachable. */
+#if !defined(__COVERITY__) && !defined(__clang_analyzer__)
+  tt_skip();
+#endif
+#endif /* defined(ALL_BUGS_ARE_FATAL) */
+
   MOCK(count_acceptable_nodes, mock_count_acceptable_nodes);
 
   tor_capture_bugs_(1);
@@ -125,10 +147,10 @@ test_new_route_len_unhandled_exit(void *arg)
             "!(exit_ei && !known_purpose)");
   expect_single_log_msg_containing("Unhandled purpose");
   expect_single_log_msg_containing("with a chosen exit; assuming routelen");
-  teardown_capture_of_logs();
-  tor_end_capture_bugs_();
 
  done:
+  teardown_capture_of_logs();
+  tor_end_capture_bugs_();
   UNMOCK(count_acceptable_nodes);
 }
 
@@ -167,16 +189,1366 @@ test_upgrade_from_guard_wait(void *arg)
   tt_assert(!list);
 
  done:
+  smartlist_free(list);
   circuit_free(circ);
   entry_guard_free_(guard);
 }
 
+static int server = 0;
+static int
+mock_server_mode(const or_options_t *options)
+{
+  (void)options;
+  return server;
+}
+
+/* Test the different cases in circuit_extend_state_valid_helper(). */
+static void
+test_circuit_extend_state_valid(void *arg)
+{
+  (void)arg;
+  circuit_t *circ = tor_malloc_zero(sizeof(circuit_t));
+
+  server = 0;
+  MOCK(server_mode, mock_server_mode);
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+  /* Clients can't extend */
+  server = 0;
+  tt_int_op(circuit_extend_state_valid_helper(NULL), OP_EQ, -1);
+  expect_log_msg("Got an extend cell, but running as a client. Closing.\n");
+  mock_clean_saved_logs();
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* n_chan and n_hop are NULL, this should succeed */
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  /* But clients still can't extend */
+  server = 0;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, -1);
+  expect_log_msg("Got an extend cell, but running as a client. Closing.\n");
+  mock_clean_saved_logs();
+
+  /* n_chan must be NULL */
+  circ->n_chan = tor_malloc_zero(sizeof(channel_t));
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, -1);
+  expect_log_msg("n_chan already set. Bug/attack. Closing.\n");
+  mock_clean_saved_logs();
+  tor_free(circ->n_chan);
+
+  /* n_hop must be NULL */
+  circ->n_hop = tor_malloc_zero(sizeof(extend_info_t));
+  server = 1;
+  tt_int_op(circuit_extend_state_valid_helper(circ), OP_EQ, -1);
+  expect_log_msg("conn to next hop already launched. Bug/attack. Closing.\n");
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(server_mode);
+  server = 0;
+
+  tor_free(circ->n_chan);
+  tor_free(circ->n_hop);
+  tor_free(circ);
+}
+
+static node_t *mocked_node = NULL;
+static const node_t *
+mock_node_get_by_id(const char *identity_digest)
+{
+  (void)identity_digest;
+  return mocked_node;
+}
+
+static int mocked_supports_ed25519_link_authentication = 0;
+static int
+mock_node_supports_ed25519_link_authentication(const node_t *node,
+                                                int compatible_with_us)
+{
+  (void)node;
+  (void)compatible_with_us;
+  return mocked_supports_ed25519_link_authentication;
+}
+
+static ed25519_public_key_t * mocked_ed25519_id = NULL;
+static const ed25519_public_key_t *
+mock_node_get_ed25519_id(const node_t *node)
+{
+  (void)node;
+  return mocked_ed25519_id;
+}
+
+/* Test the different cases in circuit_extend_add_ed25519_helper(). */
+static void
+test_circuit_extend_add_ed25519(void *arg)
+{
+  (void)arg;
+  extend_cell_t *ec = tor_malloc_zero(sizeof(extend_cell_t));
+  extend_cell_t *old_ec = tor_malloc_zero(sizeof(extend_cell_t));
+  extend_cell_t *zero_ec = tor_malloc_zero(sizeof(extend_cell_t));
+
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  ed25519_public_key_t *fake_ed25519_id = NULL;
+  fake_ed25519_id = tor_malloc_zero(sizeof(ed25519_public_key_t));
+
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  MOCK(node_supports_ed25519_link_authentication,
+       mock_node_supports_ed25519_link_authentication);
+  MOCK(node_get_ed25519_id, mock_node_get_ed25519_id);
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* The extend cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_add_ed25519_helper(NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!ec))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* The node id must be non-zero */
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, -1);
+  expect_log_msg(
+    "Client asked me to extend without specifying an id_digest.\n");
+  /* And nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  mock_clean_saved_logs();
+
+  /* Fill in fake node_id, and try again */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* There's no node with that id, so the ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  mock_clean_saved_logs();
+
+  /* Provide 2 out of 3 of node, supports link auth, and ed_id.
+   * The ed_id should remain zeroed. */
+
+  /* Provide node and supports link auth */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  mocked_supports_ed25519_link_authentication = 1;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Provide supports link auth and ed id */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_supports_ed25519_link_authentication = 1;
+  memset(fake_ed25519_id, 0xEE, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Provide node and ed id */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  memset(fake_ed25519_id, 0xEE, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should still be zeroed */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, &zero_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  /* In fact, nothing should have changed */
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Now do the real lookup */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  mocked_supports_ed25519_link_authentication = 1;
+  memset(fake_ed25519_id, 0xEE, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should match */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, fake_ed25519_id, sizeof(ec->ed_pubkey));
+  /* Nothing else should have changed */
+  memcpy(&ec->ed_pubkey, &old_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+  /* Now do the real lookup, but with a zeroed ed id */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memcpy(old_ec, ec, sizeof(extend_cell_t));
+  /* Set up the fake variables */
+  mocked_node = fake_node;
+  mocked_supports_ed25519_link_authentication = 1;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+  mocked_ed25519_id = fake_ed25519_id;
+  /* Do the test */
+  tt_int_op(circuit_extend_add_ed25519_helper(ec), OP_EQ, 0);
+  /* The ed pubkey should match */
+  tt_mem_op(&ec->ed_pubkey, OP_EQ, fake_ed25519_id, sizeof(ec->ed_pubkey));
+  /* Nothing else should have changed */
+  memcpy(&ec->ed_pubkey, &old_ec->ed_pubkey, sizeof(ec->ed_pubkey));
+  tt_mem_op(ec, OP_EQ, old_ec, sizeof(extend_cell_t));
+  /* Cleanup */
+  mock_clean_saved_logs();
+  mocked_node = NULL;
+  mocked_supports_ed25519_link_authentication = 0;
+  mocked_ed25519_id = NULL;
+  memset(fake_ed25519_id, 0x00, sizeof(ed25519_public_key_t));
+
+ done:
+  UNMOCK(node_get_by_id);
+  UNMOCK(node_supports_ed25519_link_authentication);
+  UNMOCK(node_get_ed25519_id);
+
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  tor_free(ec);
+  tor_free(old_ec);
+  tor_free(zero_ec);
+
+  tor_free(fake_ed25519_id);
+  tor_free(fake_node);
+}
+
+static or_options_t *mocked_options = NULL;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mocked_options;
+}
+
+#define PUBLIC_IPV4   "1.2.3.4"
+#define INTERNAL_IPV4 "0.0.0.1"
+
+#define PUBLIC_IPV6   "1234::cdef"
+#define INTERNAL_IPV6 "::1"
+
+#define VALID_PORT    0x1234
+
+/* Test the different cases in circuit_extend_lspec_valid_helper(). */
+static void
+test_circuit_extend_lspec_valid(void *arg)
+{
+  (void)arg;
+  extend_cell_t *ec = tor_malloc_zero(sizeof(extend_cell_t));
+  channel_t *p_chan = tor_malloc_zero(sizeof(channel_t));
+  or_circuit_t *or_circ = tor_malloc_zero(sizeof(or_circuit_t));
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+
+  or_options_t *fake_options = options_new();
+  MOCK(get_options, mock_get_options);
+  mocked_options = fake_options;
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Extend cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(NULL, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!ec))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell and circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(NULL, NULL), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), we might not log any bugs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 2);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* IPv4 and IPv6 addr and port are all zero, this should fail */
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or unspecified address '[scrubbed]'.\n");
+  mock_clean_saved_logs();
+
+  /* Now ask for the actual address in the logs */
+  fake_options->SafeLogging_ = SAFELOG_SCRUB_NONE;
+
+  /* IPv4 port is 0, IPv6 addr and port are both zero, this should fail */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv4 address '1.2.3.4:0'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv4 addr is 0, IPv6 addr and port are both zero, this should fail */
+  ec->orport_ipv4.port = VALID_PORT;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv4 address '0.0.0.0:4660'.\n");
+  mock_clean_saved_logs();
+  ec->orport_ipv4.port = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv4 addr is internal, and port is valid.
+   * (IPv6 addr and port are both zero.)
+   * Result depends on ExtendAllowPrivateAddresses. */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv4 address '0.0.0.1'.\n");
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Now do the same tests, but for IPv6 */
+
+  /* IPv6 port is 0, IPv4 addr and port are both zero, this should fail */
+  tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv6 address '[1234::cdef]:0'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv6 addr is 0, IPv4 addr and port are both zero, this should fail */
+  ec->orport_ipv6.port = VALID_PORT;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or IPv6 address '[::]:4660'.\n");
+  mock_clean_saved_logs();
+  ec->orport_ipv4.port = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* IPv6 addr is internal, and port is valid.
+   * (IPv4 addr and port are both zero.)
+   * Result depends on ExtendAllowPrivateAddresses. */
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv6 address '[::1]'.\n");
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Both addresses are internal.
+   * Result depends on ExtendAllowPrivateAddresses. */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv4 address '0.0.0.1'.\n");
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv6 address '[::1]'.\n");
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* If we pass the private address check, but don't have the right
+   * OR circuit magic number, we trigger another bug */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 1;
+
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(circ->magic != 0x98ABC04Fu))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Fail again, but this time only set an IPv4 address. */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 1;
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), expect 0-1 bug logs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 1);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Now set the right magic */
+  or_circ->base_.magic = OR_CIRCUIT_MAGIC;
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* If we pass the OR circuit magic check, but don't have p_chan,
+   * we trigger another bug */
+  fake_options->ExtendAllowPrivateAddresses = 1;
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!p_chan))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  /* We can also pass the OR circuit magic check with a public address */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  fake_options->ExtendAllowPrivateAddresses = 0;
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), expect 0-1 bug logs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 1);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  tor_addr_make_null(&ec->orport_ipv4.addr, AF_INET);
+  ec->orport_ipv4.port = 0x0000;
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Now let's fake a p_chan and the addresses */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  or_circ->p_chan = p_chan;
+
+  /* This is a trivial failure: node_id and p_chan->identity_digest are both
+   * zeroed */
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend back to the previous hop.\n");
+  mock_clean_saved_logs();
+
+  /* Let's check with non-zero identities as well */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0xAA, sizeof(p_chan->identity_digest));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend back to the previous hop.\n");
+  mock_clean_saved_logs();
+
+  memset(ec->node_id, 0, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0, sizeof(p_chan->identity_digest));
+
+  /* Let's pass the node_id test */
+  memset(ec->node_id, 0xAA, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0xBB, sizeof(p_chan->identity_digest));
+
+  /* ed_pubkey is zero, and that's allowed, so we should succeed */
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  /* Now let's check that we warn, but succeed, when only one address is
+   * private */
+  tor_addr_parse(&ec->orport_ipv4.addr, INTERNAL_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv4 address '0.0.0.1'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Now with private IPv6 */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, INTERNAL_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+  fake_options->ExtendAllowPrivateAddresses = 0;
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  expect_log_msg("Client asked me to extend "
+                 "to a private IPv6 address '[::1]'.\n");
+  mock_clean_saved_logs();
+  tor_addr_port_make_null_ap(&ec->orport_ipv4, AF_INET);
+  tor_addr_port_make_null_ap(&ec->orport_ipv6, AF_INET6);
+
+  /* Now reset to public IPv4 and IPv6 */
+  tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+  ec->orport_ipv4.port = VALID_PORT;
+  tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+  ec->orport_ipv6.port = VALID_PORT;
+
+  /* Fail on matching non-zero identities */
+  memset(&ec->ed_pubkey, 0xEE, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0xEE, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, -1);
+  expect_log_msg("Client asked me to extend back to the previous hop "
+                 "(by Ed25519 ID).\n");
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Succeed on different, non-zero identities */
+  memset(&ec->ed_pubkey, 0xDD, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0xEE, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Succeed if the client knows the identity, but we don't */
+  memset(&ec->ed_pubkey, 0xDD, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0x00, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Succeed if we know the identity, but the client doesn't */
+  memset(&ec->ed_pubkey, 0x00, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0xEE, sizeof(p_chan->ed25519_identity));
+
+  tt_int_op(circuit_extend_lspec_valid_helper(ec, circ), OP_EQ, 0);
+  mock_clean_saved_logs();
+
+  memset(&ec->ed_pubkey, 0, sizeof(ec->ed_pubkey));
+  memset(&p_chan->ed25519_identity, 0, sizeof(p_chan->ed25519_identity));
+
+  /* Cleanup the node ids */
+  memset(ec->node_id, 0, sizeof(ec->node_id));
+  memset(p_chan->identity_digest, 0, sizeof(p_chan->identity_digest));
+
+  /* Cleanup the p_chan and the addresses */
+  tor_addr_make_null(&ec->orport_ipv4.addr, AF_UNSPEC);
+  ec->orport_ipv4.port = 0;
+  or_circ->p_chan = NULL;
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(get_options);
+  or_options_free(fake_options);
+  mocked_options = NULL;
+
+  tor_free(ec);
+  tor_free(or_circ);
+  tor_free(p_chan);
+}
+
+static bool can_extend_over_ipv6_result = false;
+static int mock_router_can_extend_over_ipv6_calls = 0;
+static bool
+mock_router_can_extend_over_ipv6(const or_options_t *options)
+{
+  (void)options;
+  mock_router_can_extend_over_ipv6_calls++;
+  return can_extend_over_ipv6_result;
+}
+
+/* Test the different cases in circuit_choose_ip_ap_for_extend(). */
+static void
+test_circuit_choose_ip_ap_for_extend(void *arg)
+{
+  (void)arg;
+  tor_addr_port_t ipv4_ap;
+  tor_addr_port_t ipv6_ap;
+
+  /* Set up valid addresses */
+  tor_addr_parse(&ipv4_ap.addr, PUBLIC_IPV4);
+  ipv4_ap.port = VALID_PORT;
+  tor_addr_parse(&ipv6_ap.addr, PUBLIC_IPV6);
+  ipv6_ap.port = VALID_PORT;
+
+  or_options_t *fake_options = options_new();
+  MOCK(get_options, mock_get_options);
+  mocked_options = fake_options;
+
+  MOCK(router_can_extend_over_ipv6,
+       mock_router_can_extend_over_ipv6);
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+
+  /* No valid addresses */
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, NULL), OP_EQ, NULL);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, NULL), OP_EQ, NULL);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  /* One valid address: IPv4 */
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(&ipv4_ap, NULL), OP_EQ, &ipv4_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(&ipv4_ap, NULL), OP_EQ, &ipv4_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  /* One valid address: IPv6 */
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, &ipv6_ap), OP_EQ, &ipv6_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(NULL, &ipv6_ap), OP_EQ, NULL);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  /* Two valid addresses */
+  const tor_addr_port_t *chosen_addr = NULL;
+
+  can_extend_over_ipv6_result = true;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  chosen_addr = circuit_choose_ip_ap_for_extend(&ipv4_ap, &ipv6_ap);
+  tt_assert(chosen_addr == &ipv4_ap || chosen_addr == &ipv6_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+  can_extend_over_ipv6_result = false;
+  mock_router_can_extend_over_ipv6_calls = 0;
+  tt_ptr_op(circuit_choose_ip_ap_for_extend(&ipv4_ap, &ipv6_ap),
+            OP_EQ, &ipv4_ap);
+  tt_int_op(mock_router_can_extend_over_ipv6_calls, OP_EQ, 1);
+
+ done:
+  UNMOCK(get_options);
+  or_options_free(fake_options);
+  mocked_options = NULL;
+
+  UNMOCK(router_can_extend_over_ipv6);
+
+  tor_free(fake_options);
+}
+
+static int mock_circuit_close_calls = 0;
+static void
+mock_circuit_mark_for_close_(circuit_t *circ, int reason,
+                             int line, const char *cfile)
+{
+  (void)circ;
+  (void)reason;
+  (void)line;
+  (void)cfile;
+  mock_circuit_close_calls++;
+}
+
+static int mock_channel_connect_calls = 0;
+static channel_t *mock_channel_connect_nchan = NULL;
+static channel_t *
+mock_channel_connect_for_circuit(const tor_addr_t *addr,
+                                 uint16_t port,
+                                 const char *id_digest,
+                                 const struct ed25519_public_key_t *ed_id)
+{
+  (void)addr;
+  (void)port;
+  (void)id_digest;
+  (void)ed_id;
+  mock_channel_connect_calls++;
+  return mock_channel_connect_nchan;
+}
+
+/* Test the different cases in circuit_open_connection_for_extend().
+ * Chooses different IP addresses depending on the first character in arg:
+ *  - 4: IPv4
+ *  - 6: IPv6
+ *  - d: IPv4 and IPv6 (dual-stack)
+ */
+static void
+test_circuit_open_connection_for_extend(void *arg)
+{
+  const char ip_version = ((const char *)arg)[0];
+  const bool use_ipv4 = (ip_version == '4' || ip_version == 'd');
+  const bool use_ipv6 = (ip_version == '6' || ip_version == 'd');
+  tor_assert(use_ipv4 || use_ipv6);
+
+  extend_cell_t *ec = tor_malloc_zero(sizeof(extend_cell_t));
+  circuit_t *circ = tor_malloc_zero(sizeof(circuit_t));
+  channel_t *fake_n_chan = tor_malloc_zero(sizeof(channel_t));
+
+  or_options_t *fake_options = options_new();
+  MOCK(get_options, mock_get_options);
+  mocked_options = fake_options;
+
+  MOCK(circuit_mark_for_close_, mock_circuit_mark_for_close_);
+  mock_circuit_close_calls = 0;
+  MOCK(channel_connect_for_circuit, mock_channel_connect_for_circuit);
+  mock_channel_connect_calls = 0;
+  mock_channel_connect_nchan = NULL;
+
+  MOCK(router_can_extend_over_ipv6,
+       mock_router_can_extend_over_ipv6);
+  can_extend_over_ipv6_result = true;
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(ec, NULL, 0);
+  /* We can't close a NULL circuit */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell must be non-NULL */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(NULL, circ, 0);
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!ec))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell and circuit must be non-NULL */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(NULL, NULL, 0);
+  /* We can't close a NULL circuit */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  /* Since we're using IF_BUG_ONCE(), we might not log any bugs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 2);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Fail, because neither address is valid */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  tor_capture_bugs_(1);
+  circuit_open_connection_for_extend(ec, circ, 0);
+  /* Close the circuit, don't connect */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_EQ, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_EQ, NULL);
+  tt_int_op(circ->state, OP_EQ, 0);
+  /* Cleanup */
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Set up valid addresses */
+  if (use_ipv4) {
+    tor_addr_parse(&ec->orport_ipv4.addr, PUBLIC_IPV4);
+    ec->orport_ipv4.port = VALID_PORT;
+  }
+  if (use_ipv6) {
+    tor_addr_parse(&ec->orport_ipv6.addr, PUBLIC_IPV6);
+    ec->orport_ipv6.port = VALID_PORT;
+  }
+
+  /* Succeed, but don't try to open a connection */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  circuit_open_connection_for_extend(ec, circ, 0);
+  /* If we haven't closed the circuit, that's success */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_NE, NULL);
+  tt_int_op(circ->state, OP_EQ, CIRCUIT_STATE_CHAN_WAIT);
+  /* Cleanup */
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  circ->state = 0;
+
+  /* Try to open a connection, but fail with a NULL n_chan */
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  circuit_open_connection_for_extend(ec, circ, 1);
+  /* Try to connect, but fail, and close the circuit */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 1);
+  expect_log_msg("Launching n_chan failed. Closing circuit.\n");
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_NE, NULL);
+  tt_int_op(circ->state, OP_EQ, CIRCUIT_STATE_CHAN_WAIT);
+  /* Cleanup */
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  circ->state = 0;
+
+  /* Try to open a connection, and succeed, because n_chan is not NULL */
+  mock_channel_connect_nchan = fake_n_chan;
+  mock_circuit_close_calls = 0;
+  mock_channel_connect_calls = 0;
+  circuit_open_connection_for_extend(ec, circ, 1);
+  /* Connection attempt succeeded, leaving the circuit open */
+  tt_int_op(mock_circuit_close_calls, OP_EQ, 0);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 1);
+  /* Check state */
+  tt_ptr_op(circ->n_hop, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan_create_cell, OP_NE, NULL);
+  tt_int_op(circ->state, OP_EQ, CIRCUIT_STATE_CHAN_WAIT);
+  /* Cleanup */
+  mock_clean_saved_logs();
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  circ->state = 0;
+  mock_channel_connect_nchan = NULL;
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(circuit_mark_for_close_);
+  mock_circuit_close_calls = 0;
+  UNMOCK(channel_connect_for_circuit);
+  mock_channel_connect_calls = 0;
+
+  UNMOCK(get_options);
+  or_options_free(fake_options);
+  mocked_options = NULL;
+
+  UNMOCK(router_can_extend_over_ipv6);
+
+  tor_free(ec);
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  tor_free(circ);
+  tor_free(fake_n_chan);
+}
+
+/* Guaranteed to be initialised to zero. */
+static extend_cell_t mock_extend_cell_parse_cell_out;
+static int mock_extend_cell_parse_result = 0;
+static int mock_extend_cell_parse_calls = 0;
+
+static int
+mock_extend_cell_parse(extend_cell_t *cell_out,
+                       const uint8_t command,
+                       const uint8_t *payload_in,
+                       size_t payload_len)
+{
+  (void)command;
+  (void)payload_in;
+  (void)payload_len;
+
+  mock_extend_cell_parse_calls++;
+  memcpy(cell_out, &mock_extend_cell_parse_cell_out,
+         sizeof(extend_cell_t));
+  return mock_extend_cell_parse_result;
+}
+
+static int mock_channel_get_for_extend_calls = 0;
+static int mock_channel_get_for_extend_launch_out = 0;
+static channel_t *mock_channel_get_for_extend_nchan = NULL;
+static channel_t *
+mock_channel_get_for_extend(const char *rsa_id_digest,
+                            const ed25519_public_key_t *ed_id,
+                            const tor_addr_t *target_ipv4_addr,
+                            const tor_addr_t *target_ipv6_addr,
+                            const char **msg_out,
+                            int *launch_out)
+{
+  (void)rsa_id_digest;
+  (void)ed_id;
+  (void)target_ipv4_addr;
+  (void)target_ipv6_addr;
+
+  /* channel_get_for_extend() requires non-NULL arguments */
+  tt_ptr_op(msg_out, OP_NE, NULL);
+  tt_ptr_op(launch_out, OP_NE, NULL);
+
+  mock_channel_get_for_extend_calls++;
+  *msg_out = NULL;
+  *launch_out = mock_channel_get_for_extend_launch_out;
+  return mock_channel_get_for_extend_nchan;
+
+ done:
+  return NULL;
+}
+
+static const char *
+mock_channel_get_canonical_remote_descr(channel_t *chan)
+{
+  (void)chan;
+  return "mock_channel_get_canonical_remote_descr()";
+}
+
+static int mock_circuit_deliver_create_cell_calls = 0;
+static int mock_circuit_deliver_create_cell_result = 0;
+static int
+mock_circuit_deliver_create_cell(circuit_t *circ,
+                                 const struct create_cell_t *create_cell,
+                                 int relayed)
+{
+  (void)create_cell;
+
+  /* circuit_deliver_create_cell() requires non-NULL arguments,
+   * but we only check circ and circ->n_chan here. */
+  tt_ptr_op(circ, OP_NE, NULL);
+  tt_ptr_op(circ->n_chan, OP_NE, NULL);
+
+  /* We should only ever get relayed cells from extends */
+  tt_int_op(relayed, OP_EQ, 1);
+
+  mock_circuit_deliver_create_cell_calls++;
+  return mock_circuit_deliver_create_cell_result;
+
+ done:
+  return -1;
+}
+
+/* Test the different cases in circuit_extend(). */
+static void
+test_circuit_extend(void *arg)
+{
+  (void)arg;
+  cell_t *cell = tor_malloc_zero(sizeof(cell_t));
+  channel_t *p_chan = tor_malloc_zero(sizeof(channel_t));
+  or_circuit_t *or_circ = tor_malloc_zero(sizeof(or_circuit_t));
+  circuit_t *circ = TO_CIRCUIT(or_circ);
+  channel_t *fake_n_chan = tor_malloc_zero(sizeof(channel_t));
+
+  server = 0;
+  MOCK(server_mode, mock_server_mode);
+
+  /* Mock a debug function, but otherwise ignore it */
+  MOCK(channel_get_canonical_remote_descr,
+       mock_channel_get_canonical_remote_descr);
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(cell, NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(NULL, circ), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!cell))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Extend cell and circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(NULL, NULL), OP_EQ, -1);
+  /* Since we're using IF_BUG_ONCE(), we might not log any bugs */
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_GE, 0);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_LE, 2);
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Clients can't extend */
+  server = 0;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  expect_log_msg("Got an extend cell, but running as a client. Closing.\n");
+  mock_clean_saved_logs();
+
+  /* But servers can. Unpack the cell, but fail parsing. */
+  server = 1;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  expect_log_msg("Can't parse extend cell. Closing circuit.\n");
+  mock_clean_saved_logs();
+
+  /* Now mock parsing */
+  MOCK(extend_cell_parse, mock_extend_cell_parse);
+
+  /* And make parsing succeed, but fail on adding ed25519 */
+  memset(&mock_extend_cell_parse_cell_out, 0,
+             sizeof(mock_extend_cell_parse_cell_out));
+  mock_extend_cell_parse_result = 0;
+  mock_extend_cell_parse_calls = 0;
+
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  expect_log_msg(
+    "Client asked me to extend without specifying an id_digest.\n");
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+
+  /* Now add a node_id. Fail the lspec check because IPv4 and port are zero. */
+  memset(&mock_extend_cell_parse_cell_out.node_id, 0xAA,
+             sizeof(mock_extend_cell_parse_cell_out.node_id));
+
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  expect_log_msg("Client asked me to extend to a zero destination port "
+                 "or unspecified address '[scrubbed]'.\n");
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+
+  /* Now add a valid IPv4 and port. Fail the OR circuit magic check. */
+  tor_addr_parse(&mock_extend_cell_parse_cell_out.orport_ipv4.addr,
+                 PUBLIC_IPV4);
+  mock_extend_cell_parse_cell_out.orport_ipv4.port = VALID_PORT;
+
+#ifndef ALL_BUGS_ARE_FATAL
+  tor_capture_bugs_(1);
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(circ->magic != 0x98ABC04Fu))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Now add the right magic and a p_chan. */
+  or_circ->base_.magic = OR_CIRCUIT_MAGIC;
+  or_circ->p_chan = p_chan;
+
+  /* Mock channel_get_for_extend(), so it doesn't crash. */
+  mock_channel_get_for_extend_calls = 0;
+  MOCK(channel_get_for_extend, mock_channel_get_for_extend);
+
+  /* Test circuit not established, but don't launch another one */
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = NULL;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, 0);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+
+  /* cleanup */
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+  /* Mock channel_connect_for_circuit(), so we don't crash */
+  mock_channel_connect_calls = 0;
+  MOCK(channel_connect_for_circuit, mock_channel_connect_for_circuit);
+
+  /* Test circuit not established, and successful launch of a channel */
+  mock_channel_get_for_extend_launch_out = 1;
+  mock_channel_get_for_extend_nchan = NULL;
+  mock_channel_connect_nchan = fake_n_chan;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, 0);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 1);
+
+  /* cleanup */
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_connect_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+  /* Mock circuit_deliver_create_cell(), so it doesn't crash */
+  mock_circuit_deliver_create_cell_calls = 0;
+  MOCK(circuit_deliver_create_cell, mock_circuit_deliver_create_cell);
+
+  /* Test circuit established, re-using channel, successful delivery */
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = fake_n_chan;
+  mock_channel_connect_nchan = NULL;
+  mock_circuit_deliver_create_cell_result = 0;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, 0);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(mock_circuit_deliver_create_cell_calls, OP_EQ, 1);
+  tt_ptr_op(circ->n_chan, OP_EQ, fake_n_chan);
+
+  /* cleanup */
+  circ->n_chan = NULL;
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_connect_calls = 0;
+  mock_circuit_deliver_create_cell_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+  /* Test circuit established, re-using channel, failed delivery */
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = fake_n_chan;
+  mock_channel_connect_nchan = NULL;
+  mock_circuit_deliver_create_cell_result = -1;
+  tt_int_op(circuit_extend(cell, circ), OP_EQ, -1);
+  tt_int_op(mock_extend_cell_parse_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_get_for_extend_calls, OP_EQ, 1);
+  tt_int_op(mock_channel_connect_calls, OP_EQ, 0);
+  tt_int_op(mock_circuit_deliver_create_cell_calls, OP_EQ, 1);
+  tt_ptr_op(circ->n_chan, OP_EQ, fake_n_chan);
+
+  /* cleanup */
+  circ->n_chan = NULL;
+  mock_clean_saved_logs();
+  mock_extend_cell_parse_calls = 0;
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_connect_calls = 0;
+  mock_circuit_deliver_create_cell_calls = 0;
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  UNMOCK(server_mode);
+  server = 0;
+
+  UNMOCK(channel_get_canonical_remote_descr);
+
+  UNMOCK(extend_cell_parse);
+  memset(&mock_extend_cell_parse_cell_out, 0,
+         sizeof(mock_extend_cell_parse_cell_out));
+  mock_extend_cell_parse_result = 0;
+  mock_extend_cell_parse_calls = 0;
+
+  UNMOCK(channel_get_for_extend);
+  mock_channel_get_for_extend_calls = 0;
+  mock_channel_get_for_extend_launch_out = 0;
+  mock_channel_get_for_extend_nchan = NULL;
+
+  UNMOCK(channel_connect_for_circuit);
+  mock_channel_connect_calls = 0;
+  mock_channel_connect_nchan = NULL;
+
+  UNMOCK(circuit_deliver_create_cell);
+  mock_circuit_deliver_create_cell_calls = 0;
+  mock_circuit_deliver_create_cell_result = 0;
+
+  tor_free(cell);
+  /* circ and or_circ are the same object */
+  tor_free(circ->n_hop);
+  tor_free(circ->n_chan_create_cell);
+  tor_free(or_circ);
+  tor_free(p_chan);
+  tor_free(fake_n_chan);
+}
+
+/* Test the different cases in onionskin_answer(). */
+static void
+test_onionskin_answer(void *arg)
+{
+  (void)arg;
+  created_cell_t *created_cell = tor_malloc_zero(sizeof(created_cell_t));
+  or_circuit_t *or_circ = tor_malloc_zero(sizeof(or_circuit_t));
+  char keys[CPATH_KEY_MATERIAL_LEN] = {0};
+  uint8_t rend_circ_nonce[DIGEST_LEN] = {0};
+
+  setup_full_capture_of_logs(LOG_INFO);
+
+#ifndef ALL_BUGS_ARE_FATAL
+  /* Circuit must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(NULL, created_cell,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!circ))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Created cell must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(or_circ, NULL,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!created_cell))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* Keys must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(or_circ, created_cell,
+                             NULL, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!keys))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+
+  /* The rend circuit nonce must be non-NULL */
+  tor_capture_bugs_(1);
+  tt_int_op(onionskin_answer(or_circ, created_cell,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             NULL), OP_EQ, -1);
+  tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
+  tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
+            "!(ASSERT_PREDICT_UNLIKELY_(!rend_circ_nonce))");
+  tor_end_capture_bugs_();
+  mock_clean_saved_logs();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
+  /* Also, the keys length must be CPATH_KEY_MATERIAL_LEN, but we can't catch
+   * asserts in unit tests. */
+
+  /* Fail when formatting the created cell */
+  tt_int_op(onionskin_answer(or_circ, created_cell,
+                             keys, CPATH_KEY_MATERIAL_LEN,
+                             rend_circ_nonce), OP_EQ, -1);
+  expect_log_msg("couldn't format created cell (type=0, len=0).\n");
+  mock_clean_saved_logs();
+
+  /* TODO: test the rest of onionskin_answer(), see #33860 */
+  /* TODO: mock created_cell_format for the next test  */
+
+ done:
+  tor_end_capture_bugs_();
+  teardown_capture_of_logs();
+
+  tor_free(created_cell);
+  tor_free(or_circ);
+}
+
+#define TEST(name, flags, setup, cleanup) \
+  { #name, test_ ## name, flags, setup, cleanup }
+
+#define TEST_NEW_ROUTE_LEN(name, flags) \
+  { #name, test_new_route_len_ ## name, flags, NULL, NULL }
+
+#define TEST_CIRCUIT(name, flags) \
+  { #name, test_circuit_ ## name, flags, NULL, NULL }
+
+#ifndef COCCI
+#define TEST_CIRCUIT_PASSTHROUGH(name, flags, arg) \
+  { #name "/" arg, test_circuit_ ## name, flags, \
+    &passthrough_setup, (void *)(arg) }
+#endif
+
 struct testcase_t circuitbuild_tests[] = {
-  { "noexit", test_new_route_len_noexit, 0, NULL, NULL },
-  { "safe_exit", test_new_route_len_safe_exit, 0, NULL, NULL },
-  { "unsafe_exit", test_new_route_len_unsafe_exit, 0, NULL, NULL },
-  { "unhandled_exit", test_new_route_len_unhandled_exit, 0, NULL, NULL },
-  { "upgrade_from_guard_wait", test_upgrade_from_guard_wait, TT_FORK,
-    NULL, NULL },
+  TEST_NEW_ROUTE_LEN(noexit, 0),
+  TEST_NEW_ROUTE_LEN(safe_exit, 0),
+  TEST_NEW_ROUTE_LEN(unsafe_exit, 0),
+  TEST_NEW_ROUTE_LEN(unhandled_exit, 0),
+
+  TEST(upgrade_from_guard_wait, TT_FORK, &helper_pubsub_setup, NULL),
+
+  TEST_CIRCUIT(extend_state_valid, TT_FORK),
+  TEST_CIRCUIT(extend_add_ed25519, TT_FORK),
+  TEST_CIRCUIT(extend_lspec_valid, TT_FORK),
+  TEST_CIRCUIT(choose_ip_ap_for_extend, 0),
+  TEST_CIRCUIT_PASSTHROUGH(open_connection_for_extend, TT_FORK, "4"),
+  TEST_CIRCUIT_PASSTHROUGH(open_connection_for_extend, TT_FORK, "6"),
+  TEST_CIRCUIT_PASSTHROUGH(open_connection_for_extend, TT_FORK, "dual-stack"),
+  TEST_CIRCUIT(extend, TT_FORK),
+
+  TEST(onionskin_answer, TT_FORK, NULL, NULL),
+
   END_OF_TESTCASES
 };
diff --git a/src/test/test_circuitlist.c b/src/test/test_circuitlist.c
index 5cebdbeda0..63c4418f29 100644
--- a/src/test/test_circuitlist.c
+++ b/src/test/test_circuitlist.c
@@ -1,7 +1,7 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define HS_CIRCUITMAP_PRIVATE
@@ -136,7 +136,7 @@ test_clist_maps(void *arg)
   channel_note_destroy_pending(ch2, 200);
   channel_note_destroy_pending(ch2, 205);
   channel_note_destroy_pending(ch1, 100);
-  tt_assert(circuit_id_in_use_on_channel(205, ch2))
+  tt_assert(circuit_id_in_use_on_channel(205, ch2));
   tt_assert(circuit_id_in_use_on_channel(200, ch2));
   tt_assert(circuit_id_in_use_on_channel(100, ch1));
 
diff --git a/src/test/test_circuitmux.c b/src/test/test_circuitmux.c
index a2b3e62fe8..2c11d107d0 100644
--- a/src/test/test_circuitmux.c
+++ b/src/test/test_circuitmux.c
@@ -1,33 +1,27 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITMUX_PRIVATE
 #define CIRCUITMUX_EWMA_PRIVATE
 #define RELAY_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/circuitmux.h"
 #include "core/or/circuitmux_ewma.h"
+#include "core/or/destroy_cell_queue_st.h"
 #include "core/or/relay.h"
 #include "core/or/scheduler.h"
-#include "test/test.h"
 
-#include "core/or/destroy_cell_queue_st.h"
+#include "test/fakechans.h"
+#include "test/fakecircs.h"
+#include "test/test.h"
 
 #include <math.h>
 
-/* XXXX duplicated function from test_circuitlist.c */
-static channel_t *
-new_fake_channel(void)
-{
-  channel_t *chan = tor_malloc_zero(sizeof(channel_t));
-  channel_init(chan);
-  return chan;
-}
-
 static int
-has_queued_writes(channel_t *c)
+mock_has_queued_writes_true(channel_t *c)
 {
   (void) c;
   return 1;
@@ -44,16 +38,14 @@ test_cmux_destroy_cell_queue(void *arg)
   packed_cell_t *pc = NULL;
   destroy_cell_t *dc = NULL;
 
-  scheduler_init();
+  MOCK(scheduler_release_channel, scheduler_release_channel_mock);
 
   (void) arg;
 
-  cmux = circuitmux_alloc();
-  tt_assert(cmux);
   ch = new_fake_channel();
-  circuitmux_set_policy(cmux, &ewma_policy);
-  ch->has_queued_writes = has_queued_writes;
+  ch->has_queued_writes = mock_has_queued_writes_true;
   ch->wide_circ_ids = 1;
+  cmux = ch->cmux;
 
   circ = circuitmux_get_first_active_circuit(cmux, &cq);
   tt_ptr_op(circ, OP_EQ, NULL);
@@ -78,10 +70,11 @@ test_cmux_destroy_cell_queue(void *arg)
   tt_int_op(circuitmux_num_cells(cmux), OP_EQ, 2);
 
  done:
-  circuitmux_free(cmux);
-  channel_free(ch);
+  free_fake_channel(ch);
   packed_cell_free(pc);
   tor_free(dc);
+
+  UNMOCK(scheduler_release_channel);
 }
 
 static void
@@ -125,9 +118,363 @@ test_cmux_compute_ticks(void *arg)
   ;
 }
 
+static void
+test_cmux_allocate(void *arg)
+{
+  circuitmux_t *cmux = NULL;
+
+  (void) arg;
+
+  cmux = circuitmux_alloc();
+  tt_assert(cmux);
+  tt_assert(cmux->chanid_circid_map);
+  tt_int_op(HT_SIZE(cmux->chanid_circid_map), OP_EQ, 0);
+  tt_uint_op(cmux->n_circuits, OP_EQ, 0);
+  tt_uint_op(cmux->n_active_circuits, OP_EQ, 0);
+  tt_uint_op(cmux->n_cells, OP_EQ, 0);
+  tt_uint_op(cmux->last_cell_was_destroy, OP_EQ, 0);
+  tt_i64_op(cmux->destroy_ctr, OP_EQ, 0);
+  tt_ptr_op(cmux->policy, OP_EQ, NULL);
+  tt_ptr_op(cmux->policy_data, OP_EQ, NULL);
+
+  tt_assert(TOR_SIMPLEQ_EMPTY(&cmux->destroy_cell_queue.head));
+
+ done:
+  circuitmux_free(cmux);
+}
+
+static void
+test_cmux_attach_circuit(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+  cell_direction_t cdir;
+  unsigned int n_cells;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* While assigning a new circuit IDs, the circuitmux_attach_circuit() is
+   * called for a new channel on the circuit. This means, we should now have
+   * the created circuit attached on both the pchan and nchan cmux. */
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 1);
+
+  /* There should be _no_ active circuit due to no queued cells. */
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_active_circuits(nchan->cmux), OP_EQ, 0);
+
+  /* Circuit should not be active on the cmux. */
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(nchan->cmux, circ), OP_EQ, 0);
+
+  /* Not active so no cells. */
+  n_cells = circuitmux_num_cells_for_circuit(pchan->cmux, circ);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  n_cells = circuitmux_num_cells(pchan->cmux);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  n_cells = circuitmux_num_cells_for_circuit(nchan->cmux, circ);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  n_cells = circuitmux_num_cells(nchan->cmux);
+  tt_uint_op(n_cells, OP_EQ, 0);
+
+  /* So it should be attached :) */
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 1);
+
+  /* Query the chanid<->circid map in the cmux subsytem with what we just
+   * created and validate the cell direction. */
+  cdir = circuitmux_attached_circuit_direction(pchan->cmux, circ);
+  tt_int_op(cdir, OP_EQ, CELL_DIRECTION_IN);
+  cdir = circuitmux_attached_circuit_direction(nchan->cmux, circ);
+  tt_int_op(cdir, OP_EQ, CELL_DIRECTION_OUT);
+
+  /*
+   * We'll activate->deactivate->activate to test all code paths of
+   * circuitmux_set_num_cells().
+   */
+
+  /* Activate circuit. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+
+  /* Deactivate. */
+  circuitmux_clear_num_cells(pchan->cmux, circ);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 0);
+
+  /* Re-activate. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+
+  /* Once re-attached, it should become inactive because the circuit has no
+   * cells while the chanid<->circid object has some. The attach code will
+   * reset the count on the cmux for that circuit:
+   *
+   * if (chanid_circid_muxinfo_t->muxinfo.cell_count > 0 && cell_count == 0) {
+   */
+  circuitmux_attach_circuit(pchan->cmux, circ, CELL_DIRECTION_IN);
+  n_cells = circuitmux_num_cells_for_circuit(pchan->cmux, circ);
+  tt_uint_op(n_cells, OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 0);
+
+  /* Lets queue a cell on the circuit now so it becomes active when
+   * re-attaching:
+   *
+   * else if (chanid_circid_muxinfo_t->muxinfo.cell_count == 0 &&
+   *          cell_count > 0) {
+   */
+  orcirc->p_chan_cells.n = 1;
+  circuitmux_attach_circuit(pchan->cmux, circ, CELL_DIRECTION_IN);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_detach_circuit(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* While assigning a new circuit IDs, the circuitmux_attach_circuit() is
+   * called for a new channel on the circuit. This means, we should now have
+   * the created circuit attached on both the pchan and nchan cmux. */
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 1);
+
+  /* Now, detach the circuit from pchan and then nchan. */
+  circuitmux_detach_circuit(pchan->cmux, circ);
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 0);
+  circuitmux_detach_circuit(nchan->cmux, circ);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_detach_all_circuits(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+  smartlist_t *detached_out = smartlist_new();
+
+  (void) arg;
+
+  /* Channels need to be registered in order for the detach all circuit
+   * function to find them. */
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  channel_register(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+  channel_register(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* Just make sure it is attached. */
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 1);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 1);
+
+  /* Queue some cells so we can test if the circuit becomes inactive on the
+   * cmux after the mass detach. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  circuitmux_set_num_cells(nchan->cmux, circ, 4);
+
+  /* Detach all on pchan and then nchan. */
+  circuitmux_detach_all_circuits(pchan->cmux, detached_out);
+  tt_uint_op(circuitmux_num_circuits(pchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(smartlist_len(detached_out), OP_EQ, 1);
+  circuitmux_detach_all_circuits(nchan->cmux, NULL);
+  tt_uint_op(circuitmux_num_circuits(nchan->cmux), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_attached(nchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(nchan->cmux, circ), OP_EQ, 0);
+
+ done:
+  smartlist_free(detached_out);
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_policy(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  channel_register(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+  channel_register(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* Confirm we have the EWMA policy by default for new channels. */
+  tt_ptr_op(circuitmux_get_policy(pchan->cmux), OP_EQ, &ewma_policy);
+  tt_ptr_op(circuitmux_get_policy(nchan->cmux), OP_EQ, &ewma_policy);
+
+  /* Putting cell on the cmux means will make the notify policy code path to
+   * trigger. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+
+  /* Clear it out. */
+  circuitmux_clear_policy(pchan->cmux);
+
+  /* Set back the EWMA policy. */
+  circuitmux_set_policy(pchan->cmux, &ewma_policy);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void
+test_cmux_xmit_cell(void *arg)
+{
+  circuit_t *circ = NULL;
+  or_circuit_t *orcirc = NULL;
+  channel_t *pchan = NULL, *nchan = NULL;
+
+  (void) arg;
+
+  pchan = new_fake_channel();
+  tt_assert(pchan);
+  nchan = new_fake_channel();
+  tt_assert(nchan);
+
+  orcirc = new_fake_orcirc(nchan, pchan);
+  tt_assert(orcirc);
+  circ = TO_CIRCUIT(orcirc);
+
+  /* Queue 4 cells on the circuit. */
+  circuitmux_set_num_cells(pchan->cmux, circ, 4);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 4);
+  tt_uint_op(circuitmux_num_cells(pchan->cmux), OP_EQ, 4);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 1);
+
+  /* Emit the first cell. Circuit should still be active. */
+  circuitmux_notify_xmit_cells(pchan->cmux, circ, 1);
+  tt_uint_op(circuitmux_num_cells(pchan->cmux), OP_EQ, 3);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 3);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 1);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 1);
+
+  /* Emit the last 3 cells. Circuit should become inactive. */
+  circuitmux_notify_xmit_cells(pchan->cmux, circ, 3);
+  tt_uint_op(circuitmux_num_cells(pchan->cmux), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_cells_for_circuit(pchan->cmux, circ), OP_EQ, 0);
+  tt_int_op(circuitmux_is_circuit_active(pchan->cmux, circ), OP_EQ, 0);
+  tt_uint_op(circuitmux_num_active_circuits(pchan->cmux), OP_EQ, 0);
+
+  /* Queue a DESTROY cell. */
+  pchan->has_queued_writes = mock_has_queued_writes_true;
+  circuitmux_append_destroy_cell(pchan, pchan->cmux, orcirc->p_circ_id, 0);
+  tt_i64_op(pchan->cmux->destroy_ctr, OP_EQ, 1);
+  tt_int_op(pchan->cmux->destroy_cell_queue.n, OP_EQ, 1);
+  tt_i64_op(circuitmux_count_queued_destroy_cells(pchan, pchan->cmux),
+            OP_EQ, 1);
+
+  /* Emit the DESTROY cell. */
+  circuitmux_notify_xmit_destroy(pchan->cmux);
+  tt_i64_op(pchan->cmux->destroy_ctr, OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(orcirc);
+  free_fake_channel(pchan);
+  free_fake_channel(nchan);
+}
+
+static void *
+cmux_setup_test(const struct testcase_t *tc)
+{
+  static int whatever;
+
+  (void) tc;
+
+  cell_ewma_initialize_ticks();
+  return &whatever;
+}
+
+static int
+cmux_cleanup_test(const struct testcase_t *tc, void *ptr)
+{
+  (void) tc;
+  (void) ptr;
+
+  circuitmux_ewma_free_all();
+
+  return 1;
+}
+
+static struct testcase_setup_t cmux_test_setup = {
+  .setup_fn = cmux_setup_test,
+  .cleanup_fn = cmux_cleanup_test,
+};
+
+#define TEST_CMUX(name) \
+  { #name, test_cmux_##name, TT_FORK, &cmux_test_setup, NULL }
+
 struct testcase_t circuitmux_tests[] = {
-  { "destroy_cell_queue", test_cmux_destroy_cell_queue, TT_FORK, NULL, NULL },
-  { "compute_ticks", test_cmux_compute_ticks, TT_FORK, NULL, NULL },
+  /* Test circuitmux_t object */
+  TEST_CMUX(allocate),
+  TEST_CMUX(attach_circuit),
+  TEST_CMUX(detach_circuit),
+  TEST_CMUX(detach_all_circuits),
+  TEST_CMUX(policy),
+  TEST_CMUX(xmit_cell),
+
+  /* Misc. */
+  TEST_CMUX(compute_ticks),
+  TEST_CMUX(destroy_cell_queue),
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_circuitmux_ewma.c b/src/test/test_circuitmux_ewma.c
new file mode 100644
index 0000000000..27601e0c7d
--- /dev/null
+++ b/src/test/test_circuitmux_ewma.c
@@ -0,0 +1,228 @@
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CIRCUITMUX_PRIVATE
+#define CIRCUITMUX_EWMA_PRIVATE
+
+#include "core/or/or.h"
+#include "core/or/circuitmux.h"
+#include "core/or/circuitmux_ewma.h"
+
+#include "test/fakechans.h"
+#include "test/fakecircs.h"
+#include "test/test.h"
+
+static void
+test_cmux_ewma_active_circuit(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t *pol_data = NULL;
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  circ_data = ewma_policy.alloc_circ_data(&cmux, pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+
+  /* Get EWMA specific objects. */
+
+  /* Make circuit active. */
+  ewma_policy.notify_circ_active(&cmux, pol_data, &circ, circ_data);
+
+  circuit_t *entry = ewma_policy.pick_active_circuit(&cmux, pol_data);
+  tt_mem_op(entry, OP_EQ, &circ, sizeof(circ));
+
+ done:
+  ewma_policy.free_circ_data(&cmux, pol_data, &circ, circ_data);
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void
+test_cmux_ewma_xmit_cell(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t *pol_data = NULL;
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+  ewma_policy_data_t *ewma_pol_data;
+  ewma_policy_circ_data_t *ewma_data;
+  double old_cell_count;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  circ_data = ewma_policy.alloc_circ_data(&cmux, pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  ewma_data = TO_EWMA_POL_CIRC_DATA(circ_data);
+
+  /* Make circuit active. */
+  ewma_policy.notify_circ_active(&cmux, pol_data, &circ, circ_data);
+
+  /* Move back in time the last time we calibrated so we scale the active
+   * circuit when emitting a cell. */
+  ewma_pol_data->active_circuit_pqueue_last_recalibrated -= 100;
+  ewma_data->cell_ewma.last_adjusted_tick =
+    ewma_pol_data->active_circuit_pqueue_last_recalibrated;
+
+  /* Grab old cell count. */
+  old_cell_count = ewma_data->cell_ewma.cell_count;
+
+  ewma_policy.notify_xmit_cells(&cmux, pol_data, &circ, circ_data, 1);
+
+  /* Our old cell count should be lower to what we have since we just emitted
+   * a cell and thus we scale. */
+  tt_double_op(old_cell_count, OP_LT, ewma_data->cell_ewma.cell_count);
+
+ done:
+  ewma_policy.free_circ_data(&cmux, pol_data, &circ, circ_data);
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void
+test_cmux_ewma_notify_circ(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t *pol_data = NULL;
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+  const ewma_policy_data_t *ewma_pol_data;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  circ_data = ewma_policy.alloc_circ_data(&cmux, pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+
+  /* Currently, notify_circ_active() ignores cmux and circ. They can not be
+   * NULL so it is fine to pass garbage. */
+  ewma_policy.notify_circ_active(&cmux, pol_data, &circ, circ_data);
+
+  /* We should have an active circuit in the queue so its EWMA value can be
+   * tracked. */
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  tt_int_op(smartlist_len(ewma_pol_data->active_circuit_pqueue), OP_EQ, 1);
+  tt_uint_op(ewma_pol_data->active_circuit_pqueue_last_recalibrated, OP_NE, 0);
+
+  ewma_policy.notify_circ_inactive(&cmux, pol_data, &circ, circ_data);
+  /* Should be removed from the active queue. */
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  tt_int_op(smartlist_len(ewma_pol_data->active_circuit_pqueue), OP_EQ, 0);
+  tt_uint_op(ewma_pol_data->active_circuit_pqueue_last_recalibrated, OP_NE, 0);
+
+ done:
+  ewma_policy.free_circ_data(&cmux, pol_data, &circ, circ_data);
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void
+test_cmux_ewma_policy_circ_data(void *arg)
+{
+  circuitmux_t cmux; /* garbage */
+  circuitmux_policy_data_t pol_data; /* garbage */
+  circuit_t circ; /* garbage */
+  circuitmux_policy_circ_data_t *circ_data = NULL;
+  const ewma_policy_circ_data_t *ewma_data;
+
+  (void) arg;
+
+  /* Currently, alloc_circ_data() ignores every parameter _except_ the cell
+   * direction so it is OK to pass garbage. They can not be NULL. */
+  circ_data = ewma_policy.alloc_circ_data(&cmux, &pol_data, &circ,
+                                          CELL_DIRECTION_OUT, 42);
+  tt_assert(circ_data);
+  tt_uint_op(circ_data->magic, OP_EQ, EWMA_POL_CIRC_DATA_MAGIC);
+
+  ewma_data = TO_EWMA_POL_CIRC_DATA(circ_data);
+  tt_mem_op(ewma_data->circ, OP_EQ, &circ, sizeof(circuit_t));
+  tt_double_op(ewma_data->cell_ewma.cell_count, OP_LE, 0.0);
+  tt_int_op(ewma_data->cell_ewma.heap_index, OP_EQ, -1);
+  tt_uint_op(ewma_data->cell_ewma.is_for_p_chan, OP_EQ, 0);
+  ewma_policy.free_circ_data(&cmux, &pol_data, &circ, circ_data);
+
+  circ_data = ewma_policy.alloc_circ_data(&cmux, &pol_data, &circ,
+                                          CELL_DIRECTION_IN, 42);
+  tt_assert(circ_data);
+  tt_uint_op(circ_data->magic, OP_EQ, EWMA_POL_CIRC_DATA_MAGIC);
+
+  ewma_data = TO_EWMA_POL_CIRC_DATA(circ_data);
+  tt_mem_op(ewma_data->circ, OP_EQ, &circ, sizeof(circuit_t));
+  tt_double_op(ewma_data->cell_ewma.cell_count, OP_LE, 0.0);
+  tt_int_op(ewma_data->cell_ewma.heap_index, OP_EQ, -1);
+  tt_uint_op(ewma_data->cell_ewma.is_for_p_chan, OP_EQ, 1);
+
+ done:
+  ewma_policy.free_circ_data(&cmux, &pol_data, &circ, circ_data);
+}
+
+static void
+test_cmux_ewma_policy_data(void *arg)
+{
+  circuitmux_t cmux; /* garbage. */
+  circuitmux_policy_data_t *pol_data = NULL;
+  const ewma_policy_data_t *ewma_pol_data;
+
+  (void) arg;
+
+  pol_data = ewma_policy.alloc_cmux_data(&cmux);
+  tt_assert(pol_data);
+  tt_uint_op(pol_data->magic, OP_EQ, EWMA_POL_DATA_MAGIC);
+
+  /* Test EWMA object. */
+  ewma_pol_data = TO_EWMA_POL_DATA(pol_data);
+  tt_assert(ewma_pol_data->active_circuit_pqueue);
+  tt_uint_op(ewma_pol_data->active_circuit_pqueue_last_recalibrated, OP_NE, 0);
+
+ done:
+  ewma_policy.free_cmux_data(&cmux, pol_data);
+}
+
+static void *
+cmux_ewma_setup_test(const struct testcase_t *tc)
+{
+  static int whatever;
+
+  (void) tc;
+
+  cell_ewma_initialize_ticks();
+  cmux_ewma_set_options(NULL, NULL);
+
+  return &whatever;
+}
+
+static int
+cmux_ewma_cleanup_test(const struct testcase_t *tc, void *ptr)
+{
+  (void) tc;
+  (void) ptr;
+
+  circuitmux_ewma_free_all();
+
+  return 1;
+}
+
+static struct testcase_setup_t cmux_ewma_test_setup = {
+  .setup_fn = cmux_ewma_setup_test,
+  .cleanup_fn = cmux_ewma_cleanup_test,
+};
+
+#define TEST_CMUX_EWMA(name) \
+  { #name, test_cmux_ewma_##name, TT_FORK, &cmux_ewma_test_setup, NULL }
+
+struct testcase_t circuitmux_ewma_tests[] = {
+  TEST_CMUX_EWMA(active_circuit),
+  TEST_CMUX_EWMA(policy_data),
+  TEST_CMUX_EWMA(policy_circ_data),
+  TEST_CMUX_EWMA(notify_circ),
+  TEST_CMUX_EWMA(xmit_cell),
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitpadding.c b/src/test/test_circuitpadding.c
new file mode 100644
index 0000000000..cfb24c032c
--- /dev/null
+++ b/src/test/test_circuitpadding.c
@@ -0,0 +1,3147 @@
+#define CHANNEL_OBJECT_PRIVATE
+#define TOR_TIMERS_PRIVATE
+#define CIRCUITPADDING_PRIVATE
+#define CIRCUITPADDING_MACHINES_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define CRYPT_PATH_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "lib/testsupport/testsupport.h"
+#include "core/or/connection_or.h"
+#include "core/or/channel.h"
+#include "core/or/channeltls.h"
+#include "core/or/crypt_path.h"
+#include <event.h>
+#include "lib/evloop/compat_libevent.h"
+#include "lib/time/compat_time.h"
+#include "lib/defs/time.h"
+#include "core/or/relay.h"
+#include "core/or/circuitlist.h"
+#include "core/or/circuitbuild.h"
+#include "core/or/circuitpadding.h"
+#include "core/or/circuitpadding_machines.h"
+#include "core/mainloop/netstatus.h"
+#include "core/crypto/relay_crypto.h"
+#include "core/or/protover.h"
+#include "feature/nodelist/nodelist.h"
+#include "app/config/config.h"
+
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "core/or/cell_st.h"
+#include "core/or/crypt_path_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+
+#include "test/fakecircs.h"
+#include "test/rng_test_helpers.h"
+
+/* Start our monotime mocking at 1 second past whatever monotime_init()
+ * thought the actual wall clock time was, for platforms with bad resolution
+ * and weird timevalues during monotime_init() before mocking. */
+#define MONOTIME_MOCK_START   (monotime_absolute_nsec()+\
+                               TOR_NSEC_PER_USEC*TOR_USEC_PER_SEC)
+
+extern smartlist_t *connection_array;
+void circuit_expire_old_circuits_clientside(void);
+
+circid_t get_unique_circ_id_by_chan(channel_t *chan);
+void helper_create_basic_machine(void);
+static void helper_create_conditional_machines(void);
+
+channel_t *new_fake_channel(void);
+void test_circuitpadding_negotiation(void *arg);
+void test_circuitpadding_wronghop(void *arg);
+void test_circuitpadding_conditions(void *arg);
+
+void test_circuitpadding_serialize(void *arg);
+void test_circuitpadding_rtt(void *arg);
+void test_circuitpadding_tokens(void *arg);
+void test_circuitpadding_state_length(void *arg);
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding);
+void free_fake_origin_circuit(origin_circuit_t *circ);
+
+static int deliver_negotiated = 1;
+static int64_t curr_mocked_time;
+
+static node_t padding_node;
+static node_t non_padding_node;
+
+static channel_t dummy_channel;
+static circpad_machine_spec_t circ_client_machine;
+
+static void
+timers_advance_and_run(int64_t msec_update)
+{
+  curr_mocked_time += msec_update*TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+  timers_run_pending();
+}
+
+static void
+nodes_init(void)
+{
+  padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  padding_node.rs->pv.supports_hs_setup_padding = 1;
+
+  non_padding_node.rs = tor_malloc_zero(sizeof(routerstatus_t));
+  non_padding_node.rs->pv.supports_hs_setup_padding = 0;
+}
+
+static void
+nodes_free(void)
+{
+  tor_free(padding_node.rs);
+
+  tor_free(non_padding_node.rs);
+}
+
+static const node_t *
+node_get_by_id_mock(const char *identity_digest)
+{
+  if (identity_digest[0] == 1) {
+    return &padding_node;
+  } else if (identity_digest[0] == 0) {
+    return &non_padding_node;
+  }
+
+  return NULL;
+}
+
+static const node_t *
+circuit_get_nth_node_mock(origin_circuit_t *circ, int hop)
+{
+  (void) circ;
+  (void) hop;
+
+  return &padding_node;
+}
+
+void
+free_fake_origin_circuit(origin_circuit_t *circ)
+{
+  circpad_circuit_free_all_machineinfos(TO_CIRCUIT(circ));
+  circuit_clear_cpath(circ);
+  tor_free(circ);
+}
+
+void dummy_nop_timer(void);
+
+//static int dont_stop_libevent = 0;
+
+static circuit_t *client_side;
+static circuit_t *relay_side;
+
+static int n_client_cells = 0;
+static int n_relay_cells = 0;
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction);
+
+static void
+circuitmux_attach_circuit_mock(circuitmux_t *cmux, circuit_t *circ,
+                               cell_direction_t direction)
+{
+  (void)cmux;
+  (void)circ;
+  (void)direction;
+
+  return;
+}
+
+static int
+circuit_package_relay_cell_mock(cell_t *cell, circuit_t *circ,
+                           cell_direction_t cell_direction,
+                           crypt_path_t *layer_hint, streamid_t on_stream,
+                           const char *filename, int lineno)
+{
+  (void)cell; (void)on_stream; (void)filename; (void)lineno;
+
+  if (circ == client_side) {
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      // Deliver to relay
+      circpad_handle_padding_negotiate(relay_side, cell);
+    } else {
+
+      int is_target_hop = circpad_padding_is_from_expected_hop(circ,
+                                                             layer_hint);
+      tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_OUT);
+      tt_int_op(is_target_hop, OP_EQ, 1);
+
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(client_side);
+
+      // Receive padding cell at middle
+      circpad_deliver_recognized_relay_cell_events(relay_side,
+              cell->payload[0], NULL);
+    }
+    n_client_cells++;
+  } else if (circ == relay_side) {
+    tt_int_op(cell_direction, OP_EQ, CELL_DIRECTION_IN);
+
+    if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATED) {
+      // XXX: blah need right layer_hint..
+      if (deliver_negotiated)
+        circpad_handle_padding_negotiated(client_side, cell,
+                                          TO_ORIGIN_CIRCUIT(client_side)
+                                             ->cpath->next);
+    } else if (cell->payload[0] == RELAY_COMMAND_PADDING_NEGOTIATE) {
+      circpad_handle_padding_negotiate(client_side, cell);
+    } else {
+      // No need to pretend a padding cell was sent: This event is
+      // now emitted internally when the circuitpadding code sends them.
+      //circpad_cell_event_padding_sent(relay_side);
+
+      // Receive padding cell at client
+      circpad_deliver_recognized_relay_cell_events(client_side,
+              cell->payload[0],
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+    }
+
+    n_relay_cells++;
+  }
+
+ done:
+  timers_advance_and_run(1);
+  return 0;
+}
+
+// Test reading and writing padding to strings (or options_t + consensus)
+void
+test_circuitpadding_serialize(void *arg)
+{
+  (void)arg;
+}
+
+static signed_error_t
+circpad_send_command_to_hop_mock(origin_circuit_t *circ, uint8_t hopnum,
+                                 uint8_t relay_command, const uint8_t *payload,
+                                 ssize_t payload_len)
+{
+  (void) circ;
+  (void) hopnum;
+  (void) relay_command;
+  (void) payload;
+  (void) payload_len;
+  return 0;
+}
+
+void
+test_circuitpadding_rtt(void *arg)
+{
+  /* Test Plan:
+   *
+   * 1. Test RTT measurement server side
+   *    a. test usage of measured RTT
+   * 2. Test termination of RTT measurement
+   *    a. test non-update of RTT
+   * 3. Test client side circuit and non-application of RTT..
+   */
+  circpad_delay_t rtt_estimate;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+  helper_create_basic_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,0);
+
+  /* Test 1: Test measuring RTT */
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+
+  timers_advance_and_run(20);
+
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 19000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 30000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_NE, 0);
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  circpad_cell_event_nonpadding_sent(relay_side);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_GE, 20000);
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_LE, 21000);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 2: Termination of RTT measurement (from the previous test) */
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  rtt_estimate = relay_side->padding_info[0]->rtt_estimate_usec;
+
+  circpad_cell_event_nonpadding_received(relay_side);
+  timers_advance_and_run(4);
+  circpad_cell_event_nonpadding_sent(relay_side);
+
+  tt_int_op(relay_side->padding_info[0]->rtt_estimate_usec, OP_EQ,
+            rtt_estimate);
+  tt_u64_op(relay_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->stop_rtt_update, OP_EQ, 1);
+  tt_int_op(circpad_histogram_bin_to_usec(relay_side->padding_info[0], 0),
+            OP_EQ,
+            relay_side->padding_info[0]->rtt_estimate_usec+
+            circpad_machine_current_state(
+             relay_side->padding_info[0])->histogram_edges[0]);
+
+  /* Test 3: Make sure client side machine properly ignores RTT */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  timers_advance_and_run(20);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->last_received_time_usec, OP_EQ, 0);
+
+  tt_int_op(client_side->padding_info[0]->rtt_estimate_usec, OP_EQ, 0);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_NE, client_side->padding_info[0]->rtt_estimate_usec);
+  tt_int_op(circpad_histogram_bin_to_usec(client_side->padding_info[0], 0),
+            OP_EQ,
+            circpad_machine_current_state(
+                client_side->padding_info[0])->histogram_edges[0]);
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+
+  return;
+}
+
+void
+helper_create_basic_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.name = "basic";
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_START].use_rtt_estimate = 1;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_NONPADDING_SENT] = CIRCPAD_STATE_CANCEL;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_len = 5;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 2500;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 5000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[4] = 20000;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[0] = 1;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[2] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[3] = 2;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram[4] = 2;
+
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_total_tokens = 7;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].use_rtt_estimate = 1;
+
+  return;
+}
+
+#define BIG_HISTOGRAM_LEN 10
+
+/** Setup a machine with a big histogram */
+static void
+helper_create_machine_with_big_histogram(circpad_removal_t removal_strategy)
+{
+  const int tokens_per_bin = 2;
+
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circpad_state_t *burst_state =
+    &circ_client_machine.states[CIRCPAD_STATE_BURST];
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+    next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_PADDING_RECV] = CIRCPAD_STATE_BURST;
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_RECV] =CIRCPAD_STATE_BURST;
+
+  burst_state->next_state[CIRCPAD_EVENT_NONPADDING_SENT] =CIRCPAD_STATE_CANCEL;
+
+  burst_state->token_removal = CIRCPAD_TOKEN_REMOVAL_HIGHER;
+
+  burst_state->histogram_len = BIG_HISTOGRAM_LEN;
+
+  int n_tokens = 0;
+  int i;
+  for (i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    burst_state->histogram[i] = tokens_per_bin;
+    n_tokens += tokens_per_bin;
+  }
+
+  burst_state->histogram_edges[0] = 0;
+  burst_state->histogram_edges[1] = 1;
+  burst_state->histogram_edges[2] = 7;
+  burst_state->histogram_edges[3] = 15;
+  burst_state->histogram_edges[4] = 31;
+  burst_state->histogram_edges[5] = 62;
+  burst_state->histogram_edges[6] = 125;
+  burst_state->histogram_edges[7] = 250;
+  burst_state->histogram_edges[8] = 500;
+  burst_state->histogram_edges[9] = 1000;
+
+  burst_state->histogram_total_tokens = n_tokens;
+  burst_state->length_dist.type = CIRCPAD_DIST_UNIFORM;
+  burst_state->length_dist.param1 = n_tokens;
+  burst_state->length_dist.param2 = n_tokens;
+  burst_state->max_length = n_tokens;
+  burst_state->length_includes_nonpadding = 1;
+  burst_state->use_rtt_estimate = 0;
+  burst_state->token_removal = removal_strategy;
+}
+
+static circpad_decision_t
+circpad_machine_schedule_padding_mock(circpad_machine_runtime_t *mi)
+{
+  (void)mi;
+  return 0;
+}
+
+static uint64_t
+mock_monotime_absolute_usec(void)
+{
+  return 100;
+}
+
+/** Test higher token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_higher(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_HIGHER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Test right boundaries of each histogram bin: */
+  const circpad_delay_t bin_right_bounds[] =
+    {0,    6,  14,  30,  61,  124, 249, 499, 999, CIRCPAD_DELAY_INFINITE-1};
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_right_bounds[i], OP_EQ,
+               histogram_get_bin_upper_bound(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 5, 6, 7, 8};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all lower bins are not touched */
+  for (i=0; i < 4 ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  mi->padding_scheduled_at_usec = current_time;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test lower token removal strategy by bin  */
+static void
+test_circuitpadding_token_removal_lower(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_LOWER);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 2, 1, 0};
+  unsigned i;
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (i = 0; i < sizeof(bin_removal_order)/sizeof(int) ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all higher bins are untouched */
+  for (i = 5; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy by bin  */
+static void
+test_circuitpadding_closest_token_removal(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 6, 1, 7, 0, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_closest_token_removal_usec(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_CLOSEST_USEC);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /*************************************************************************/
+
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Test left boundaries of each histogram bin: */
+  const circpad_delay_t bin_left_bounds[] =
+    {0, 1,    7,  15,  31,  62,  125, 250, 500, 1000, CIRCPAD_DELAY_INFINITE};
+  for (int i = 0; i <= BIG_HISTOGRAM_LEN ; i++) {
+    tt_uint_op(bin_left_bounds[i], OP_EQ,
+               circpad_histogram_bin_to_usec(mi, i));
+  }
+
+  /* XXX we want to test remove_token_exact and
+     circpad_machine_remove_closest_token() with usec */
+
+  /* Check that all bins have two tokens right now */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* This is the right order to remove tokens from this histogram. That is, we
+   * first remove tokens from the 4th bin since 57 usec is nearest to the 4th
+   * bin midpoint (31 + (62-31)/2 == 46). Then we remove from the 3rd bin for
+   * the same reason, then from the 5th, etc. */
+  const int bin_removal_order[] = {4, 3, 5, 2, 1, 0, 6, 7, 8, 9};
+
+  /* Remove all tokens from all bins apart from the infinity bin */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN-1 ; i++) {
+    int bin_to_remove = bin_removal_order[i];
+    log_debug(LD_GENERAL, "Testing that %d attempt removes %d bin",
+              i, bin_to_remove);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 2);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 1);
+
+    mi->padding_scheduled_at_usec = current_time - 57;
+    circpad_cell_event_nonpadding_sent(client_side);
+
+    /* Test that we cleaned out this bin. Don't do this in the case of the last
+       bin since the tokens will get refilled */
+    if (i != BIG_HISTOGRAM_LEN - 2) {
+      tt_int_op(mi->histogram[bin_to_remove], OP_EQ, 0);
+    }
+  }
+
+  /* Check that all bins have been refilled */
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    tt_int_op(mi->histogram[i], OP_EQ, 2);
+  }
+
+  /* Test below the lowest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[0] = 100;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[1] = 101;
+  circ_client_machine.states[CIRCPAD_STATE_BURST].histogram_edges[2] = 120;
+  mi->padding_scheduled_at_usec = current_time - 102;
+  mi->histogram[0] = 0;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[1], OP_EQ, 1);
+
+  /* Test above the highest bin, for coverage */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+  circ_client_machine.states[CIRCPAD_STATE_BURST].
+    histogram_edges[BIG_HISTOGRAM_LEN-2] = 100;
+  mi->padding_scheduled_at_usec = current_time - 29202;
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[BIG_HISTOGRAM_LEN-2], OP_EQ, 1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+/** Test closest token removal strategy with usec  */
+static void
+test_circuitpadding_token_removal_exact(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  (void)arg;
+
+  /* Mock it up */
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup test environment (time etc.) */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+
+  /* Create test machine */
+  helper_create_machine_with_big_histogram(CIRCPAD_TOKEN_REMOVAL_EXACT);
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+
+  /* move the machine to the right state */
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  /* Get the machine and setup tokens */
+  mi = client_side->padding_info[0];
+  tt_assert(mi);
+
+  /**********************************************************************/
+  uint64_t current_time = monotime_absolute_usec();
+
+  /* Ensure that we will clear out bin #4 with this usec */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_cell_event_nonpadding_sent(client_side);
+  mi->padding_scheduled_at_usec = current_time - 57;
+  tt_int_op(mi->histogram[4], OP_EQ, 1);
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_int_op(mi->histogram[4], OP_EQ, 0);
+
+  /* Ensure that we will not remove any other tokens even tho we try to, since
+   * this is what the exact strategy dictates */
+  mi->padding_scheduled_at_usec = current_time - 57;
+  circpad_cell_event_nonpadding_sent(client_side);
+  for (int i = 0; i < BIG_HISTOGRAM_LEN ; i++) {
+    if (i != 4) {
+      tt_int_op(mi->histogram[i], OP_EQ, 2);
+    }
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+#undef BIG_HISTOGRAM_LEN
+
+void
+test_circuitpadding_tokens(void *arg)
+{
+  const circpad_state_t *state;
+  circpad_machine_runtime_t *mi;
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  testing_enable_reproducible_rng();
+
+  /** Test plan:
+   *
+   * 1. Test symmetry between bin_to_usec and usec_to_bin
+   *    a. Test conversion
+   *    b. Test edge transitions (lower, upper)
+   * 2. Test remove higher on an empty bin
+   *    a. Normal bin
+   *    b. Infinity bin
+   *    c. Bin 0
+   *    d. No higher
+   * 3. Test remove lower
+   *    a. Normal bin
+   *    b. Bin 0
+   *    c. No lower
+   * 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] = circpad_circuit_machineinfo_new(client_side,
+                                                                 0);
+
+  mi = client_side->padding_info[0];
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_sent(client_side);
+  /* We have to save the infinity bin because one inf delay
+   * could have been chosen when we transition to burst */
+  circpad_hist_token_t inf_bin = mi->histogram[4];
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_BURST);
+
+  state = circpad_machine_current_state(client_side->padding_info[0]);
+
+  // Test 0: convert bin->usec->bin
+  // Bin 0+1 have different semantics
+  for (int bin = 0; bin < 2; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+  }
+  for (int bin = 2; bin < state->histogram_len-1; bin++) {
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    /* Verify we round down */
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec+3);
+    tt_int_op(bin, OP_EQ, bin2);
+
+    bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec-1);
+    tt_int_op(bin, OP_EQ, bin2+1);
+  }
+
+  // Test 1: converting usec->bin->usec->bin
+  // Bin 0+1 have different semantics.
+  for (circpad_delay_t i = 0; i <= state->histogram_edges[0]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_LE, usec);
+  }
+  for (circpad_delay_t i = state->histogram_edges[0]+1;
+       i <= state->histogram_edges[0] +
+         state->histogram_edges[state->histogram_len-2]; i++) {
+    int bin = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                            i);
+    circpad_delay_t usec =
+        circpad_histogram_bin_to_usec(client_side->padding_info[0], bin);
+    int bin2 = circpad_histogram_usec_to_bin(client_side->padding_info[0],
+                                             usec);
+    tt_int_op(bin, OP_EQ, bin2);
+    tt_int_op(i, OP_GE, usec);
+  }
+
+  /* 2.a. Normal higher bin */
+  {
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+  }
+
+  /* 2.b. Higher Infinity bin */
+  {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1);
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+
+    /* Test past the infinity bin */
+    circpad_machine_remove_higher_token(mi,
+         circpad_histogram_bin_to_usec(mi, 5)+1000000);
+
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+  }
+
+  /* 2.c. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+    mi->histogram[0] = 1;
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* Drain the infinity bin and cause a refill */
+  while (inf_bin != 0) {
+    tt_int_op(mi->histogram[4], OP_EQ, inf_bin);
+    circpad_cell_event_nonpadding_received(client_side);
+    inf_bin--;
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+
+  // We should have refilled here.
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 3.a. Bin 0 */
+  {
+    tt_int_op(mi->histogram[0], OP_EQ, 1);
+    circpad_machine_remove_higher_token(mi, state->histogram_edges[0]/2);
+    tt_int_op(mi->histogram[0], OP_EQ, 0);
+  }
+
+  /* 3.b. Test remove lower normal bin */
+  {
+    tt_int_op(mi->histogram[3], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[3], OP_EQ, 0);
+    tt_int_op(mi->histogram[2], OP_EQ, 2);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    /* 3.c. No lower */
+    circpad_machine_remove_lower_token(mi,
+         circpad_histogram_bin_to_usec(mi, 3)+1);
+    tt_int_op(mi->histogram[2], OP_EQ, 0);
+  }
+
+  /* 4. Test remove closest
+   *    a. Closest lower
+   *    b. Closest higher
+   *    c. Closest 0
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 2)+1, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  /* 5. Test remove closest usec
+   *    a. Closest 0
+   *    b. Closest lower (below midpoint)
+   *    c. Closest higher (above midpoint)
+   *    d. Closest Infinity
+   */
+  circpad_machine_setup_tokens(mi);
+
+  tt_int_op(mi->histogram[0], OP_EQ, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[0], OP_EQ, 0);
+  tt_int_op(mi->histogram[2], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 0)/3, 1);
+  tt_int_op(mi->histogram[2], OP_EQ, 0);
+  tt_int_op(mi->histogram[3], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[3], OP_EQ, 0);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  circpad_machine_remove_closest_token(mi,
+         circpad_histogram_bin_to_usec(mi, 4), 1);
+  tt_int_op(mi->histogram[4], OP_EQ, 2);
+
+  // XXX: Need more coverage of the actual usec branches
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  monotime_disable_test_mocking();
+  tor_free(circ_client_machine.states);
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_wronghop(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Padding sent from hop 1 and 3 to client
+   * 2. Send negotiated from hop 1 and 3 to client
+   * 3. Garbled negotiated cell
+   * 4. Padding negotiate sent to client
+   * 5. Send negotiate stop command for unknown machine
+   * 6. Send negotiated to relay
+   * 7. Garbled padding negotiate cell
+   */
+  (void)arg;
+  uint32_t read_bw = 0, overhead_bw = 0;
+  cell_t cell;
+  signed_error_t ret;
+  origin_circuit_t *orig_client;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  /* Mock this function so that our cell counting tests don't get confused by
+   * padding that gets sent by scheduled timers. */
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  orig_client = TO_ORIGIN_CIRCUIT(client_side);
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build three hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+
+  /* 1. Test padding from first and third hop */
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  circpad_deliver_recognized_relay_cell_events(client_side,
+              RELAY_COMMAND_DROP,
+              TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_LT,
+            orig_client->n_overhead_read_circ_bw);
+
+  /* 2. Test padding negotiated not handled from hops 1,3 */
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath);
+  tt_int_op(ret, OP_EQ, -1);
+
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 3. Garbled negotiated cell */
+  memset(&cell, 255, sizeof(cell));
+  ret = circpad_handle_padding_negotiated(client_side, &cell,
+          TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 4. Test that negotiate is dropped at origin */
+  read_bw = orig_client->n_delivered_read_circ_bw;
+  overhead_bw = orig_client->n_overhead_read_circ_bw;
+  relay_send_command_from_edge(0, relay_side,
+                               RELAY_COMMAND_PADDING_NEGOTIATE,
+                               (void*)cell.payload,
+                               (size_t)3, NULL);
+  tt_int_op(read_bw, OP_EQ,
+            orig_client->n_delivered_read_circ_bw);
+  tt_int_op(overhead_bw, OP_EQ,
+            orig_client->n_overhead_read_circ_bw);
+
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 5. Test that asking to stop the wrong machine does nothing */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            255, 2, CIRCPAD_COMMAND_STOP);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* 6. Sending negotiated command to relay does nothing */
+  ret = circpad_handle_padding_negotiated(relay_side, &cell, NULL);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* 7. Test garbled negotated cell (bad command 255) */
+  memset(&cell, 0, sizeof(cell));
+  ret = circpad_handle_padding_negotiate(relay_side, &cell);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Spoof padding negotiated on circuit with no padding */
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_OK);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  circpad_padding_negotiated(relay_side,
+                             CIRCPAD_MACHINE_CIRC_SETUP,
+                             CIRCPAD_COMMAND_START,
+                             CIRCPAD_RESPONSE_ERR);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+  testing_disable_reproducible_rng();
+}
+
+void
+test_circuitpadding_negotiation(void *arg)
+{
+  /**
+   * Test plan:
+   * 1. Test circuit where padding is supported by middle
+   *    a. Make sure padding negotiation is sent
+   *    b. Test padding negotiation delivery and parsing
+   * 2. Test circuit where padding is unsupported by middle
+   *    a. Make sure padding negotiation is not sent
+   * 3. Test failure to negotiate a machine due to desync.
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+
+  /* Build two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+  /* verify echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Test 2: Test no padding */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 0);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* Finish circuit */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Force negotiate padding. */
+  circpad_negotiate_padding(TO_ORIGIN_CIRCUIT(client_side),
+                            CIRCPAD_MACHINE_CIRC_SETUP,
+                            2, CIRCPAD_COMMAND_START);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* verify no echo was sent */
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(n_client_cells, OP_EQ, 1);
+
+  /* 3. Test failure to negotiate a machine due to desync */
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  SMARTLIST_FOREACH(relay_padding_machines,
+          circpad_machine_spec_t *,
+          m, tor_free(m->states); tor_free(m));
+  smartlist_free(relay_padding_machines);
+  relay_padding_machines = smartlist_new();
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* verify echo was sent */
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  /* verify no padding was negotiated */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  monotime_disable_test_mocking();
+  UNMOCK(node_get_by_id);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  nodes_free();
+}
+
+static void
+simulate_single_hop_extend(circuit_t *client, circuit_t *mid_relay,
+                           int padding)
+{
+  char whatevs_key[CPATH_KEY_MATERIAL_LEN];
+  char digest[DIGEST_LEN];
+  tor_addr_t addr;
+
+  // Pretend a non-padding cell was sent
+  circpad_cell_event_nonpadding_sent(client);
+
+  // Receive extend cell at middle
+  circpad_cell_event_nonpadding_received(mid_relay);
+
+  // Advance time a tiny bit so we can calculate an RTT
+  curr_mocked_time += 10 * TOR_NSEC_PER_MSEC;
+  monotime_coarse_set_mock_time_nsec(curr_mocked_time);
+  monotime_set_mock_time_nsec(curr_mocked_time);
+
+  // Receive extended cell at middle
+  circpad_cell_event_nonpadding_sent(mid_relay);
+
+  // Receive extended cell at first hop
+  circpad_cell_event_nonpadding_received(client);
+
+  // Add a hop to cpath
+  crypt_path_t *hop = tor_malloc_zero(sizeof(crypt_path_t));
+  cpath_extend_linked_list(&TO_ORIGIN_CIRCUIT(client)->cpath, hop);
+
+  hop->magic = CRYPT_PATH_MAGIC;
+  hop->state = CPATH_STATE_OPEN;
+
+  // add an extend info to indicate if this node supports padding or not.
+  // (set the first byte of the digest for our mocked node_get_by_id)
+  digest[0] = padding;
+
+  hop->extend_info = extend_info_new(
+          padding ? "padding" : "non-padding",
+          digest, NULL, NULL, NULL,
+          &addr, padding);
+
+  cpath_init_circuit_crypto(hop, whatevs_key, sizeof(whatevs_key), 0, 0);
+
+  hop->package_window = circuit_initial_package_window();
+  hop->deliver_window = CIRCWINDOW_START;
+
+  // Signal that the hop was added
+  circpad_machine_event_circ_added_hop(TO_ORIGIN_CIRCUIT(client));
+}
+
+static circpad_machine_spec_t *
+helper_create_length_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_BINS_EMPTY] = CIRCPAD_STATE_END;
+
+  /* No token removal.. end via state_length only */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_NONE;
+
+  /* Let's have this one end after 12 packets */
+  ret->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param1 = 12;
+  ret->states[CIRCPAD_STATE_BURST].length_dist.param2 = 13;
+  ret->states[CIRCPAD_STATE_BURST].max_length = 12;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 4;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[3] = 10000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 6;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 0;
+
+  return ret;
+}
+
+static circpad_machine_spec_t *
+helper_create_conditional_machine(void)
+{
+  circpad_machine_spec_t *ret =
+    tor_malloc_zero(sizeof(circpad_machine_spec_t));
+
+  /* Start, burst */
+  circpad_machine_states_init(ret, 2);
+
+  ret->states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_PADDING_SENT] = CIRCPAD_STATE_BURST;
+
+  ret->states[CIRCPAD_STATE_BURST].
+      next_state[CIRCPAD_EVENT_LENGTH_COUNT] = CIRCPAD_STATE_END;
+
+  /* Use EXACT removal strategy, otherwise setup_tokens() does not work */
+  ret->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_len = 3;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[0] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[1] = 1;
+  ret->states[CIRCPAD_STATE_BURST].histogram_edges[2] = 1000000;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram[0] = 6;
+  ret->states[CIRCPAD_STATE_BURST].histogram[1] = 0;
+  ret->states[CIRCPAD_STATE_BURST].histogram[2] = 0;
+
+  ret->states[CIRCPAD_STATE_BURST].histogram_total_tokens = 6;
+  ret->states[CIRCPAD_STATE_BURST].use_rtt_estimate = 0;
+  ret->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  return ret;
+}
+
+static void
+helper_create_conditional_machines(void)
+{
+  circpad_machine_spec_t *add = helper_create_conditional_machine();
+
+  if (!origin_padding_machines)
+    origin_padding_machines = smartlist_new();
+  if (!relay_padding_machines)
+    relay_padding_machines = smartlist_new();
+
+  add->machine_num = 2;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 0;
+  add->conditions.min_hops = 2;
+  add->conditions.state_mask = CIRCPAD_CIRC_BUILDING|
+           CIRCPAD_CIRC_NO_STREAMS|CIRCPAD_CIRC_HAS_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  add->is_origin_side = 1;
+  add->should_negotiate_end = 1;
+  add->target_hopnum = 2;
+
+  /* Let's have this one end after 4 packets */
+  add->states[CIRCPAD_STATE_BURST].length_dist.type = CIRCPAD_DIST_UNIFORM;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param1 = 4;
+  add->states[CIRCPAD_STATE_BURST].length_dist.param2 = 4;
+  add->states[CIRCPAD_STATE_BURST].max_length = 4;
+
+  add->conditions.requires_vanguards = 1;
+  add->conditions.min_hops = 3;
+  add->conditions.state_mask = CIRCPAD_CIRC_OPENED|
+           CIRCPAD_CIRC_STREAMS|CIRCPAD_CIRC_HAS_NO_RELAY_EARLY;
+  add->conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+  circpad_register_padding_machine(add, origin_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 2;
+  circpad_register_padding_machine(add, relay_padding_machines);
+
+  add = helper_create_conditional_machine();
+  add->machine_num = 3;
+  circpad_register_padding_machine(add, relay_padding_machines);
+}
+
+void
+test_circuitpadding_state_length(void *arg)
+{
+  /**
+   * Test plan:
+   *  * Explicitly test that with no token removal enabled, we hit
+   *    the state length limit due to either padding, or non-padding.
+   *  * Repeat test with an arbitrary token removal strategy, and
+   *    verify that if we run out of tokens due to padding before we
+   *    hit the state length, we still go to state end (all our
+   *    token removal tests only test nonpadding token removal).
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  circpad_machine_spec_t *client_machine =
+      helper_create_length_machine();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  client_side->padding_machine[0] = client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  circpad_machine_runtime_t *mi = client_side->padding_info[0];
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_EQ, NULL);
+
+  /* Verify that non-padding does not change our state length */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* verify that sending padding changes our state length */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+  circpad_send_padding_cell_for_callback(mi);
+
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+
+  /* Now, count nonpadding as part of the state length */
+  client_machine->states[CIRCPAD_STATE_BURST].length_includes_nonpadding = 1;
+
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+
+  /* Verify that non-padding does change our state length now */
+  for (uint64_t i = mi->state_length-1; i > 0; i--) {
+    circpad_cell_event_nonpadding_sent(client_side);
+    tt_i64_op(mi->state_length, OP_EQ, i);
+  }
+
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, -1);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Now, just test token removal when we send padding */
+  client_machine->states[CIRCPAD_STATE_BURST].token_removal =
+      CIRCPAD_TOKEN_REMOVAL_EXACT;
+
+  /* Restart machine */
+  mi->current_state = CIRCPAD_STATE_START;
+  circpad_cell_event_padding_sent(client_side);
+  tt_i64_op(mi->state_length, OP_EQ, 12);
+  tt_ptr_op(mi->histogram, OP_NE, NULL);
+  tt_int_op(mi->chosen_bin, OP_EQ, 2);
+
+  /* verify that sending padding changes our state length and
+   * our histogram now */
+  for (uint32_t i = mi->histogram[2]-1; i > 0; i--) {
+    circpad_send_padding_cell_for_callback(mi);
+    tt_int_op(mi->chosen_bin, OP_EQ, 2);
+    tt_int_op(mi->histogram[2], OP_EQ, i);
+  }
+
+  tt_i64_op(mi->state_length, OP_EQ, 7);
+  tt_int_op(mi->histogram[2], OP_EQ, 1);
+
+  circpad_send_padding_cell_for_callback(mi);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+ done:
+  tor_free(client_machine->states);
+  tor_free(client_machine);
+
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(node_get_by_id);
+
+  return;
+}
+
+void
+test_circuitpadding_conditions(void *arg)
+{
+  /**
+   * Test plan:
+   *  0. Make a few origin and client machines with diff conditions
+   *     * vanguards, purposes, has_opened circs, no relay early
+   *     * Client side should_negotiate_end
+   *     * Length limits
+   *  1. Test STATE_END transitions
+   *  2. Test new machine after end with same conditions
+   *  3. Test new machine due to changed conditions
+   *     * Esp: built event, no relay early, no streams
+   * XXX: Diff test:
+   *  1. Test STATE_END with pending timers
+   *  2. Test marking a circuit before padding callback fires
+   *  3. Test freeing a circuit before padding callback fires
+   */
+  int64_t actual_mocked_monotime_start;
+  (void)arg;
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel));
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  timers_initialize();
+  helper_create_conditional_machines();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Send another event.. verify machine gets re-added properly
+   * (test race with shutdown) */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = 0;
+  circpad_machine_event_circ_has_no_streams(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Now make the circuit opened and send built event */
+  TO_ORIGIN_CIRCUIT(client_side)->has_opened = 1;
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  TO_ORIGIN_CIRCUIT(client_side)->remaining_relay_early_cells = 0;
+  circpad_machine_event_circ_has_no_relay_early(
+          TO_ORIGIN_CIRCUIT(client_side));
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  get_options_mutable()->HSLayer2Nodes = (void*)1;
+  TO_ORIGIN_CIRCUIT(client_side)->p_streams = (void*)1;
+  circpad_machine_event_circ_has_streams(TO_ORIGIN_CIRCUIT(client_side));
+
+  /* Verify different machine is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  /* Hold off on negotiated */
+  deliver_negotiated = 0;
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client
+   * but not the response for the padding machine */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify machine is gone from the relay (but negotiated not back yet */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Add another hop and verify it's back */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 3);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 3);
+
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+
+ done:
+  /* XXX: Free everything */
+  testing_disable_reproducible_rng();
+  return;
+}
+
+/** Disabled unstable test until #29298 is implemented (see #29122) */
+#if 0
+void
+test_circuitpadding_circuitsetup_machine(void *arg)
+{
+  int64_t actual_mocked_monotime_start;
+  /**
+   * Test case plan:
+   *
+   * 1. Simulate a normal circuit setup pattern
+   *    a. Application traffic
+   *
+   * FIXME: This should focus more on exercising the machine
+   * features rather than actual traffic patterns. For example,
+   * test cancellation and bins empty/refill
+   */
+  (void)arg;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+
+  dummy_channel.cmux = circuitmux_alloc();
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+
+  timers_initialize();
+  circpad_machines_init();
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Test case #1: Build a 3 hop circuit, then wait and let pad */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  tt_int_op(n_client_cells, OP_EQ, 1);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_BURST);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_int_op(relay_side->padding_info[0]->is_padding_timer_scheduled,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 1);
+
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 2);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 2);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 3);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 3);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 4);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 4);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 5);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  timers_advance_and_run(2000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 5);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  timers_advance_and_run(5000);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 6);
+
+  tt_int_op(client_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_END);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  tt_int_op(relay_side->padding_info[0]->current_state,
+            OP_EQ, CIRCPAD_STATE_GAP);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+
+  /* Verify we can't schedule padding in END state */
+  circpad_decision_t ret =
+      circpad_machine_schedule_padding(client_side->padding_info[0]);
+  tt_int_op(ret, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Simulate application traffic */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+  tt_int_op(n_client_cells, OP_EQ, 6);
+  tt_int_op(n_relay_cells, OP_EQ, 7);
+
+  // Test timer cancellation
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 8);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Test timer cancel due to state rules */
+  circpad_cell_event_nonpadding_sent(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_EQ, 0);
+  circpad_cell_event_padding_received(client_side);
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+
+  /* Simulate application traffic to cancel timer */
+  circpad_cell_event_nonpadding_sent(client_side);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_OUT);
+  circpad_deliver_unrecognized_cell_events(relay_side, CELL_DIRECTION_IN);
+  circpad_deliver_recognized_relay_cell_events(client_side, RELAY_COMMAND_DATA,
+                                  TO_ORIGIN_CIRCUIT(client_side)->cpath->next);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* No cells sent, except negotiate end from relay */
+  tt_int_op(n_client_cells, OP_EQ, 8);
+  tt_int_op(n_relay_cells, OP_EQ, 9);
+
+  /* Test mark for close and free */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  timers_advance_and_run(5000);
+  circpad_cell_event_padding_received(client_side);
+
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+                CIRCPAD_STATE_BURST);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+          CIRCPAD_STATE_GAP);
+
+  tt_u64_op(client_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  tt_u64_op(relay_side->padding_info[0]->padding_scheduled_at_usec,
+            OP_NE, 0);
+  circuit_mark_for_close(client_side, END_CIRC_REASON_FLAG_REMOTE);
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  timers_advance_and_run(5000);
+
+  /* No cells sent */
+  tt_int_op(n_client_cells, OP_EQ, 10);
+  tt_int_op(n_relay_cells, OP_EQ, 10);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  timers_shutdown();
+  monotime_disable_test_mocking();
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuitmux_attach_circuit);
+
+  return;
+}
+#endif /* 0 */
+
+/** Helper function: Initializes a padding machine where every state uses the
+ *  uniform probability distribution.  */
+static void
+helper_circpad_circ_distribution_machine_setup(int min, int max)
+{
+  circpad_machine_states_init(&circ_client_machine, 7);
+
+  circpad_state_t *zero_st = &circ_client_machine.states[0];
+  zero_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 1;
+  zero_st->iat_dist.type = CIRCPAD_DIST_UNIFORM;
+  /* param2 is upper bound, param1 is lower */
+  zero_st->iat_dist.param1 = min;
+  zero_st->iat_dist.param2 = max;
+  zero_st->dist_added_shift_usec = min;
+  zero_st->dist_max_sample_usec = max;
+
+  circpad_state_t *first_st = &circ_client_machine.states[1];
+  first_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 2;
+  first_st->iat_dist.type = CIRCPAD_DIST_LOGISTIC;
+  /* param1 is Mu, param2 is sigma. */
+  first_st->iat_dist.param1 = 9;
+  first_st->iat_dist.param2 = 3;
+  first_st->dist_added_shift_usec = min;
+  first_st->dist_max_sample_usec = max;
+
+  circpad_state_t *second_st = &circ_client_machine.states[2];
+  second_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 3;
+  second_st->iat_dist.type = CIRCPAD_DIST_LOG_LOGISTIC;
+  /* param1 is Alpha, param2 is 1.0/Beta */
+  second_st->iat_dist.param1 = 1;
+  second_st->iat_dist.param2 = 0.5;
+  second_st->dist_added_shift_usec = min;
+  second_st->dist_max_sample_usec = max;
+
+  circpad_state_t *third_st = &circ_client_machine.states[3];
+  third_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 4;
+  third_st->iat_dist.type = CIRCPAD_DIST_GEOMETRIC;
+  /* param1 is 'p' (success probability) */
+  third_st->iat_dist.param1 = 0.2;
+  third_st->dist_added_shift_usec = min;
+  third_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fourth_st = &circ_client_machine.states[4];
+  fourth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 5;
+  fourth_st->iat_dist.type = CIRCPAD_DIST_WEIBULL;
+  /* param1 is k, param2 is Lambda */
+  fourth_st->iat_dist.param1 = 1.5;
+  fourth_st->iat_dist.param2 = 1;
+  fourth_st->dist_added_shift_usec = min;
+  fourth_st->dist_max_sample_usec = max;
+
+  circpad_state_t *fifth_st = &circ_client_machine.states[5];
+  fifth_st->next_state[CIRCPAD_EVENT_NONPADDING_RECV] = 6;
+  fifth_st->iat_dist.type = CIRCPAD_DIST_PARETO;
+  /* param1 is sigma, param2 is xi */
+  fifth_st->iat_dist.param1 = 1;
+  fifth_st->iat_dist.param2 = 5;
+  fifth_st->dist_added_shift_usec = min;
+  fifth_st->dist_max_sample_usec = max;
+}
+
+/** Simple test that the padding delays sampled from a uniform distribution
+ *  actually faill within the uniform distribution range. */
+static void
+test_circuitpadding_sample_distribution(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+  int n_samples;
+  int n_states;
+
+  (void) arg;
+
+  /* mock this function so that we dont actually schedule any padding */
+  MOCK(circpad_machine_schedule_padding,
+       circpad_machine_schedule_padding_mock);
+  testing_enable_reproducible_rng();
+
+  /* Initialize a machine with multiple probability distributions */
+  circpad_machines_init();
+  helper_circpad_circ_distribution_machine_setup(0, 10);
+
+  /* Initialize machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* For every state, sample a bunch of values from the distribution and ensure
+   * they fall within range. */
+  for (n_states = 0 ; n_states < 6; n_states++) {
+    /* Make sure we in the right state */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ, n_states);
+
+    for (n_samples = 0; n_samples < 100; n_samples++) {
+      circpad_delay_t delay = circpad_machine_sample_delay(mi);
+      tt_int_op(delay, OP_GE, 0);
+      tt_int_op(delay, OP_LE, 10);
+    }
+
+    /* send a non-padding cell to move to the next machine state */
+    circpad_cell_event_nonpadding_received(client_side);
+  }
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  UNMOCK(circpad_machine_schedule_padding);
+  testing_disable_reproducible_rng();
+}
+
+static circpad_decision_t
+circpad_machine_spec_transition_mock(circpad_machine_runtime_t *mi,
+                                circpad_event_t event)
+{
+  (void) mi;
+  (void) event;
+
+  return CIRCPAD_STATE_UNCHANGED;
+}
+
+/* Test per-machine padding rate limits */
+static void
+test_circuitpadding_machine_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circpad_send_command_to_hop, circpad_send_command_to_hop_mock);
+  testing_enable_reproducible_rng();
+
+  /* Setup machine and circuits */
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  helper_create_basic_machine();
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  /* First we are going to test the per-machine rate limits */
+  circ_client_machine.max_padding_percent = 50;
+  circ_client_machine.allowed_padding_count = 100;
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Cover wrap */
+  for (;i<UINT16_MAX;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/2+1);
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, mi);
+  for (i=0;i<UINT16_MAX;i++) {
+    circpad_cell_event_nonpadding_sent(client_side);
+  }
+
+  tt_int_op(mi->nonpadding_sent, OP_EQ, UINT16_MAX/2);
+  tt_int_op(mi->padding_sent, OP_EQ, UINT16_MAX/4+1);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  testing_disable_reproducible_rng();
+}
+
+/* Test global padding rate limits */
+static void
+test_circuitpadding_global_rate_limiting(void *arg)
+{
+  (void) arg;
+  bool retval;
+  circpad_machine_runtime_t *mi;
+  int i;
+  int64_t actual_mocked_monotime_start;
+
+  /* Ignore machine transitions for the purposes of this function, we only
+   * really care about padding counts */
+  MOCK(circpad_machine_spec_transition, circpad_machine_spec_transition_mock);
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(monotime_absolute_usec, mock_monotime_absolute_usec);
+  testing_enable_reproducible_rng();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  client_side = TO_CIRCUIT(origin_circuit_new());
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  dummy_channel.cmux = circuitmux_alloc();
+
+  /* Setup machine and circuits */
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  helper_create_basic_machine();
+  relay_side->padding_machine[0] = &circ_client_machine;
+  relay_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(relay_side, 0);
+  mi = relay_side->padding_info[0];
+  /* Set up the machine info so that we can get through the basic functions */
+  mi->state_length = CIRCPAD_STATE_LENGTH_INFINITE;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Now test the global limits by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_global_allowed_cells=100 circpad_global_max_padding_pct=50",
+                         NULL, 0, 0);
+  /* Register global limits with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  /* Check padding limit, should be fine since we haven't sent anything yet. */
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Send 99 padding cells which is below circpad_global_allowed_cells=100, so
+   * the rate limit will not trigger */
+  for (i=0;i<99;i++) {
+    circpad_send_padding_cell_for_callback(mi);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Now send another padding cell to pass circpad_global_allowed_cells=100,
+     and see that the limit will trigger */
+  circpad_send_padding_cell_for_callback(mi);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  retval = circpad_machine_schedule_padding(mi);
+  tt_int_op(retval, OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Now send 92 non-padding cells to get near the
+   * circpad_global_max_padding_pct=50 limit; in particular with 96 non-padding
+   * cells, the padding traffic is still 51% of total traffic so limit should
+   * trigger */
+  for (i=0;i<92;i++) {
+    circpad_cell_event_nonpadding_sent(relay_side);
+  }
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 1);
+
+  /* Send another non-padding cell to bring the padding traffic to 50% of total
+   * traffic and get past the limit */
+  circpad_cell_event_nonpadding_sent(relay_side);
+  retval = circpad_machine_reached_padding_limit(mi);
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  testing_disable_reproducible_rng();
+}
+
+/* Test reduced and disabled padding */
+static void
+test_circuitpadding_reduce_disable(void *arg)
+{
+  (void) arg;
+  int64_t actual_mocked_monotime_start;
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  testing_enable_reproducible_rng();
+
+  nodes_init();
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = (circuit_t *)new_fake_orcirc(&dummy_channel,
+                                            &dummy_channel);
+  client_side = (circuit_t *)origin_circuit_new();
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  circpad_machines_init();
+  helper_create_conditional_machines();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  actual_mocked_monotime_start = MONOTIME_MOCK_START;
+  monotime_set_mock_time_nsec(actual_mocked_monotime_start);
+  monotime_coarse_set_mock_time_nsec(actual_mocked_monotime_start);
+  curr_mocked_time = actual_mocked_monotime_start;
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  MOCK(circuit_package_relay_cell,
+       circuit_package_relay_cell_mock);
+  MOCK(node_get_by_id,
+       node_get_by_id_mock);
+
+  /* Simulate extend. This should result in the original machine getting
+   * added, since the circuit is not built */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #2 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  /* Deliver a padding cell to the client, to trigger burst state */
+  circpad_cell_event_padding_sent(client_side);
+
+  /* This should have trigger length shutdown condition on client.. */
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Verify machine is gone from both sides */
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  /* Now test the reduced padding machine by setting up the consensus */
+  networkstatus_t vote1;
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_reduced=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  /* Test that machines get torn down when padding is disabled */
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = smartlist_new();
+  smartlist_split_string(vote1.net_params,
+         "circpad_padding_disabled=1", NULL, 0, 0);
+
+  /* Register reduced padding machine with the padding subsystem */
+  circpad_new_consensus_params(&vote1);
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+  SMARTLIST_FOREACH(vote1.net_params, char *, cp, tor_free(cp));
+  smartlist_free(vote1.net_params);
+  vote1.net_params = NULL;
+  circpad_new_consensus_params(&vote1);
+
+  get_options_mutable()->ReducedCircuitPadding = 1;
+
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* Verify that machine #0 is added */
+  tt_int_op(client_side->padding_machine[0]->machine_num, OP_EQ, 2);
+  tt_int_op(relay_side->padding_machine[0]->machine_num, OP_EQ, 2);
+
+  tt_int_op(
+    circpad_machine_reached_padding_limit(client_side->padding_info[0]),
+    OP_EQ, 0);
+  tt_int_op(
+    circpad_machine_reached_padding_limit(relay_side->padding_info[0]),
+    OP_EQ, 0);
+
+  get_options_mutable()->CircuitPadding = 0;
+
+  tt_int_op(
+    circpad_machine_schedule_padding(client_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+  tt_int_op(
+    circpad_machine_schedule_padding(relay_side->padding_info[0]),
+    OP_EQ, CIRCPAD_STATE_UNCHANGED);
+
+  /* Signal that circuit is built: this event causes us to re-evaluate
+   * machine conditions (which don't apply because padding is disabled). */
+
+  circpad_machine_event_circ_built(TO_ORIGIN_CIRCUIT(client_side));
+
+  tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_EQ, NULL);
+
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  testing_disable_reproducible_rng();
+}
+
+/** Just a basic machine whose whole purpose is to reach the END state */
+static void
+helper_create_ender_machine(void)
+{
+  /* Start, burst */
+  circpad_machine_states_init(&circ_client_machine, 2);
+
+  circ_client_machine.states[CIRCPAD_STATE_START].
+      next_state[CIRCPAD_EVENT_NONPADDING_RECV] = CIRCPAD_STATE_END;
+
+  circ_client_machine.conditions.state_mask = CIRCPAD_STATE_ALL;
+  circ_client_machine.conditions.purpose_mask = CIRCPAD_PURPOSE_ALL;
+}
+
+static time_t mocked_timeofday;
+/** Set timeval to a mock date and time. This is necessary
+ * to make tor_gettimeofday() mockable. */
+static void
+mock_tor_gettimeofday(struct timeval *timeval)
+{
+  timeval->tv_sec = mocked_timeofday;
+  timeval->tv_usec = 0;
+}
+
+/** Test manual managing of circuit lifetimes by the circuitpadding
+ *  subsystem. In particular this test goes through all the cases of the
+ *  circpad_marked_circuit_for_padding() function, via
+ *  circuit_mark_for_close() as well as
+ *  circuit_expire_old_circuits_clientside(). */
+static void
+test_circuitpadding_manage_circuit_lifetime(void *arg)
+{
+  circpad_machine_runtime_t *mi;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  monotime_enable_test_mocking();
+  MOCK(tor_gettimeofday, mock_tor_gettimeofday);
+  mocked_timeofday = 23;
+
+  helper_create_ender_machine();
+
+  /* Enable manual circuit lifetime manage for this test */
+  circ_client_machine.manage_circ_lifetime = 1;
+
+  /* Test setup */
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Check that the circuit is not marked for close */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+
+  /* Mark this circuit for close due to a remote reason */
+  circuit_mark_for_close(client_side,
+                         END_CIRC_REASON_FLAG_REMOTE|END_CIRC_REASON_NONE);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close due to a protocol issue */
+  circuit_mark_for_close(client_side, END_CIRC_REASON_TORPROTOCOL);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_GENERAL);
+  client_side->marked_for_close = 0;
+
+  /* Mark a measurement circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_MEASURE_TIMEOUT);
+  client_side->marked_for_close = 0;
+
+  /* Mark a general circuit for close */
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+  circuit_mark_for_close(client_side, END_CIRC_REASON_NONE);
+
+  /* Check that this circuit is still not marked for close since we are
+   * managing the lifetime manually, but the circuit was tagged as such by the
+   * circpadding subsystem */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* We just tested case (1) from the comments of
+   * circpad_circuit_should_be_marked_for_close() */
+
+  /* Transition the machine to the END state but did not delete its machine */
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* We just tested case (3) from the comments of
+   * circpad_circuit_should_be_marked_for_close().
+   * Now let's go for case (2). */
+
+  /* Reset the close mark */
+  client_side->marked_for_close = 0;
+
+  /* Mark this circuit for close */
+  circuit_mark_for_close(client_side, 0);
+
+  /* See that the circ got closed since we are already in END state */
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* We just tested case (2). Now let's see that case (4) is unreachable as
+     that comment claims */
+
+  /* First, reset all close marks and tags */
+  client_side->marked_for_close = 0;
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  /* Now re-create the ender machine so that we can transition to END again */
+  /* Free up some stuff first */
+  circpad_circuit_free_all_machineinfos(client_side);
+  tor_free(circ_client_machine.states);
+  helper_create_ender_machine();
+
+  client_side->padding_machine[0] = &circ_client_machine;
+  client_side->padding_info[0] =
+    circpad_circuit_machineinfo_new(client_side, 0);
+  mi = client_side->padding_info[0];
+
+  /* Check we are in START. */
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_START);
+
+  /* Test that we don't expire this circuit yet */
+  client_side->timestamp_dirty = 0;
+  client_side->state = CIRCUIT_STATE_OPEN;
+  tor_gettimeofday(&client_side->timestamp_began);
+  TO_ORIGIN_CIRCUIT(client_side)->circuit_idle_timeout = 23;
+  mocked_timeofday += 24;
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->timestamp_dirty, OP_NE, 0);
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  tt_int_op(client_side->purpose, OP_EQ, CIRCUIT_PURPOSE_C_CIRCUIT_PADDING);
+
+  /* Runaway circpad test: if the machine does not transition to end,
+   * test that after CIRCPAD_DELAY_MAX_SECS, we get marked anyway */
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  client_side->padding_info[0]->last_cell_time_sec =
+      approx_time()-(CIRCPAD_DELAY_MAX_SECS+10);
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+  /* Test back to normal: if we had activity, we won't close */
+  client_side->padding_info[0]->last_cell_time_sec = approx_time();
+  client_side->marked_for_close = 0;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Transition to END, but before we're past the dirty timer */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circpad_cell_event_nonpadding_received(client_side);
+  tt_int_op(mi->current_state, OP_EQ, CIRCPAD_STATE_END);
+
+  /* Verify that the circuit was not closed. */
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+
+  /* Now that we are in END state, we can be closed by expiry, but via
+   * the timestamp_dirty path, not the idle path. So first test not dirty
+   * enough. */
+  mocked_timeofday = client_side->timestamp_dirty;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_EQ, 0);
+  mocked_timeofday = client_side->timestamp_dirty
+      + get_options()->MaxCircuitDirtiness + 2;
+  circuit_expire_old_circuits_clientside();
+  tt_int_op(client_side->marked_for_close, OP_NE, 0);
+
+ done:
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+  tor_free(circ_client_machine.states);
+  monotime_disable_test_mocking();
+  UNMOCK(tor_gettimeofday);
+}
+
+/** Helper for the test_circuitpadding_hs_machines test:
+ *
+ *  - Create a client and relay circuit.
+ *  - Setup right circuit purpose and attach a machine to the client circuit.
+ *  - Verify that state transitions work as intended and state length gets
+ *    enforced.
+ *
+ *  This function is able to do this test both for intro and rend circuits
+ *  depending on the value of <b>test_intro_circs</b>.
+ */
+static void
+helper_test_hs_machines(bool test_intro_circs)
+{
+  /* Setup the circuits */
+  origin_circuit_t *origin_client_side = origin_circuit_new();
+  client_side = TO_CIRCUIT(origin_client_side);
+  client_side->purpose = CIRCUIT_PURPOSE_C_GENERAL;
+
+  dummy_channel.cmux = circuitmux_alloc();
+  relay_side = TO_CIRCUIT(new_fake_orcirc(&dummy_channel, &dummy_channel));
+  relay_side->purpose = CIRCUIT_PURPOSE_OR;
+
+  /* extend the client circ to two hops */
+  simulate_single_hop_extend(client_side, relay_side, 1);
+  simulate_single_hop_extend(client_side, relay_side, 1);
+
+  /* machines only apply on opened circuits */
+  origin_client_side->has_opened = 1;
+
+  /************************************/
+
+  /* Attaching the client machine now won't work here because of a wrong
+   * purpose */
+  tt_assert(!client_side->padding_machine[0]);
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_assert(!client_side->padding_machine[0]);
+
+  /* Change the purpose, see the machine getting attached */
+  client_side->purpose = test_intro_circs ?
+    CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT : CIRCUIT_PURPOSE_C_REND_JOINED;
+  circpad_add_matching_machines(origin_client_side, origin_padding_machines);
+  tt_ptr_op(client_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(client_side->padding_machine[0], OP_NE, NULL);
+
+  tt_ptr_op(relay_side->padding_info[0], OP_NE, NULL);
+  tt_ptr_op(relay_side->padding_machine[0], OP_NE, NULL);
+
+  /* Verify that the right machine is attached */
+  tt_str_op(client_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "client_ip_circ" : "client_rp_circ");
+  tt_str_op(relay_side->padding_machine[0]->name, OP_EQ,
+            test_intro_circs ? "relay_ip_circ": "relay_rp_circ");
+
+  /***********************************/
+
+  /* Intro machines are at START state, but rend machines have already skipped
+   * to OBFUSCATE_CIRC_SETUP because of the sent PADDING_NEGOTIATE. */
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /*Send non-padding to move the machines from START to OBFUSCATE_CIRC_SETUP */
+  circpad_cell_event_nonpadding_received(client_side);
+  circpad_cell_event_nonpadding_received(relay_side);
+  tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+  tt_int_op(relay_side->padding_info[0]->current_state, OP_EQ,
+            CIRCPAD_STATE_OBFUSCATE_CIRC_SETUP);
+
+  /* Check that the state lengths have been sampled and are within range */
+  circpad_machine_runtime_t *client_machine_runtime =
+    client_side->padding_info[0];
+  circpad_machine_runtime_t *relay_machine_runtime =
+    relay_side->padding_info[0];
+
+  if (test_intro_circs) {
+    /* on the client side, we don't send any padding so
+     * state length is not set */
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, -1);
+    /* relay side has state limits. check them */
+    tt_i64_op(relay_machine_runtime->state_length, OP_GE,
+              INTRO_MACHINE_MINIMUM_PADDING);
+    tt_i64_op(relay_machine_runtime->state_length, OP_LT,
+              INTRO_MACHINE_MAXIMUM_PADDING);
+  } else {
+    tt_i64_op(client_machine_runtime->state_length, OP_EQ, 1);
+    tt_i64_op(relay_machine_runtime->state_length, OP_EQ, 1);
+  }
+
+  if (test_intro_circs) {
+    int i;
+    /* Send state_length worth of padding from the relay and see that the
+     * client state goes to END */
+    for (i = (int) relay_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(relay_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted (the padding info should now be null on both sides) */
+    tt_ptr_op(relay_side->padding_info[0], OP_EQ, NULL);
+    tt_ptr_op(client_side->padding_info[0], OP_EQ, NULL);
+  } else {
+    int i;
+    /* Send state_length worth of padding and see that the state goes to END */
+    for (i = (int) client_machine_runtime->state_length ; i > 0 ; i--) {
+      circpad_send_padding_cell_for_callback(client_machine_runtime);
+    }
+    /* See that the machine has been teared down after all the length has been
+     * exhausted. */
+    tt_int_op(client_side->padding_info[0]->current_state, OP_EQ,
+              CIRCPAD_STATE_END);
+  }
+
+ done:
+  free_fake_orcirc(TO_OR_CIRCUIT(relay_side));
+  circuitmux_detach_all_circuits(dummy_channel.cmux, NULL);
+  circuitmux_free(dummy_channel.cmux);
+  free_fake_origin_circuit(TO_ORIGIN_CIRCUIT(client_side));
+}
+
+/** Test that the HS circuit padding machines work as intended. */
+static void
+test_circuitpadding_hs_machines(void *arg)
+{
+  (void)arg;
+
+  /* Test logic:
+   *
+   * 1) Register the HS machines, which aim to hide the presense of
+   *    onion service traffic on the client-side
+   *
+   * 2) Call helper_test_hs_machines() to perform tests for the intro circuit
+   *    machines and for the rend circuit machines.
+  */
+
+  MOCK(circuitmux_attach_circuit, circuitmux_attach_circuit_mock);
+  MOCK(circuit_package_relay_cell, circuit_package_relay_cell_mock);
+  MOCK(circuit_get_nth_node, circuit_get_nth_node_mock);
+  MOCK(circpad_machine_schedule_padding,circpad_machine_schedule_padding_mock);
+
+  origin_padding_machines = smartlist_new();
+  relay_padding_machines = smartlist_new();
+
+  nodes_init();
+
+  monotime_init();
+  monotime_enable_test_mocking();
+  monotime_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  monotime_coarse_set_mock_time_nsec(1*TOR_NSEC_PER_USEC);
+  curr_mocked_time = 1*TOR_NSEC_PER_USEC;
+
+  timers_initialize();
+
+  /* This is needed so that we are not considered to be dormant */
+  note_user_activity(20);
+
+  /************************************/
+
+  /* Register the HS machines */
+  circpad_machine_client_hide_intro_circuits(origin_padding_machines);
+  circpad_machine_client_hide_rend_circuits(origin_padding_machines);
+  circpad_machine_relay_hide_intro_circuits(relay_padding_machines);
+  circpad_machine_relay_hide_rend_circuits(relay_padding_machines);
+
+  /***********************************/
+
+  /* Do the tests for the intro circuit machines */
+  helper_test_hs_machines(true);
+  /* Do the tests for the rend circuit machines */
+  helper_test_hs_machines(false);
+
+  timers_shutdown();
+  monotime_disable_test_mocking();
+
+  SMARTLIST_FOREACH_BEGIN(origin_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  SMARTLIST_FOREACH_BEGIN(relay_padding_machines,
+                          circpad_machine_spec_t *, m) {
+    machine_spec_free(m);
+  } SMARTLIST_FOREACH_END(m);
+
+  smartlist_free(origin_padding_machines);
+  smartlist_free(relay_padding_machines);
+
+  UNMOCK(circuitmux_attach_circuit);
+  UNMOCK(circuit_package_relay_cell);
+  UNMOCK(circuit_get_nth_node);
+  UNMOCK(circpad_machine_schedule_padding);
+}
+
+/** Test that we effectively ignore non-padding cells in padding circuits. */
+static void
+test_circuitpadding_ignore_non_padding_cells(void *arg)
+{
+  int retval;
+  relay_header_t rh;
+
+  (void) arg;
+
+  client_side = (circuit_t *)origin_circuit_new();
+  client_side->purpose = CIRCUIT_PURPOSE_C_CIRCUIT_PADDING;
+
+  rh.command = RELAY_COMMAND_BEGIN;
+
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = handle_relay_cell_command(NULL, client_side, NULL, NULL, &rh, 0);
+  tt_int_op(retval, OP_EQ, 0);
+  expect_log_msg_containing("Ignored cell");
+
+ done:
+  ;
+}
+
+#define TEST_CIRCUITPADDING(name, flags) \
+    { #name, test_##name, (flags), NULL, NULL }
+
+struct testcase_t circuitpadding_tests[] = {
+  TEST_CIRCUITPADDING(circuitpadding_tokens, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_state_length, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_negotiation, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_wronghop, TT_FORK),
+  /** Disabled unstable test until #29298 is implemented (see #29122) */
+  //  TEST_CIRCUITPADDING(circuitpadding_circuitsetup_machine, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_conditions, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_rtt, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_sample_distribution, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_machine_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_global_rate_limiting, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_reduce_disable, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_lower, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_higher, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_closest_token_removal_usec, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_token_removal_exact, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_manage_circuit_lifetime, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_hs_machines, TT_FORK),
+  TEST_CIRCUITPADDING(circuitpadding_ignore_non_padding_cells, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_circuitstats.c b/src/test/test_circuitstats.c
index 1cbcb14f2b..e15dec5a01 100644
--- a/src/test/test_circuitstats.c
+++ b/src/test/test_circuitstats.c
@@ -1,10 +1,10 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITSTATS_PRIVATE
 #define CIRCUITLIST_PRIVATE
-#define CHANNEL_PRIVATE_
+#define CHANNEL_FILE_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
@@ -28,7 +28,7 @@ origin_circuit_t *subtest_fourhop_circuit(struct timeval, int);
 origin_circuit_t *add_opened_threehop(void);
 origin_circuit_t *build_unopened_fourhop(struct timeval);
 
-int onion_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
+int cpath_append_hop(crypt_path_t **head_ptr, extend_info_t *choice);
 
 static int marked_for_close;
 /* Mock function because we are not trying to test the close circuit that does
@@ -57,9 +57,9 @@ add_opened_threehop(void)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = DEFAULT_ROUTE_LEN;
 
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
-  onion_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
+  cpath_append_hop(&or_circ->cpath, &fakehop);
 
   or_circ->has_opened = 1;
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -82,10 +82,10 @@ build_unopened_fourhop(struct timeval circ_start_time)
   or_circ->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
   or_circ->build_state->desired_path_len = 4;
 
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
-  onion_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
+  cpath_append_hop(&or_circ->cpath, fakehop);
 
   tor_free(fakehop);
 
@@ -197,7 +197,7 @@ test_circuitstats_hoplen(void *arg)
 }
 
 #define TEST_CIRCUITSTATS(name, flags) \
-    { #name, test_##name, (flags), NULL, NULL }
+    { #name, test_##name, (flags), &helper_pubsub_setup, NULL }
 
 struct testcase_t circuitstats_tests[] = {
   TEST_CIRCUITSTATS(circuitstats_hoplen, TT_FORK),
diff --git a/src/test/test_circuituse.c b/src/test/test_circuituse.c
index 3acfc12044..49438d9d3b 100644
--- a/src/test/test_circuituse.c
+++ b/src/test/test_circuituse.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITLIST_PRIVATE
diff --git a/src/test/test_cmdline.sh b/src/test/test_cmdline.sh
new file mode 100755
index 0000000000..ded58af63d
--- /dev/null
+++ b/src/test/test_cmdline.sh
@@ -0,0 +1,65 @@
+#!/bin/sh
+
+umask 077
+set -e
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+die() { echo "$1" >&2 ; exit 5; }
+
+echo "A"
+
+DATA_DIR=$(mktemp -d -t tor_cmdline_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# 1. Test list-torrc-options.
+OUT="${DATA_DIR}/output"
+
+echo "B"
+"${TOR_BINARY}" --list-torrc-options > "$OUT"
+
+echo "C"
+
+# regular options are given.
+grep -i "SocksPort" "$OUT" >/dev/null || die "Did not find SocksPort"
+
+
+echo "D"
+
+# unlisted options are given, since they do not have the NOSET flag.
+grep -i "__SocksPort" "$OUT" > /dev/null || die "Did not find __SocksPort"
+
+echo "E"
+
+# unsettable options are not given.
+if grep -i "DisableIOCP" "$OUT"  /dev/null; then
+    die "Found DisableIOCP"
+fi
+if grep -i "HiddenServiceOptions" "$OUT" /dev/null ; then
+    die "Found HiddenServiceOptions"
+fi
+echo "OK"
diff --git a/src/test/test_compat_libevent.c b/src/test/test_compat_libevent.c
index 2f8646e897..5376e08fb3 100644
--- a/src/test/test_compat_libevent.c
+++ b/src/test/test_compat_libevent.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define COMPAT_LIBEVENT_PRIVATE
@@ -13,8 +13,6 @@
 
 #include "test/log_test_helpers.h"
 
-#define NS_MODULE compat_libevent
-
 static void
 test_compat_libevent_logging_callback(void *ignored)
 {
@@ -151,8 +149,6 @@ test_compat_libevent_postloop_events(void *arg)
   mainloop_event_t *a = NULL, *b = NULL;
   periodic_timer_t *timed = NULL;
 
-  tor_libevent_postfork();
-
   /* If postloop events don't work, then these events will activate one
    * another ad infinitum and, and the periodic event will never occur. */
   b = mainloop_event_postloop_new(activate_event_cb, &a);
@@ -187,4 +183,3 @@ struct testcase_t compat_libevent_tests[] = {
     TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_config.c b/src/test/test_config.c
index d648666f6e..71beb93f67 100644
--- a/src/test/test_config.c
+++ b/src/test/test_config.c
@@ -1,11 +1,13 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CONFIG_PRIVATE
+#define RELAY_CONFIG_PRIVATE
+#define RELAY_TRANSPORT_CONFIG_PRIVATE
 #define PT_PRIVATE
 #define ROUTERSET_PRIVATE
 #include "core/or/or.h"
@@ -16,7 +18,10 @@
 #include "core/or/circuitmux_ewma.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "app/config/resolve_addr.h"
+#include "feature/relay/relay_config.h"
+#include "feature/relay/transport_config.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "test/test.h"
@@ -24,6 +29,7 @@
 #include "feature/control/control.h"
 #include "core/mainloop/cpuworker.h"
 #include "feature/dircache/dirserv.h"
+#include "feature/dirclient/dirclient_modes.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/relay/dns.h"
 #include "feature/client/entrynodes.h"
@@ -37,6 +43,7 @@
 #include "core/or/policies.h"
 #include "feature/rend/rendclient.h"
 #include "feature/rend/rendservice.h"
+#include "feature/relay/relay_find_addr.h"
 #include "feature/relay/router.h"
 #include "feature/relay/routermode.h"
 #include "feature/nodelist/dirlist.h"
@@ -45,6 +52,7 @@
 #include "app/config/statefile.h"
 
 #include "test/test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "feature/dirclient/dir_server_st.h"
 #include "core/or/port_cfg_st.h"
@@ -54,6 +62,7 @@
 #include "lib/meminfo/meminfo.h"
 #include "lib/net/gethostname.h"
 #include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
@@ -670,6 +679,54 @@ transport_is_needed_mock(const char *transport_name)
   return transport_is_needed_mock_return;
 }
 
+static void
+test_config_parse_tcp_proxy_line(void *arg)
+{
+  (void)arg;
+
+  int ret;
+  char *msg = NULL;
+  or_options_t *options = get_options_mutable();
+
+  /* Bad TCPProxy line - too short. */
+  ret = parse_tcp_proxy_line("haproxy", options, &msg);
+  /* Return error. */
+  tt_int_op(ret, OP_EQ, -1);
+  /* Correct error message. */
+  tt_str_op(msg, OP_EQ, "TCPProxy has no address/port. Please fix.");
+  /* Free error message. */
+  tor_free(msg);
+
+  /* Bad TCPProxy line - unsupported protocol. */
+  ret = parse_tcp_proxy_line("unsupported 95.216.163.36:443", options, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "TCPProxy protocol is not supported. Currently the "
+                        "only supported protocol is 'haproxy'. Please fix.");
+  tor_free(msg);
+
+  /* Bad TCPProxy line - unparsable address/port. */
+  MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
+  ret = parse_tcp_proxy_line("haproxy bogus_address!/300", options, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "TCPProxy address/port failed to parse or resolve. "
+                        "Please fix.");
+  tor_free(msg);
+  UNMOCK(tor_addr_lookup);
+
+  /* Good TCPProxy line - ipv4. */
+  ret = parse_tcp_proxy_line("haproxy 95.216.163.36:443", options, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(options->TCPProxyProtocol, OP_EQ, TCP_PROXY_PROTOCOL_HAPROXY);
+  /* Correct the address. */
+  tt_assert(tor_addr_eq_ipv4h(&options->TCPProxyAddr, 0x5fd8a324));
+  tt_int_op(options->TCPProxyPort, OP_EQ, 443);
+  tor_free(msg);
+
+ done:
+  UNMOCK(tor_addr_lookup);
+}
+
 /**
  * Test parsing for the ClientTransportPlugin and ServerTransportPlugin config
  * options.
@@ -687,84 +744,84 @@ test_config_parse_transport_plugin_line(void *arg)
   int old_transport_is_needed_mock_call_count;
 
   /* Bad transport lines - too short */
-  r = parse_transport_line(options, "bad", 1, 0);
+  r = pt_parse_transport_line(options, "bad", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options, "bad", 1, 1);
+  r = pt_parse_transport_line(options, "bad", 1, 1);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options, "bad bad", 1, 0);
+  r = pt_parse_transport_line(options, "bad bad", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options, "bad bad", 1, 1);
+  r = pt_parse_transport_line(options, "bad bad", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* Test transport list parsing */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
    "transport_1 exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_EQ, 0);
   /* Bad transport identifiers */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_* exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* Check SOCKS cases for client transport */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks4 1.2.3.4:567", 1, 0);
   tt_int_op(r, OP_EQ, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 1, 0);
   tt_int_op(r, OP_EQ, 0);
   /* Proxy case for server transport */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 1, 1);
   tt_int_op(r, OP_EQ, 0);
   /* Multiple-transport error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 socks5 1.2.3.4:567", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1,transport_2 proxy 1.2.3.4:567", 1, 1);
   tt_int_op(r, OP_LT, 0);
   /* No port error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3.4", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
      "transport_1 proxy 1.2.3.4", 1, 1);
   tt_int_op(r, OP_LT, 0);
   /* Unparsable address error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3:6x7", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 proxy 1.2.3:6x7", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* "Strange {Client|Server}TransportPlugin field" error exit */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 foo bar", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 foo bar", 1, 1);
   tt_int_op(r, OP_LT, 0);
 
   /* No sandbox mode error exit */
   tmp = options->Sandbox;
   options->Sandbox = 1;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 0);
   tt_int_op(r, OP_LT, 0);
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 1, 1);
   tt_int_op(r, OP_LT, 0);
   options->Sandbox = tmp;
@@ -776,7 +833,7 @@ test_config_parse_transport_plugin_line(void *arg)
   MOCK(pt_kickstart_proxy, pt_kickstart_proxy_mock);
   old_pt_kickstart_proxy_mock_call_count =
     pt_kickstart_proxy_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 1);
   tt_int_op(r, OP_EQ, 0);
   tt_assert(pt_kickstart_proxy_mock_call_count ==
@@ -784,7 +841,7 @@ test_config_parse_transport_plugin_line(void *arg)
   UNMOCK(pt_kickstart_proxy);
 
   /* This one hits a log line in the !validate_only case only */
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 proxy 1.2.3.4:567", 0, 1);
   tt_int_op(r, OP_EQ, 0);
 
@@ -801,7 +858,7 @@ test_config_parse_transport_plugin_line(void *arg)
     transport_add_from_config_mock_call_count;
   old_transport_is_needed_mock_call_count =
     transport_is_needed_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
   tt_int_op(r, OP_EQ, 0);
@@ -825,7 +882,7 @@ test_config_parse_transport_plugin_line(void *arg)
     transport_add_from_config_mock_call_count;
   old_transport_is_needed_mock_call_count =
     transport_is_needed_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 exec /usr/bin/fake-transport", 0, 0);
   /* Should have succeeded */
   tt_int_op(r, OP_EQ, 0);
@@ -849,7 +906,7 @@ test_config_parse_transport_plugin_line(void *arg)
     transport_add_from_config_mock_call_count;
   old_transport_is_needed_mock_call_count =
     transport_is_needed_mock_call_count;
-  r = parse_transport_line(options,
+  r = pt_parse_transport_line(options,
       "transport_1 socks5 1.2.3.4:567", 0, 0);
   /* Should have succeeded */
   tt_int_op(r, OP_EQ, 0);
@@ -904,14 +961,12 @@ test_config_fix_my_family(void *arg)
   family3->next = NULL;
 
   or_options_t* options = options_new();
-  or_options_t* defaults = options_new();
   (void) arg;
 
   options_init(options);
-  options_init(defaults);
   options->MyFamily_lines = family;
 
-  options_validate(NULL, options, defaults, 0, &err) ;
+  options_validate(NULL, options, &err) ;
 
   if (err != NULL) {
     TT_FAIL(("options_validate failed: %s", err));
@@ -933,7 +988,6 @@ test_config_fix_my_family(void *arg)
  done:
   tor_free(err);
   or_options_free(options);
-  or_options_free(defaults);
 }
 
 static int n_hostname_01010101 = 0;
@@ -1754,6 +1808,18 @@ add_default_fallback_dir_servers_known_default(void)
   n_add_default_fallback_dir_servers_known_default++;
 }
 
+/* Helper for test_config_adding_dir_servers(), which should be
+ * refactored: clear the fields in the options which the options object
+ * does not really own. */
+static void
+ads_clear_helper(or_options_t *options)
+{
+  options->DirAuthorities = NULL;
+  options->AlternateBridgeAuthority = NULL;
+  options->AlternateDirAuthority = NULL;
+  options->FallbackDir = NULL;
+}
+
 /* Test all the different combinations of adding dir servers */
 static void
 test_config_adding_dir_servers(void *arg)
@@ -1761,7 +1827,7 @@ test_config_adding_dir_servers(void *arg)
   (void)arg;
 
   /* allocate options */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
 
   /* Allocate and populate configuration lines:
    *
@@ -1884,7 +1950,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -1966,7 +2034,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2107,7 +2177,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2248,7 +2320,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2390,7 +2464,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2542,7 +2618,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2696,7 +2774,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -2859,7 +2939,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3016,7 +3098,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3182,7 +3266,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3345,7 +3431,9 @@ test_config_adding_dir_servers(void *arg)
     n_add_default_fallback_dir_servers_known_default = 0;
 
     /* clear options*/
-    memset(options, 0, sizeof(or_options_t));
+    ads_clear_helper(options);
+    or_options_free(options);
+    options = options_new();
 
     /* clear any previous dir servers:
      consider_adding_dir_servers() should do this anyway */
@@ -3514,10 +3602,7 @@ test_config_adding_dir_servers(void *arg)
   tor_free(test_fallback_directory->value);
   tor_free(test_fallback_directory);
 
-  options->DirAuthorities = NULL;
-  options->AlternateBridgeAuthority = NULL;
-  options->AlternateDirAuthority = NULL;
-  options->FallbackDir = NULL;
+  ads_clear_helper(options);
   or_options_free(options);
 
   UNMOCK(add_default_fallback_dir_servers);
@@ -3532,7 +3617,7 @@ test_config_default_dir_servers(void *arg)
   int fallback_count = 0;
 
   /* new set of options should stop fallback parsing */
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 0;
   /* set old_options to NULL to force dir update */
   consider_adding_dir_servers(opts, NULL);
@@ -3546,7 +3631,7 @@ test_config_default_dir_servers(void *arg)
   /* if we disable the default fallbacks, there must not be any extra */
   tt_assert(fallback_count == trusted_count);
 
-  opts = tor_malloc_zero(sizeof(or_options_t));
+  opts = options_new();
   opts->UseDefaultFallbackDirs = 1;
   consider_adding_dir_servers(opts, opts);
   trusted_count = smartlist_len(router_get_trusted_dir_servers());
@@ -3606,7 +3691,7 @@ test_config_directory_fetch(void *arg)
   (void)arg;
 
   /* Test Setup */
-  or_options_t *options = tor_malloc_zero(sizeof(or_options_t));
+  or_options_t *options = options_new();
   routerinfo_t routerinfo;
   memset(&routerinfo, 0, sizeof(routerinfo));
   mock_router_pick_published_address_result = -1;
@@ -3618,70 +3703,76 @@ test_config_directory_fetch(void *arg)
        mock_router_my_exit_policy_is_reject_star);
   MOCK(advertised_server_mode, mock_advertised_server_mode);
   MOCK(router_get_my_routerinfo, mock_router_get_my_routerinfo);
+  or_options_free(options);
+  options = options_new();
 
   /* Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
   options->ClientOnly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* Bridge Clients can use multiple directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->UseBridges = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* Bridge Relays (Bridges) must act like clients, and use multiple
    * directory mirrors for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->BridgeRelay = 1;
   options->ORPort_set = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* Clients set to FetchDirInfoEarly must fetch it from the authorities,
    * but can use multiple authorities for bootstrap */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->FetchDirInfoEarly = 1;
   tt_assert(server_mode(options) == 0);
   tt_assert(public_server_mode(options) == 0);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 1);
 
   /* OR servers only fetch the consensus from the authorities when they don't
    * know their own address, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
 
   mock_router_pick_published_address_result = -1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
   /* Exit OR servers only fetch the consensus from the authorities when they
    * refuse unknown exits, but never use multiple directories for bootstrap
    */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->ORPort_set = 1;
   options->ExitRelay = 1;
   mock_router_pick_published_address_result = 0;
@@ -3694,7 +3785,7 @@ test_config_directory_fetch(void *arg)
   options->RefuseUnknownExits = 1;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3702,7 +3793,7 @@ test_config_directory_fetch(void *arg)
   mock_router_pick_published_address_result = 0;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3711,7 +3802,8 @@ test_config_directory_fetch(void *arg)
    * advertising their dirport, and never use multiple directories for
    * bootstrap. This only applies if they are also OR servers.
    * (We don't care much about the behaviour of non-OR directory servers.) */
-  memset(options, 0, sizeof(or_options_t));
+  or_options_free(options);
+  options = options_new();
   options->DirPort_set = 1;
   options->ORPort_set = 1;
   options->DirCache = 1;
@@ -3723,7 +3815,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3732,7 +3824,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3740,7 +3832,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = NULL;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3750,7 +3842,7 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 0);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 0);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
@@ -3760,12 +3852,12 @@ test_config_directory_fetch(void *arg)
   mock_router_get_my_routerinfo_result = &routerinfo;
   tt_assert(server_mode(options) == 1);
   tt_assert(public_server_mode(options) == 1);
-  tt_int_op(directory_fetches_from_authorities(options), OP_EQ, 1);
+  tt_int_op(dirclient_fetches_from_authorities(options), OP_EQ, 1);
   tt_int_op(networkstatus_consensus_can_use_multiple_directories(options),
             OP_EQ, 0);
 
  done:
-  tor_free(options);
+  or_options_free(options);
   UNMOCK(router_pick_published_address);
   UNMOCK(router_get_my_routerinfo);
   UNMOCK(advertised_server_mode);
@@ -3776,7 +3868,9 @@ static void
 test_config_default_fallback_dirs(void *arg)
 {
   const char *fallback[] = {
+#ifndef COCCI
 #include "app/config/fallback_dirs.inc"
+#endif
     NULL
   };
 
@@ -3958,40 +4052,40 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   slout = smartlist_new();
 
   // Test no defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and no out
-  ret = parse_port_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = port_parse_config(NULL, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test no defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 0, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, no defaultaddress and with out
-  ret = parse_port_config(slout, NULL, "DNS", 0, NULL, 42, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, NULL, 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test no defaultport, with defaultaddress and with out
-  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, "127.0.0.2", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
 
   // Test with defaultport, with defaultaddress and out, adds a new port cfg
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
+  ret = port_parse_config(slout, NULL, "DNS", 0, "127.0.0.2", 42, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
@@ -4002,7 +4096,7 @@ test_config_parse_port_config__ports__no_ports_given(void *data)
   // for a unix address
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
+  ret = port_parse_config(slout, NULL, "DNS", 0, "/foo/bar/unixdomain",
                           42, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4029,30 +4123,32 @@ test_config_parse_port_config__ports__ports_given(void *data)
 
   slout = smartlist_new();
 
+  mock_hostname_resolver();
+
   // Test error when encounters an invalid Port specification
   config_port_invalid = mock_config_line("DNSPort", "");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters an empty unix domain specification
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "unix:");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0, NULL,
                           0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test error when encounters a unix domain specification but the listener
   // doesn't support domain sockets
   config_port_valid = mock_config_line("DNSPort", "unix:/tmp/foo/bar");
-  ret = parse_port_config(NULL, config_port_valid, "DNS",
+  ret = port_parse_config(NULL, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test valid unix domain
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0, 0);
 #ifdef _WIN32
   tt_int_op(ret, OP_EQ, -1);
@@ -4063,7 +4159,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   tt_int_op(port_cfg->port, OP_EQ, 0);
   tt_int_op(port_cfg->is_unix_addr, OP_EQ, 1);
   tt_str_op(port_cfg->unix_addr, OP_EQ, "/tmp/foo/bar");
-  /* Test entry port defaults as initialised in parse_port_config */
+  /* Test entry port defaults as initialised in port_parse_config */
   tt_int_op(port_cfg->entry_cfg.dns_request, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.ipv4_traffic, OP_EQ, 1);
   tt_int_op(port_cfg->entry_cfg.onion_traffic, OP_EQ, 1);
@@ -4077,7 +4173,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
                                          "unix:/tmp/foo/bar NoIPv4Traffic "
                                          "NoIPv6Traffic "
                                          "NoOnionTraffic");
-  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
+  ret = port_parse_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4086,7 +4182,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.1:80 NoDNSRequest");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS",
+  ret = port_parse_config(NULL, config_port_invalid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4099,7 +4195,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("DNSPort", "127.0.0.1:80 "
                                        "NoIPv6Traffic "
                                        "NoIPv4Traffic NoOnionTraffic");
-  ret = parse_port_config(slout, config_port_valid, "DNS",
+  ret = port_parse_config(slout, config_port_valid, "DNS",
                           CONN_TYPE_AP_DNS_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4115,7 +4211,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("SOCKSPort",
                                          "NoIPv6Traffic "
                                          "unix:/tmp/foo/bar NoIPv4Traffic");
-  ret = parse_port_config(NULL, config_port_invalid, "SOCKS",
+  ret = port_parse_config(NULL, config_port_invalid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4128,7 +4224,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4150,7 +4246,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar\" "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4172,7 +4268,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"/tmp/foo/ bar "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4184,7 +4280,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_valid = mock_config_line("SOCKSPort", "unix:\"\" "
                                        "NoIPv6Traffic "
                                        "NoDNSRequest NoIPv4Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, -1);
@@ -4195,7 +4291,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "OnionTrafficOnly");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4216,7 +4312,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "NoIPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4235,7 +4331,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/foo/bar "
                                        "IPv4Traffic IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, NULL, 0,
                           CL_PORT_TAKES_HOSTNAMES);
 #ifdef _WIN32
@@ -4251,28 +4347,28 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test failure if we specify world writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 WorldWritable");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 GroupWritable");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test failure if we specify group writable for an IP Port
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "42 RelaxDirModeCheck");
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with only a port (this will fail without a default address)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42");
-  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4281,7 +4377,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestPort");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4294,7 +4390,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateDestPorts");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4307,7 +4403,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateDestAddr");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4320,7 +4416,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4333,7 +4429,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientProtocol");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4346,7 +4442,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IsolateClientAddr");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4357,7 +4453,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with ignored unknown options
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort", "42 ThisOptionDoesntExist");
-  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4366,7 +4462,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoIsolateSOCKSAuth");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.3", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4379,7 +4475,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort",
                                        "42 IPv6Traffic PreferIPv6");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.42", 0,
                           CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
@@ -4392,7 +4488,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4405,7 +4501,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheIPv6DNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4418,7 +4514,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoCacheIPv4DNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4431,7 +4527,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 CacheDNS");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_TAKES_HOSTNAMES);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4444,7 +4540,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv4Cache");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4457,7 +4553,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseIPv6Cache");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4470,7 +4566,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 UseDNSCache");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4483,7 +4579,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 NoPreferIPv6Automap");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4495,7 +4591,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 PreferSOCKSNoAuth");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4510,14 +4606,14 @@ test_config_parse_port_config__ports__ports_given(void *data)
   config_port_invalid = mock_config_line("DNSPort", "0");
   config_port_valid = mock_config_line("DNSPort", "42");
   config_port_invalid->next = config_port_valid;
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.42", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
   // Test success with warn non-local control
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "Control",
+  ret = port_parse_config(slout, config_port_valid, "Control",
                           CONN_TYPE_CONTROL_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4525,7 +4621,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local listener
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "ExtOR",
+  ret = port_parse_config(slout, config_port_valid, "ExtOR",
                           CONN_TYPE_EXT_OR_LISTENER, "127.0.0.42", 0,
                           CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
@@ -4533,12 +4629,12 @@ test_config_parse_port_config__ports__ports_given(void *data)
   // Test success with warn non-local other
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
   // Test success with warn non-local other without out
-  ret = parse_port_config(NULL, config_port_valid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_valid, "DNS", 0,
                           "127.0.0.42", 0, CL_PORT_WARN_NONLOCAL);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -4549,7 +4645,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 IPv4Traffic "
                                        "IPv6Traffic");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0,
                           CL_PORT_TAKES_HOSTNAMES |
                           CL_PORT_NO_STREAM_OPTIONS);
@@ -4564,7 +4660,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=invalid");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4574,7 +4670,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, CL_PORT_NO_STREAM_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4584,7 +4680,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "42 SessionGroup=123 "
                                          "SessionGroup=321");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4593,7 +4689,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "42 SessionGroup=1111122");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.44", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4605,7 +4701,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "0");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 0);
@@ -4615,7 +4711,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "something");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.45", 0, CL_PORT_IS_UNIXSOCKET);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4628,48 +4724,48 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "auto");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
   tor_addr_parse(&addr, "127.0.0.46");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test success with a port of auto in mixed case
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "AuTo");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
   tor_addr_parse(&addr, "127.0.0.46");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test success with parsing both an address and an auto port
   config_free_lines(config_port_valid); config_port_valid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.122:auto");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, CFG_AUTO_PORT);
   tor_addr_parse(&addr, "127.0.0.122");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test failure when asked to parse an invalid address followed by auto
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   config_port_invalid = mock_config_line("DNSPort", "invalidstuff!!:auto");
   MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
-  ret = parse_port_config(NULL, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(NULL, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
@@ -4679,21 +4775,21 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.123:656");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
   port_cfg = (port_cfg_t *)smartlist_get(slout, 0);
   tt_int_op(port_cfg->port, OP_EQ, 656);
   tor_addr_parse(&addr, "127.0.0.123");
-  tt_assert(tor_addr_eq(&port_cfg->addr, &addr))
+  tt_assert(tor_addr_eq(&port_cfg->addr, &addr));
 
   // Test failure if we can't parse anything at all
   config_free_lines(config_port_invalid); config_port_invalid = NULL;
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "something wrong");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4702,7 +4798,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.1.0:123:auto");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0,
                           "127.0.0.46", 0, 0);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4712,7 +4808,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("SOCKSPort", "unix:/tmp/somewhere");
-  ret = parse_port_config(slout, config_port_valid, "SOCKS",
+  ret = port_parse_config(slout, config_port_valid, "SOCKS",
                           CONN_TYPE_AP_LISTENER, "127.0.0.46", 0,
                           CL_PORT_DFLT_GROUP_WRITABLE);
 #ifdef _WIN32
@@ -4725,6 +4821,7 @@ test_config_parse_port_config__ports__ports_given(void *data)
 #endif /* defined(_WIN32) */
 
  done:
+  unmock_hostname_resolver();
   if (slout)
     SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_free(slout);
@@ -4747,7 +4844,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   config_free_lines(config_port_valid); config_port_valid = NULL;
   config_port_valid = mock_config_line("DNSPort",
                                        "127.0.0.124:656 NoAdvertise");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4760,7 +4857,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4774,7 +4871,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 NoListen "
                                          "NoAdvertise");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4783,7 +4880,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4796,7 +4893,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "[::1]:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4810,7 +4907,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "127.0.0.124:656 IPv6Only "
                                          "IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4819,7 +4916,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_valid = mock_config_line("DNSPort", "127.0.0.124:656 unknown");
-  ret = parse_port_config(slout, config_port_valid, "DNS", 0, NULL, 0,
+  ret = port_parse_config(slout, config_port_valid, "DNS", 0, NULL, 0,
                           CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(slout), OP_EQ, 1);
@@ -4830,7 +4927,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort",
                                          "127.0.0.124:656 IPv6Only");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4839,7 +4936,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("DNSPort", "[::1]:656 IPv4Only");
-  ret = parse_port_config(slout, config_port_invalid, "DNS", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "DNS", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -4848,7 +4945,7 @@ test_config_parse_port_config__ports__server_options(void *data)
   SMARTLIST_FOREACH(slout,port_cfg_t *,pf,port_cfg_free(pf));
   smartlist_clear(slout);
   config_port_invalid = mock_config_line("ORPort", "unix:\"\"");
-  ret = parse_port_config(slout, config_port_invalid, "ORPort", 0, NULL,
+  ret = port_parse_config(slout, config_port_invalid, "ORPort", 0, NULL,
                           0, CL_PORT_SERVER_OPTIONS);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -5128,7 +5225,7 @@ test_config_include_no_permission(void *data)
     chmod(dir, 0700);
   tor_free(dir);
 }
-#endif
+#endif /* !defined(_WIN32) */
 
 static void
 test_config_include_recursion_before_after(void *data)
@@ -5662,7 +5759,6 @@ test_config_check_bridge_distribution_setting_not_a_bridge(void *arg)
 {
   or_options_t* options = get_options_mutable();
   or_options_t* old_options = options;
-  or_options_t* default_options = options;
   char* message = NULL;
   int ret;
 
@@ -5671,7 +5767,7 @@ test_config_check_bridge_distribution_setting_not_a_bridge(void *arg)
   options->BridgeRelay = 0;
   options->BridgeDistribution = (char*)("https");
 
-  ret = options_validate(old_options, options, default_options, 0, &message);
+  ret = options_validate(old_options, options, &message);
 
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(message, OP_EQ, "You set BridgeDistribution, but you "
@@ -5843,7 +5939,7 @@ test_config_compute_max_mem_in_queues(void *data)
 #else
   /* We are on a 32-bit system. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ, GIGABYTE(1));
-#endif
+#endif /* SIZEOF_VOID_P >= 8 */
 
   /* We are able to detect the amount of RAM on the system. */
   total_system_memory_return = 0;
@@ -5884,7 +5980,7 @@ test_config_compute_max_mem_in_queues(void *data)
   /* We will at maximum get MAX_DEFAULT_MEMORY_QUEUE_SIZE here. */
   tt_u64_op(compute_real_max_mem_in_queues(0, 0), OP_EQ,
             MAX_DEFAULT_MEMORY_QUEUE_SIZE);
-#endif
+#endif /* SIZEOF_SIZE_T > 4 */
 
  done:
   UNMOCK(get_total_system_memory);
@@ -5928,6 +6024,7 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "is back here");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_NORMAL);
   lp = lp->next;
+  tt_assert(!lp);
   config_free_lines(lines);
 
   /* Try with the "extended" flag enabled. */
@@ -5954,11 +6051,198 @@ test_config_extended_fmt(void *arg)
   tt_str_op(lp->value, OP_EQ, "");
   tt_int_op(lp->command, OP_EQ, CONFIG_LINE_CLEAR);
   lp = lp->next;
+  tt_assert(!lp);
 
  done:
   config_free_lines(lines);
 }
 
+static void
+test_config_kvline_parse(void *arg)
+{
+  (void)arg;
+
+  config_line_t *lines = NULL;
+  char *enc = NULL;
+
+  lines = kvline_parse("A=B CD=EF", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "A");
+  tt_str_op(lines->value, OP_EQ, "B");
+  tt_str_op(lines->next->key, OP_EQ, "CD");
+  tt_str_op(lines->next->value, OP_EQ, "EF");
+  enc = kvline_encode(lines, 0);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "A=B CD=EF");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB CDE=F", 0);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB CDE=F", KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "");
+  tt_str_op(lines->value, OP_EQ, "AB");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F");
+  tt_assert(lines);
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB CDE=F");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\"", 0);
+  tt_assert(!lines);
+
+  lines = kvline_parse("AB=C CDE=\"F G\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "C");
+  tt_str_op(lines->next->key, OP_EQ, "CDE");
+  tt_str_op(lines->next->value, OP_EQ, "F G");
+  tt_str_op(lines->next->next->key, OP_EQ, "");
+  tt_str_op(lines->next->next->value, OP_EQ, "GHI");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_QUOTED|KV_OMIT_KEYS);
+  tt_str_op(enc, OP_EQ, "AB=C CDE=\"F G\" GHI");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("A\"B=C CDE=\"F\" \"GHI\" ", KV_QUOTED|KV_OMIT_KEYS);
+  tt_assert(! lines);
+
+  lines = kvline_parse("AB=", KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", 0);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse(" AB ", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD DE FGH=I", KV_OMIT_VALS);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS);
+  tt_str_op(enc, OP_EQ, "AB=CD DE FGH=I");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=\"CD E\" DE FGH=\"I\"", KV_OMIT_VALS|KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD E");
+  tt_str_op(lines->next->key, OP_EQ, "DE");
+  tt_str_op(lines->next->value, OP_EQ, "");
+  tt_str_op(lines->next->next->key, OP_EQ, "FGH");
+  tt_str_op(lines->next->next->value, OP_EQ, "I");
+  enc = kvline_encode(lines, KV_OMIT_VALS|KV_QUOTED);
+  tt_str_op(enc, OP_EQ, "AB=\"CD E\" DE FGH=I");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = kvline_parse("AB=CD \"EF=GH\"", KV_OMIT_KEYS|KV_QUOTED);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "AB");
+  tt_str_op(lines->value, OP_EQ, "CD");
+  tt_str_op(lines->next->key, OP_EQ, "");
+  tt_str_op(lines->next->value, OP_EQ, "EF=GH");
+  enc = kvline_encode(lines, KV_OMIT_KEYS);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_OMIT_KEYS|KV_QUOTED);
+  tt_assert(enc);
+  tt_str_op(enc, OP_EQ, "AB=CD \"EF=GH\"");
+  tor_free(enc);
+  config_free_lines(lines);
+
+  lines = tor_malloc_zero(sizeof(*lines));
+  lines->key = tor_strdup("A=B");
+  lines->value = tor_strdup("CD");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  config_free_lines(lines);
+
+  config_line_append(&lines, "A", "B C");
+  enc = kvline_encode(lines, 0);
+  tt_assert(!enc);
+  enc = kvline_encode(lines, KV_RAW);
+  tt_assert(enc);
+  tt_str_op(enc, OP_EQ, "A=B C");
+
+ done:
+  config_free_lines(lines);
+  tor_free(enc);
+}
+
+static void
+test_config_getinfo_config_names(void *arg)
+{
+  (void)arg;
+  char *answer = NULL;
+  const char *error = NULL;
+  int rv;
+
+  rv = getinfo_helper_config(NULL, "config/names", &answer, &error);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_ptr_op(error, OP_EQ, NULL);
+
+  // ContactInfo should be listed.
+  tt_assert(strstr(answer, "\nContactInfo String\n"));
+
+  // V1AuthoritativeDirectory should not be listed, since it is obsolete.
+  tt_assert(! strstr(answer, "V1AuthoritativeDirectory"));
+
+  // ___UsingTestNetworkDefaults should not be listed, since it is invisible.
+  tt_assert(! strstr(answer, "UsingTestNetworkDefaults"));
+
+ done:
+  tor_free(answer);
+}
+
 #define CONFIG_TEST(name, flags)                          \
   { #name, test_config_ ## name, flags, NULL, NULL }
 
@@ -5977,6 +6261,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(parse_bridge_line, 0),
   CONFIG_TEST(parse_transport_options_line, 0),
   CONFIG_TEST(parse_transport_plugin_line, TT_FORK),
+  CONFIG_TEST(parse_tcp_proxy_line, TT_FORK),
   CONFIG_TEST(check_or_create_data_subdir, TT_FORK),
   CONFIG_TEST(write_to_data_subdir, TT_FORK),
   CONFIG_TEST(fix_my_family, 0),
@@ -6012,5 +6297,7 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_opened_file_list, 0),
   CONFIG_TEST(compute_max_mem_in_queues, 0),
   CONFIG_TEST(extended_fmt, 0),
+  CONFIG_TEST(kvline_parse, 0),
+  CONFIG_TEST(getinfo_config_names, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_confmgr.c b/src/test/test_confmgr.c
new file mode 100644
index 0000000000..a647b92e0a
--- /dev/null
+++ b/src/test/test_confmgr.c
@@ -0,0 +1,499 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confmgt.c's features that support multiple configuration
+ * formats and configuration objects.
+ */
+
+#define CONFMGT_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "lib/confmgt/confmgt.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+/*
+ * Set up a few objects: a pasture_cfg is toplevel; it has a llama_cfg and an
+ * alpaca_cfg.
+ */
+
+typedef struct {
+  uint32_t magic;
+  char *address;
+  int opentopublic;
+  config_suite_t *subobjs;
+} pasture_cfg_t;
+
+typedef struct {
+  char *llamaname;
+  int cuteness;
+  uint32_t magic;
+  int eats_meat; /* deprecated; llamas are never carnivorous. */
+
+  char *description; // derived from other fields.
+} llama_cfg_t;
+
+typedef struct {
+  uint32_t magic;
+  int fuzziness;
+  char *alpacaname;
+  int n_wings; /* deprecated; alpacas don't have wings. */
+
+  int square_fuzziness; /* Derived from fuzziness. */
+} alpaca_cfg_t;
+
+/*
+ * Make the above into configuration objects.
+ */
+
+static pasture_cfg_t pasture_cfg_t_dummy;
+static llama_cfg_t llama_cfg_t_dummy;
+static alpaca_cfg_t alpaca_cfg_t_dummy;
+
+#define PV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(pasture_cfg_t, #name, type, name, 0, dflt)
+#define LV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(llama_cfg_t, #name, type, name, 0, dflt)
+#define AV(name, type, dflt) \
+  CONFIG_VAR_ETYPE(alpaca_cfg_t, #name, type, name, 0, dflt)
+static const config_var_t pasture_vars[] = {
+  PV(address, STRING, NULL),
+  PV(opentopublic, BOOL, "1"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t llama_vars[] =
+{
+  LV(llamaname, STRING, NULL),
+  LV(eats_meat, BOOL, NULL),
+  LV(cuteness, POSINT, "100"),
+  END_OF_CONFIG_VARS
+};
+static const config_var_t alpaca_vars[] =
+{
+  AV(alpacaname, STRING, NULL),
+  AV(fuzziness, POSINT, "50"),
+  AV(n_wings, POSINT, "0"),
+  END_OF_CONFIG_VARS
+};
+
+static config_deprecation_t llama_deprecations[] = {
+  { "eats_meat", "Llamas are herbivores." },
+  {NULL,NULL}
+};
+
+static config_deprecation_t alpaca_deprecations[] = {
+  { "n_wings", "Alpacas are quadrupeds." },
+  {NULL,NULL}
+};
+
+static int clear_llama_cfg_called = 0;
+static void
+clear_llama_cfg(const config_mgr_t *mgr, void *llamacfg)
+{
+  (void)mgr;
+  llama_cfg_t *lc = llamacfg;
+  tor_free(lc->description);
+  ++clear_llama_cfg_called;
+}
+
+static config_abbrev_t llama_abbrevs[] = {
+  { "gracia", "cuteness", 0, 0 },
+  { "gentillesse", "cuteness", 0, 0 },
+  { NULL, NULL, 0, 0 },
+};
+
+static int
+legacy_validate_pasture(const void *old_, void *obj, char **msg_out)
+{
+  const pasture_cfg_t *old = old_;
+  pasture_cfg_t *p = obj;
+
+  // llamas can't find their way home if the letters are lowercase.
+  if (p->address)
+    tor_strupper(p->address);
+
+  if (old && old->address &&
+      (!p->address || strcmp(old->address, p->address))) {
+    *msg_out = tor_strdup("You can't move a pasture.");
+    return -1;
+  }
+
+  return 0;
+}
+
+static int
+validate_llama(const void *obj, char **msg_out)
+{
+  const llama_cfg_t *llama = obj;
+  tor_assert(llama->magic == 0x11aa11);
+
+  if (! llama->llamaname || strlen(llama->llamaname) == 0) {
+    *msg_out = tor_strdup("A llama has no name!?");
+    return -1;
+  }
+
+  if (strspn(llama->llamaname, "0123456789") == strlen(llama->llamaname)) {
+    *msg_out = tor_strdup("It is not a number; it is a free llama!");
+    return -1;
+  }
+
+  return 0;
+}
+
+static int
+check_transition_alpaca(const void *old_, const void *new_, char **msg_out)
+{
+  const alpaca_cfg_t *old_alpaca = old_;
+  const alpaca_cfg_t *new_alpaca = new_;
+
+  tor_assert(old_alpaca && new_alpaca);
+  tor_assert(old_alpaca->magic == 0xa15aca);
+  tor_assert(new_alpaca->magic == 0xa15aca);
+
+  if (old_alpaca->fuzziness > new_alpaca->fuzziness) {
+    *msg_out = tor_strdup("An alpaca only becomes more fuzzy over time.");
+    return -1;
+  }
+
+  return 0;
+}
+
+static int
+post_normalize_llama(void *obj, char **msg_out)
+{
+  (void)msg_out;
+  llama_cfg_t *llama = obj;
+  tor_assert(llama->magic == 0x11aa11);
+  tor_assert(llama->llamaname); // we have already checked for a NULL name.
+  tor_free(llama->description);
+  tor_asprintf(&llama->description, "A llama called %s.", llama->llamaname);
+  return 0;
+}
+
+static int
+pre_normalize_alpaca(void *obj, char **msg_out)
+{
+  (void)msg_out;
+  alpaca_cfg_t *alpaca = obj;
+  tor_assert(alpaca->magic == 0xa15aca);
+  alpaca->square_fuzziness = alpaca->fuzziness * alpaca->fuzziness;
+  return 0;
+}
+
+static const config_format_t pasture_fmt = {
+  sizeof(pasture_cfg_t),
+  {
+    "pasture_cfg_t",
+    8989,
+    offsetof(pasture_cfg_t, magic)
+  },
+  .vars = pasture_vars,
+  .has_config_suite = true,
+  .config_suite_offset = offsetof(pasture_cfg_t, subobjs),
+  .legacy_validate_fn = legacy_validate_pasture,
+};
+
+static const config_format_t llama_fmt = {
+  sizeof(llama_cfg_t),
+  {
+    "llama_cfg_t",
+    0x11aa11,
+    offsetof(llama_cfg_t, magic)
+  },
+  .vars = llama_vars,
+  .deprecations = llama_deprecations,
+  .abbrevs = llama_abbrevs,
+  .clear_fn = clear_llama_cfg,
+  .validate_fn = validate_llama,
+  .post_normalize_fn = post_normalize_llama,
+};
+
+static const config_format_t alpaca_fmt = {
+  sizeof(alpaca_cfg_t),
+  {
+    "alpaca_cfg_t",
+    0xa15aca,
+    offsetof(alpaca_cfg_t, magic)
+  },
+  .vars = alpaca_vars,
+  .deprecations = alpaca_deprecations,
+  .pre_normalize_fn = pre_normalize_alpaca,
+  .check_transition_fn = check_transition_alpaca,
+};
+
+#define LLAMA_IDX 0
+#define ALPACA_IDX 1
+
+static config_mgr_t *
+get_mgr(bool freeze)
+{
+  config_mgr_t *mgr = config_mgr_new(&pasture_fmt);
+  tt_int_op(LLAMA_IDX, OP_EQ, config_mgr_add_format(mgr, &llama_fmt));
+  tt_int_op(ALPACA_IDX, OP_EQ, config_mgr_add_format(mgr, &alpaca_fmt));
+  if (freeze)
+    config_mgr_freeze(mgr);
+  return mgr;
+
+ done:
+  config_mgr_free(mgr);
+  return NULL;
+}
+
+static void
+test_confmgr_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  smartlist_t *vars = NULL;
+  tt_ptr_op(mgr, OP_NE, NULL);
+
+  vars = config_mgr_list_vars(mgr);
+  tt_int_op(smartlist_len(vars), OP_EQ, 8); // 8 vars total.
+
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "CUTENESS"));
+  tt_str_op("cuteness", OP_EQ, config_find_option_name(mgr, "GRACIA"));
+  smartlist_free(vars);
+
+  vars = config_mgr_list_deprecated_vars(mgr); // 2 deprecated vars.
+  tt_int_op(smartlist_len(vars), OP_EQ, 2);
+  tt_assert(smartlist_contains_string(vars, "eats_meat"));
+  tt_assert(smartlist_contains_string(vars, "n_wings"));
+
+  tt_str_op("Llamas are herbivores.", OP_EQ,
+            config_find_deprecation(mgr, "EATS_MEAT"));
+  tt_str_op("Alpacas are quadrupeds.", OP_EQ,
+            config_find_deprecation(mgr, "N_WINGS"));
+
+ done:
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confmgr_magic(void *args)
+{
+  (void)args;
+  // Every time we build a manager, it is supposed to get a different magic
+  // number.  Let's test that.
+  config_mgr_t *mgr1 = get_mgr(true);
+  config_mgr_t *mgr2 = get_mgr(true);
+  config_mgr_t *mgr3 = get_mgr(true);
+
+  pasture_cfg_t *p1 = NULL, *p2 = NULL, *p3 = NULL;
+
+  tt_assert(mgr1);
+  tt_assert(mgr2);
+  tt_assert(mgr3);
+
+  p1 = config_new(mgr1);
+  p2 = config_new(mgr2);
+  p3 = config_new(mgr3);
+
+  tt_assert(p1);
+  tt_assert(p2);
+  tt_assert(p3);
+
+  // By chance, two managers get the same magic with P=2^-32.  Let's
+  // make sure that at least two of them are different, so that our
+  // odds of a false positive are 1/2^-64.
+  tt_assert((p1->magic != p2->magic) || (p2->magic != p3->magic));
+
+ done:
+  config_free(mgr1, p1);
+  config_free(mgr2, p2);
+  config_free(mgr3, p3);
+
+  config_mgr_free(mgr1);
+  config_mgr_free(mgr2);
+  config_mgr_free(mgr3);
+}
+
+static const char *simple_pasture =
+  "LLamaname hugo\n"
+  "Alpacaname daphne\n"
+  "gentillesse 42\n"
+  "address 123 Camelid ave\n";
+
+static void
+test_confmgr_parse(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, p); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_int_op(p->opentopublic, OP_EQ, 1);
+  tt_str_op(p->address, OP_EQ, "123 Camelid ave");
+
+  // We are using this API directly; modules outside confparse will, in the
+  // future, not.
+  const alpaca_cfg_t *ac = config_mgr_get_obj(mgr, p, ALPACA_IDX);
+  const llama_cfg_t *lc = config_mgr_get_obj(mgr, p, LLAMA_IDX);
+  tt_str_op(lc->llamaname, OP_EQ, "hugo");
+  tt_str_op(ac->alpacaname, OP_EQ, "daphne");
+  tt_int_op(lc->cuteness, OP_EQ, 42);
+  tt_int_op(ac->fuzziness, OP_EQ, 50);
+
+  // We set the description for the llama here, so that the clear function
+  // can clear it.  (Later we can do this in a verification function.)
+  clear_llama_cfg_called = 0;
+  llama_cfg_t *mut_lc = config_mgr_get_obj_mutable(mgr, p, LLAMA_IDX);
+  mut_lc->description = tor_strdup("A llama named Hugo.");
+  config_free(mgr, p);
+  tt_int_op(clear_llama_cfg_called, OP_EQ, 1);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_mgr_free(mgr);
+  tor_free(msg);
+}
+
+static void
+test_confmgr_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p = config_new(mgr);
+  pasture_cfg_t *defaults = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+  char *s = NULL;
+
+  config_init(mgr, p); // set defaults.
+  config_init(mgr, defaults); // set defaults.
+
+  int r = config_get_lines(simple_pasture, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  s = config_dump(mgr, defaults, p, 1, 0);
+  tt_str_op("address 123 Camelid ave\n"
+            "alpacaname daphne\n"
+            "cuteness 42\n"
+            "llamaname hugo\n", OP_EQ, s);
+
+ done:
+  config_free_lines(lines);
+  config_free(mgr, p);
+  config_free(mgr, defaults);
+  config_mgr_free(mgr);
+
+  tor_free(msg);
+  tor_free(s);
+}
+
+static pasture_cfg_t *
+parse_and_validate(config_mgr_t *mgr,
+                   const char *inp, const pasture_cfg_t *old, char **msg_out)
+{
+  pasture_cfg_t *p = config_new(mgr);
+  pasture_cfg_t *result = NULL;
+  config_line_t *lines = NULL;
+
+  config_init(mgr, p); // set defaults.
+  int r = config_get_lines(inp, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, p, lines, 0, msg_out);
+  tt_int_op(r, OP_EQ, 0);
+  tor_free(*msg_out); // sets it to NULL
+  r = config_validate(mgr, old, p, msg_out);
+  if (r < 0)
+    goto done;
+
+  tt_ptr_op(*msg_out, OP_EQ, NULL);
+  result = p;
+  p = NULL; // prevent free
+ done:
+  config_free(mgr, p);
+  config_free_lines(lines);
+  return result;
+}
+
+static void
+test_confmgr_validate(void *arg)
+{
+  (void)arg;
+  char *msg = NULL;
+  config_mgr_t *mgr = get_mgr(true);
+  pasture_cfg_t *p_orig, *p=NULL;
+
+  p_orig = parse_and_validate(mgr, "Llamaname Quest\n"
+                                   "Address 99 camelid way\n"
+                                   "Fuzziness 8\n", NULL, &msg);
+  tt_assert(p_orig);
+
+  // Make sure normalization code was run.
+  const alpaca_cfg_t *ac0 = config_mgr_get_obj(mgr, p_orig, ALPACA_IDX);
+  const llama_cfg_t *lc0 = config_mgr_get_obj(mgr, p_orig, LLAMA_IDX);
+  tt_int_op(ac0->fuzziness, OP_EQ, 8);
+  tt_int_op(ac0->square_fuzziness, OP_EQ, 64);
+  tt_str_op(lc0->description, OP_EQ, "A llama called Quest.");
+  tt_str_op(p_orig->address, OP_EQ, "99 CAMELID WAY");
+
+  // try a bad llamaname.
+  p = parse_and_validate(mgr, "llamaname 123", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "It is not a number; it is a free llama!");
+  tor_free(msg);
+
+  // try a llamaname that would crash the post_normalize step, if it ran.
+  p = parse_and_validate(mgr, "", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "A llama has no name!?");
+  tor_free(msg);
+
+  // Verify that a transition to a less fuzzy alpaca fails.
+  p = parse_and_validate(mgr, "Llamaname Quest\n"
+                              "Address 99 camelid way\n"
+                              "Fuzziness 4\n", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "An alpaca only becomes more fuzzy over time.");
+  tor_free(msg);
+
+  // Try a transition to a more fuzzy alpaca; it should work fine.
+  p = parse_and_validate(mgr, "Llamaname Mercutio\n"
+                              // the default fuzziness is 50
+                              "Address 99 camelid way\n", p_orig, &msg);
+  tt_assert(p);
+  config_free(mgr, p);
+
+  // Verify that we can't move the pasture.
+  p = parse_and_validate(mgr, "Llamaname Montague\n"
+                              // the default fuzziness is 50
+                              "Address 99 ungulate st\n", p_orig, &msg);
+  tt_assert(!p);
+  tt_str_op(msg, OP_EQ, "You can't move a pasture.");
+
+ done:
+  config_free(mgr, p);
+  config_free(mgr, p_orig);
+  config_mgr_free(mgr);
+  tor_free(msg);
+}
+
+#define CONFMGR_TEST(name, flags)                       \
+  { #name, test_confmgr_ ## name, flags, NULL, NULL }
+
+struct testcase_t confmgr_tests[] = {
+  CONFMGR_TEST(init, 0),
+  CONFMGR_TEST(magic, 0),
+  CONFMGR_TEST(parse, 0),
+  CONFMGR_TEST(dump, 0),
+  CONFMGR_TEST(validate, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_confparse.c b/src/test/test_confparse.c
new file mode 100644
index 0000000000..21301ce75e
--- /dev/null
+++ b/src/test/test_confparse.c
@@ -0,0 +1,1091 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Tests for confmgt.c module that we use to parse various
+ * configuration/state file types.
+ */
+
+#define CONFMGT_PRIVATE
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "lib/encoding/confline.h"
+#include "feature/nodelist/routerset.h"
+#include "lib/confmgt/confmgt.h"
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+#include "lib/confmgt/unitparse.h"
+
+typedef struct test_struct_t {
+  uint32_t magic;
+  char *s;
+  char *fn;
+  int pos;
+  int i;
+  int deprecated_int;
+  uint64_t u64;
+  int interval;
+  int msec_interval;
+  uint64_t mem;
+  double dbl;
+  int boolean;
+  int autobool;
+  time_t time;
+  smartlist_t *csv;
+  int csv_interval;
+  config_line_t *lines;
+  config_line_t *mixed_lines;
+  routerset_t *routerset;
+  int hidden_int;
+  config_line_t *mixed_hidden_lines;
+
+  config_line_t *extra_lines;
+} test_struct_t;
+
+static test_struct_t test_struct_t_dummy;
+
+#define VAR(varname,conftype,member,initvalue)                          \
+  CONFIG_VAR_ETYPE(test_struct_t, varname, conftype, member, 0, initvalue)
+#define V(member,conftype,initvalue)            \
+  VAR(#member, conftype, member, initvalue)
+#define OBSOLETE(varname)                       \
+  CONFIG_VAR_OBSOLETE(varname)
+
+static const config_var_t test_vars[] = {
+  V(s, STRING, "hello"),
+  V(fn, FILENAME, NULL),
+  V(pos, POSINT, NULL),
+  V(i, INT, "-10"),
+  V(deprecated_int, INT, "3"),
+  V(u64, UINT64, NULL),
+  V(interval, INTERVAL, "10 seconds"),
+  V(msec_interval, MSEC_INTERVAL, "150 msec"),
+  V(mem, MEMUNIT, "10 MB"),
+  V(dbl, DOUBLE, NULL),
+  V(boolean, BOOL, "0"),
+  V(autobool, AUTOBOOL, "auto"),
+  V(time, ISOTIME, NULL),
+  V(csv, CSV, NULL),
+  V(csv_interval, CSV_INTERVAL, "5 seconds"),
+  V(lines, LINELIST, NULL),
+  VAR("MixedLines", LINELIST_V, mixed_lines, NULL),
+  VAR("LineTypeA", LINELIST_S, mixed_lines, NULL),
+  VAR("LineTypeB", LINELIST_S, mixed_lines, NULL),
+  OBSOLETE("obsolete"),
+  {
+   .member = { .name = "routerset",
+               .type = CONFIG_TYPE_EXTENDED,
+               .type_def = &ROUTERSET_type_defn,
+               .offset = offsetof(test_struct_t, routerset),
+             },
+  },
+  VAR("__HiddenInt", POSINT, hidden_int, "0"),
+  VAR("MixedHiddenLines", LINELIST_V, mixed_hidden_lines, NULL),
+  VAR("__HiddenLineA", LINELIST_S, mixed_hidden_lines, NULL),
+  VAR("VisibleLineB", LINELIST_S, mixed_hidden_lines, NULL),
+
+  END_OF_CONFIG_VARS,
+};
+
+static config_abbrev_t test_abbrevs[] = {
+  { "uint", "pos", 0, 0 },
+  { "float", "dbl", 0, 1 },
+  { NULL, NULL, 0, 0 }
+};
+
+static config_deprecation_t test_deprecation_notes[] = {
+  { "deprecated_int", "This integer is deprecated." },
+  { NULL, NULL }
+};
+
+static int
+test_validate_cb(const void *old_options, void *options, char **msg)
+{
+  (void)old_options;
+  (void)msg;
+  test_struct_t *ts = options;
+
+  if (ts->i == 0xbad) {
+    *msg = tor_strdup("bad value for i");
+    return -1;
+  }
+  return 0;
+}
+
+#define TEST_MAGIC 0x1337
+
+static const config_format_t test_fmt = {
+  .size = sizeof(test_struct_t),
+  .magic = {
+   "test_struct_t",
+   TEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  .abbrevs = test_abbrevs,
+  .deprecations = test_deprecation_notes,
+  .vars = test_vars,
+  .legacy_validate_fn = test_validate_cb,
+};
+
+/* Make sure that config_init sets everything to the right defaults. */
+static void
+test_confparse_init(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_init(mgr, tst);
+
+  // Make sure that options are initialized right. */
+  tt_str_op(tst->s, OP_EQ, "hello");
+  tt_ptr_op(tst->fn, OP_EQ, NULL);
+  tt_int_op(tst->pos, OP_EQ, 0);
+  tt_int_op(tst->i, OP_EQ, -10);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, 0);
+  tt_int_op(tst->interval, OP_EQ, 10);
+  tt_int_op(tst->msec_interval, OP_EQ, 150);
+  tt_u64_op(tst->mem, OP_EQ, 10 * 1024 * 1024);
+  tt_double_op(tst->dbl, OP_LT, .0000000001);
+  tt_double_op(tst->dbl, OP_GT, -0.0000000001);
+  tt_int_op(tst->boolean, OP_EQ, 0);
+  tt_int_op(tst->autobool, OP_EQ, -1);
+  tt_i64_op(tst->time, OP_EQ, 0);
+  tt_ptr_op(tst->csv, OP_EQ, NULL);
+  tt_int_op(tst->csv_interval, OP_EQ, 5);
+  tt_ptr_op(tst->lines, OP_EQ, NULL);
+  tt_ptr_op(tst->mixed_lines, OP_EQ, NULL);
+  tt_int_op(tst->hidden_int, OP_EQ, 0);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static const char simple_settings[] =
+      "s this is a \n"
+      "fn /simple/test of the\n"
+      "uint 77\n" // this is an abbrev
+      "i 3\n"
+      "u64   1000000000000  \n"
+      "interval 5 minutes \n"
+      "msec_interval 5 minutes \n"
+      "mem 10\n"
+      "dbl 6.060842\n"
+      "BOOLEAN 1\n"
+      "aUtObOOl 0\n"
+      "time 2019-06-14 13:58:51\n"
+      "csv configuration, parsing  , system  \n"
+      "csv_interval 10 seconds, 5 seconds, 10 hours\n"
+      "lines hello\n"
+      "LINES world\n"
+      "linetypea i d\n"
+      "linetypeb i c\n"
+      "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+      "__hiddenint 11\n"
+      "__hiddenlineA XYZ\n"
+      "visiblelineB ABC\n";
+
+/* Return a configuration object set up from simple_settings above. */
+static test_struct_t *
+get_simple_config(const config_mgr_t *mgr)
+{
+  test_struct_t *result = NULL;
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(simple_settings, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  result = tst;
+  tst = NULL; // prevent free
+ done:
+  tor_free(msg);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  return result;
+}
+
+/* Make sure that config_assign can parse things. */
+static void
+test_confparse_assign_simple(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  tt_str_op(tst->s, OP_EQ, "this is a");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the");
+  tt_int_op(tst->pos, OP_EQ, 77);
+  tt_int_op(tst->i, OP_EQ, 3);
+  tt_int_op(tst->deprecated_int, OP_EQ, 3);
+  tt_u64_op(tst->u64, OP_EQ, UINT64_C(1000000000000));
+  tt_int_op(tst->interval, OP_EQ, 5 * 60);
+  tt_int_op(tst->msec_interval, OP_EQ, 5 * 60 * 1000);
+  tt_u64_op(tst->mem, OP_EQ, 10);
+  tt_double_op(tst->dbl, OP_LT, 6.060843);
+  tt_double_op(tst->dbl, OP_GT, 6.060841);
+  tt_int_op(tst->boolean, OP_EQ, 1);
+  tt_int_op(tst->autobool, OP_EQ, 0);
+  tt_i64_op(tst->time, OP_EQ, 1560520731);
+  tt_ptr_op(tst->csv, OP_NE, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 3);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "configuration");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "parsing");
+  tt_str_op(smartlist_get(tst->csv, 2), OP_EQ, "system");
+  tt_int_op(tst->csv_interval, OP_EQ, 10);
+  tt_int_op(tst->hidden_int, OP_EQ, 11);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(!tst->lines->next->next);
+
+  tt_assert(tst->mixed_lines);
+  tt_str_op(tst->mixed_lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(tst->mixed_lines->value, OP_EQ, "i d");
+  tt_assert(tst->mixed_lines->next);
+  tt_str_op(tst->mixed_lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(tst->mixed_lines->next->value, OP_EQ, "i c");
+  tt_assert(!tst->mixed_lines->next->next);
+
+  tt_assert(tst->mixed_hidden_lines);
+  tt_str_op(tst->mixed_hidden_lines->key, OP_EQ, "__HiddenLineA");
+  tt_str_op(tst->mixed_hidden_lines->value, OP_EQ, "XYZ");
+  tt_assert(tst->mixed_hidden_lines->next);
+  tt_str_op(tst->mixed_hidden_lines->next->key, OP_EQ, "VisibleLineB");
+  tt_str_op(tst->mixed_hidden_lines->next->value, OP_EQ, "ABC");
+  tt_assert(!tst->mixed_hidden_lines->next->next);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an obsolete option, and make sure we get a warning. */
+static void
+test_confparse_assign_obsolete(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("obsolete option here",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("Skipping obsolete configuration option");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to assign to an deprecated option, and make sure we get a warning
+ * but the assignment works anyway. */
+static void
+test_confparse_assign_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("deprecated_int 7",
+                           &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("This integer is deprecated.");
+
+  tt_int_op(tst->deprecated_int, OP_EQ, 7);
+
+  tt_assert(config_check_ok(mgr, tst, LOG_ERR));
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to re-assign an option name that has been depreacted in favor of
+ * another. */
+static void
+test_confparse_assign_replaced(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("float 1000\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, CAL_WARN_DEPRECATIONS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("use 'dbl' instead.");
+
+  tt_double_op(tst->dbl, OP_GT, 999.999);
+  tt_double_op(tst->dbl, OP_LT, 1000.001);
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a linelist value with no option. */
+static void
+test_confparse_assign_emptystring(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("lines\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("has no value");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Try to set a the same option twice; make sure we get a warning. */
+static void
+test_confparse_assign_twice(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines("pos 10\n"
+                           "pos 99\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  expect_single_log_msg_containing("used more than once");
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+typedef struct badval_test_t {
+  const char *cfg;
+  const char *expect_msg;
+} badval_test_t;
+
+/* Try to set an option and make sure that we get a failure and an expected
+ * warning. */
+static void
+test_confparse_assign_badval(void *arg)
+{
+  const badval_test_t *bt = arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(bt->cfg, &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  setup_capture_of_logs(LOG_WARN);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_ptr_op(msg, OP_NE, NULL);
+  if (! strstr(msg, bt->expect_msg)) {
+    TT_DIE(("'%s' did not contain '%s'" , msg, bt->expect_msg));
+  }
+
+ done:
+  teardown_capture_of_logs();
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Various arguments for badval test.
+ *
+ * Note that the expected warnings here are _very_ truncated, since we
+ * are writing these tests before a refactoring that we expect will
+ * change them.
+ */
+static const badval_test_t bv_notint = { "pos X\n", "malformed" };
+static const badval_test_t bv_negint = { "pos -10\n", "out of bounds" };
+static const badval_test_t bv_badu64 = { "u64 u64\n", "malformed" };
+static const badval_test_t bv_dbl1 = { "dbl xxx\n", "Could not convert" };
+static const badval_test_t bv_dbl2 = { "dbl 1.0 xx\n", "Could not convert" };
+static const badval_test_t bv_dbl3 = {
+   "dbl 1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl4 = {
+   "dbl 1e1000\n", "too large to express" };
+static const badval_test_t bv_dbl5 = {
+   "dbl -1e-10000\n", "too small to express" };
+static const badval_test_t bv_dbl6 = {
+   "dbl -1e1000\n", "too large to express" };
+static const badval_test_t bv_badcsvi1 =
+  { "csv_interval 10 wl\n", "malformed" };
+static const badval_test_t bv_badcsvi2 =
+  { "csv_interval cl,10\n", "malformed" };
+static const badval_test_t bv_nonoption = { "fnord 10\n", "Unknown option" };
+static const badval_test_t bv_badmem = { "mem 3 trits\n", "malformed" };
+static const badval_test_t bv_badbool = { "boolean 7\n", "Unrecognized value"};
+static const badval_test_t bv_badabool =
+  { "autobool 7\n", "Unrecognized value" };
+static const badval_test_t bv_badtime = { "time lunchtime\n", "Invalid time" };
+static const badval_test_t bv_virt = { "MixedLines 7\n", "virtual option" };
+static const badval_test_t bv_rs = { "Routerset 2.2.2.2.2\n", "Invalid" };
+static const badval_test_t bv_big_interval =
+  { "interval 1000 months", "too large" };
+
+/* Try config_dump(), and make sure it behaves correctly */
+static void
+test_confparse_dump(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  char *dumped = NULL;
+
+  /* Minimal version. */
+  dumped = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 0);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+  /* commented */
+  tor_free(dumped);
+  dumped = config_dump(mgr, NULL, tst, 0, 1);
+  tt_str_op(dumped, OP_EQ,
+            "autobool 0\n"
+            "boolean 1\n"
+            "csv configuration,parsing,system\n"
+            "csv_interval 10\n"
+            "dbl 6.060842\n"
+            "# deprecated_int 3\n"
+            "fn /simple/test of the\n"
+            "i 3\n"
+            "interval 300\n"
+            "lines hello\n"
+            "lines world\n"
+            "mem 10\n"
+            "VisibleLineB ABC\n"
+            "LineTypeA i d\n"
+            "LineTypeB i c\n"
+            "msec_interval 300000\n"
+            "pos 77\n"
+            "routerset $FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF\n"
+            "s this is a\n"
+            "time 2019-06-14 13:58:51\n"
+            "u64 1000000000000\n");
+
+ done:
+  config_free(mgr, tst);
+  tor_free(dumped);
+  config_mgr_free(mgr);
+}
+
+/* Try confparse_reset_line(), and make sure it behaves correctly */
+static void
+test_confparse_reset(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+
+  config_reset_line(mgr, tst, "interval", 0);
+  tt_int_op(tst->interval, OP_EQ, 0);
+
+  config_reset_line(mgr, tst, "interval", 1);
+  tt_int_op(tst->interval, OP_EQ, 10);
+
+  tt_ptr_op(tst->routerset, OP_NE, NULL);
+  config_reset_line(mgr, tst, "routerset", 0);
+  tt_ptr_op(tst->routerset, OP_EQ, NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, and make sure
+ * it behaves correctly. */
+static void
+test_confparse_reassign(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *rs = NULL;
+
+  int r = config_get_lines(
+         "s eleven\n"
+         "i 12\n"
+         "lines 13\n"
+         "csv 14,15\n"
+         "routerset 127.0.0.1\n",
+         &lines, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  tt_str_op(tst->fn, OP_EQ, "/simple/test of the"); // unchanged
+  tt_int_op(tst->pos, OP_EQ, 77); // unchanged
+  tt_int_op(tst->i, OP_EQ, 12);
+  tt_ptr_op(tst->lines, OP_NE, NULL);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "13");
+  tt_ptr_op(tst->lines->next, OP_EQ, NULL);
+  tt_int_op(smartlist_len(tst->csv), OP_EQ, 2);
+  tt_str_op(smartlist_get(tst->csv, 0), OP_EQ, "14");
+  tt_str_op(smartlist_get(tst->csv, 1), OP_EQ, "15");
+
+  rs = routerset_to_string(tst->routerset);
+  tt_str_op(rs, OP_EQ, "127.0.0.1");
+
+  // Try again with the CLEAR_FIRST and USE_DEFAULTS flags
+  r = config_assign(mgr, tst, lines,
+                    CAL_CLEAR_FIRST|CAL_USE_DEFAULTS, &msg);
+  tt_int_op(r, OP_EQ, 0);
+
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_str_op(tst->s, OP_EQ, "eleven");
+  // tt_ptr_op(tst->fn, OP_EQ, NULL); //XXXX why is this not cleared?
+  // tt_int_op(tst->pos, OP_EQ, 0); //XXXX why is this not cleared?
+  tt_int_op(tst->i, OP_EQ, 12);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  tor_free(rs);
+  config_mgr_free(mgr);
+}
+
+/* Try setting options a second time on a config object, using the +foo
+ * linelist-extending syntax. */
+static void
+test_confparse_reassign_extend(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+
+  int r = config_get_lines(
+         "+lines 13\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst,lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->lines);
+  tt_str_op(tst->lines->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->value, OP_EQ, "hello");
+  tt_assert(tst->lines->next);
+  tt_str_op(tst->lines->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->value, OP_EQ, "world");
+  tt_assert(tst->lines->next->next);
+  tt_str_op(tst->lines->next->next->key, OP_EQ, "lines");
+  tt_str_op(tst->lines->next->next->value, OP_EQ, "13");
+  tt_assert(tst->lines->next->next->next == NULL);
+  config_free_lines(lines);
+
+  r = config_get_lines(
+         "/lines\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+  config_free_lines(lines);
+
+  config_free(mgr, tst);
+  tst = get_simple_config(mgr);
+  r = config_get_lines(
+         "/lines away!\n",
+         &lines, 1); // allow extended format.
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_assert(tst->lines == NULL);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  tor_free(msg);
+  config_mgr_free(mgr);
+}
+
+/* Test out confparse_get_assigned(). */
+static void
+test_confparse_get_assigned(void *arg)
+{
+  (void)arg;
+
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = get_simple_config(mgr);
+  config_line_t *lines = NULL;
+
+  lines = config_get_assigned_option(mgr, tst, "I", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "i");
+  tt_str_op(lines->value, OP_EQ, "3");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "this is a");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "obsolete", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "nonesuch", 1);
+  tt_assert(!lines);
+
+  lines = config_get_assigned_option(mgr, tst, "mixedlines", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeA");
+  tt_str_op(lines->value, OP_EQ, "i d");
+  tt_assert(lines->next);
+  tt_str_op(lines->next->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->next->value, OP_EQ, "i c");
+  tt_assert(lines->next->next == NULL);
+  config_free_lines(lines);
+
+  lines = config_get_assigned_option(mgr, tst, "linetypeb", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "LineTypeB");
+  tt_str_op(lines->value, OP_EQ, "i c");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+  tor_free(tst->s);
+  tst->s = tor_strdup("Hello\nWorld");
+  lines = config_get_assigned_option(mgr, tst, "s", 1);
+  tt_assert(lines);
+  tt_str_op(lines->key, OP_EQ, "s");
+  tt_str_op(lines->value, OP_EQ, "\"Hello\\nWorld\"");
+  tt_assert(lines->next == NULL);
+  config_free_lines(lines);
+
+ done:
+  config_free(mgr, tst);
+  config_free_lines(lines);
+  config_mgr_free(mgr);
+}
+
+/* Another variant, which accepts and stores unrecognized lines.*/
+#define ETEST_MAGIC 13371337
+
+static struct_member_t extra = {
+  .name = "__extra",
+  .type = CONFIG_TYPE_LINELIST,
+  .offset = offsetof(test_struct_t, extra_lines),
+};
+
+static config_format_t etest_fmt = {
+  .size = sizeof(test_struct_t),
+  .magic = {
+   "test_struct_t (with extra lines)",
+   ETEST_MAGIC,
+   offsetof(test_struct_t, magic),
+  },
+  .abbrevs = test_abbrevs,
+  .deprecations = test_deprecation_notes,
+  .vars = test_vars,
+  .legacy_validate_fn = test_validate_cb,
+  .extra = &extra,
+};
+
+/* Try out the feature where we can store unrecognized lines and dump them
+ * again.  (State files use this.) */
+static void
+test_confparse_extra_lines(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&etest_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  config_line_t *lines = NULL;
+  char *msg = NULL, *dump = NULL;
+
+  config_init(mgr, tst);
+
+  int r = config_get_lines(
+      "unknotty addita\n"
+      "pos 99\n"
+      "wombat knish\n", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = config_assign(mgr, tst, lines, 0, &msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+
+  tt_assert(tst->extra_lines);
+
+  dump = config_dump(mgr, NULL, tst, 1, 0);
+  tt_str_op(dump, OP_EQ,
+      "pos 99\n"
+      "unknotty addita\n"
+      "wombat knish\n");
+
+ done:
+  tor_free(msg);
+  tor_free(dump);
+  config_free_lines(lines);
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_unitparse(void *args)
+{
+  (void)args;
+  /* spot-check a few memunit values. */
+  int ok = 3;
+  tt_u64_op(config_parse_memunit("100 MB", &ok), OP_EQ, 100<<20);
+  tt_assert(ok);
+  tt_u64_op(config_parse_memunit("100 TB", &ok), OP_EQ, UINT64_C(100)<<40);
+  tt_assert(ok);
+  // This is a floating-point value, but note that 1.5 can be represented
+  // precisely.
+  tt_u64_op(config_parse_memunit("1.5 MB", &ok), OP_EQ, 3<<19);
+  tt_assert(ok);
+
+  /* Try some good intervals and msec intervals */
+  tt_int_op(config_parse_interval("2 days", &ok), OP_EQ, 48*3600);
+  tt_assert(ok);
+  tt_int_op(config_parse_interval("1.5 hour", &ok), OP_EQ, 5400);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("1 minute", &ok), OP_EQ, 60);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("2 days", &ok), OP_EQ, 48*3600*1000);
+  tt_assert(ok);
+  tt_int_op(config_parse_msec_interval("10 msec", &ok), OP_EQ, 10);
+  tt_assert(ok);
+
+  /* Try a couple of unitless values. */
+  tt_int_op(config_parse_interval("10", &ok), OP_EQ, 10);
+  tt_assert(ok);
+  tt_u64_op(config_parse_interval("15.0", &ok), OP_EQ, 15);
+  tt_assert(ok);
+
+  /* u64 overflow */
+  tt_u64_op(config_parse_memunit("20000000 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // This test fails the double check as the float representing 15000000.5 TB
+  // is greater than (double) INT64_MAX
+  tt_u64_op(config_parse_memunit("15000000.5 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // 8388608.1 TB passes double check because it falls in the same float
+  // value as (double)INT64_MAX (which is 2^63) due to precision.
+  // But will fail the int check because the unsigned representation of
+  // the float, which is 2^63, is strictly greater than INT64_MAX (2^63-1)
+  tt_u64_op(config_parse_memunit("8388608.1 TB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+  /* negative float */
+  tt_u64_op(config_parse_memunit("-1.5 GB", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+  /* i32 overflow */
+  tt_int_op(config_parse_interval("1000 months", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("4 weeks", &ok), OP_EQ, -1);
+  tt_assert(!ok);
+
+  /* bad units */
+  tt_u64_op(config_parse_memunit("7 nybbles", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  // XXXX these next two should return -1 according to the documentation.
+  tt_int_op(config_parse_interval("7 cowznofski", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+  tt_int_op(config_parse_msec_interval("1 kalpa", &ok), OP_EQ, 0);
+  tt_assert(!ok);
+
+ done:
+  ;
+}
+
+static void
+test_confparse_check_ok_fail(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  config_mgr_freeze(mgr);
+  test_struct_t *tst = config_new(mgr);
+  tst->pos = -10;
+  tt_assert(! config_check_ok(mgr, tst, LOG_INFO));
+
+ done:
+  config_free(mgr, tst);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_vars(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_vars(mgr);
+  smartlist_t *varnames = smartlist_new();
+  char *joined = NULL;
+
+  tt_assert(vars);
+  SMARTLIST_FOREACH(vars, config_var_t *, cv,
+                    smartlist_add(varnames, (void*)cv->member.name));
+  smartlist_sort_strings(varnames);
+  joined = smartlist_join_strings(varnames, "::", 0, NULL);
+  tt_str_op(joined, OP_EQ,
+            "LineTypeA::"
+            "LineTypeB::"
+            "MixedHiddenLines::"
+            "MixedLines::"
+            "VisibleLineB::"
+            "__HiddenInt::"
+            "__HiddenLineA::"
+            "autobool::"
+            "boolean::"
+            "csv::"
+            "csv_interval::"
+            "dbl::"
+            "deprecated_int::"
+            "fn::"
+            "i::"
+            "interval::"
+            "lines::"
+            "mem::"
+            "msec_interval::"
+            "obsolete::"
+            "pos::"
+            "routerset::"
+            "s::"
+            "time::"
+            "u64");
+
+ done:
+  tor_free(joined);
+  smartlist_free(varnames);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_list_deprecated(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+  smartlist_t *vars = config_mgr_list_deprecated_vars(mgr);
+  char *joined = NULL;
+
+  tt_assert(vars);
+  smartlist_sort_strings(vars);
+  joined = smartlist_join_strings(vars, "::", 0, NULL);
+
+  tt_str_op(joined, OP_EQ, "deprecated_int");
+
+ done:
+  tor_free(joined);
+  smartlist_free(vars);
+  config_mgr_free(mgr);
+}
+
+static void
+test_confparse_find_option_name(void *arg)
+{
+  (void)arg;
+  config_mgr_t *mgr = config_mgr_new(&test_fmt);
+
+  // exact match
+  tt_str_op(config_find_option_name(mgr, "u64"), OP_EQ, "u64");
+  // case-insensitive match
+  tt_str_op(config_find_option_name(mgr, "S"), OP_EQ, "s");
+  tt_str_op(config_find_option_name(mgr, "linetypea"), OP_EQ, "LineTypeA");
+  // prefix match
+  tt_str_op(config_find_option_name(mgr, "deprec"), OP_EQ, "deprecated_int");
+  // explicit abbreviation
+  tt_str_op(config_find_option_name(mgr, "uint"), OP_EQ, "pos");
+  tt_str_op(config_find_option_name(mgr, "UINT"), OP_EQ, "pos");
+  // no match
+  tt_ptr_op(config_find_option_name(mgr, "absent"), OP_EQ, NULL);
+
+ done:
+  config_mgr_free(mgr);
+}
+
+#ifndef COCCI
+#define CONFPARSE_TEST(name, flags)                          \
+  { #name, test_confparse_ ## name, flags, NULL, NULL }
+
+#define BADVAL_TEST(name)                               \
+  { "badval_" #name, test_confparse_assign_badval, 0,   \
+      &passthrough_setup, (void*)&bv_ ## name }
+#endif /* !defined(COCCI) */
+
+struct testcase_t confparse_tests[] = {
+  CONFPARSE_TEST(init, 0),
+  CONFPARSE_TEST(assign_simple, 0),
+  CONFPARSE_TEST(assign_obsolete, 0),
+  CONFPARSE_TEST(assign_deprecated, 0),
+  CONFPARSE_TEST(assign_replaced, 0),
+  CONFPARSE_TEST(assign_emptystring, 0),
+  CONFPARSE_TEST(assign_twice, 0),
+  BADVAL_TEST(notint),
+  BADVAL_TEST(negint),
+  BADVAL_TEST(badu64),
+  BADVAL_TEST(dbl1),
+  BADVAL_TEST(dbl2),
+  BADVAL_TEST(dbl3),
+  BADVAL_TEST(dbl4),
+  BADVAL_TEST(dbl5),
+  BADVAL_TEST(dbl6),
+  BADVAL_TEST(badcsvi1),
+  BADVAL_TEST(badcsvi2),
+  BADVAL_TEST(nonoption),
+  BADVAL_TEST(badmem),
+  BADVAL_TEST(badbool),
+  BADVAL_TEST(badabool),
+  BADVAL_TEST(badtime),
+  BADVAL_TEST(virt),
+  BADVAL_TEST(rs),
+  BADVAL_TEST(big_interval),
+  CONFPARSE_TEST(dump, 0),
+  CONFPARSE_TEST(reset, 0),
+  CONFPARSE_TEST(reassign, 0),
+  CONFPARSE_TEST(reassign_extend, 0),
+  CONFPARSE_TEST(get_assigned, 0),
+  CONFPARSE_TEST(extra_lines, 0),
+  CONFPARSE_TEST(unitparse, 0),
+  CONFPARSE_TEST(check_ok_fail, 0),
+  CONFPARSE_TEST(list_vars, 0),
+  CONFPARSE_TEST(list_deprecated, 0),
+  CONFPARSE_TEST(find_option_name, 0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_connection.c b/src/test/test_connection.c
index 7234d01627..b1252864f5 100644
--- a/src/test/test_connection.c
+++ b/src/test/test_connection.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -10,6 +10,7 @@
 #include "core/or/or.h"
 #include "test/test.h"
 
+#include "app/config/or_options_st.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/hs/hs_common.h"
@@ -312,6 +313,25 @@ test_conn_download_status_teardown(const struct testcase_t *tc, void *arg)
   return rv;
 }
 
+static void *
+test_conn_proxy_connect_setup(const struct testcase_t *tc)
+{
+  return test_conn_get_proxy_or_connection(*(unsigned int *)tc->setup_data);
+}
+
+static int
+test_conn_proxy_connect_teardown(const struct testcase_t *tc, void *arg)
+{
+  (void)tc;
+  or_connection_t *conn = arg;
+
+  tt_assert(conn);
+  assert_connection_ok(&conn->base_, time(NULL));
+
+ done:
+  return 1;
+}
+
 /* Like connection_ap_make_link(), but does much less */
 static connection_t *
 test_conn_get_linked_connection(connection_t *l_conn, uint8_t state)
@@ -360,6 +380,10 @@ static struct testcase_setup_t test_conn_download_status_st = {
   test_conn_download_status_setup, test_conn_download_status_teardown
 };
 
+static struct testcase_setup_t test_conn_proxy_connect_st = {
+  test_conn_proxy_connect_setup, test_conn_proxy_connect_teardown
+};
+
 static void
 test_conn_get_basic(void *arg)
 {
@@ -789,6 +813,64 @@ test_conn_download_status(void *arg)
   /* the teardown function removes all the connections in the global list*/;
 }
 
+static void
+test_conn_https_proxy_connect(void *arg)
+{
+  size_t sz;
+  char *buf = NULL;
+  or_connection_t *conn = arg;
+
+  MOCK(connection_or_change_state, mock_connection_or_change_state);
+
+  tt_int_op(conn->base_.proxy_state, OP_EQ, PROXY_HTTPS_WANT_CONNECT_OK);
+
+  buf = buf_get_contents(conn->base_.outbuf, &sz);
+  tt_str_op(buf, OP_EQ, "CONNECT 127.0.0.1:12345 HTTP/1.0\r\n\r\n");
+
+ done:
+  UNMOCK(connection_or_change_state);
+  tor_free(buf);
+}
+
+static int handshake_start_called = 0;
+
+static int
+handshake_start(or_connection_t *conn, int receiving)
+{
+  (void)receiving;
+
+  tor_assert(conn);
+
+  handshake_start_called = 1;
+  return 0;
+}
+
+static void
+test_conn_haproxy_proxy_connect(void *arg)
+{
+  size_t sz;
+  char *buf = NULL;
+  or_connection_t *conn = arg;
+
+  MOCK(connection_or_change_state, mock_connection_or_change_state);
+  MOCK(connection_tls_start_handshake, handshake_start);
+
+  tt_int_op(conn->base_.proxy_state, OP_EQ, PROXY_HAPROXY_WAIT_FOR_FLUSH);
+
+  buf = buf_get_contents(conn->base_.outbuf, &sz);
+  tt_str_op(buf, OP_EQ, "PROXY TCP4 0.0.0.0 127.0.0.1 0 12345\r\n");
+
+  connection_or_finished_flushing(conn);
+
+  tt_int_op(conn->base_.proxy_state, OP_EQ, PROXY_CONNECTED);
+  tt_int_op(handshake_start_called, OP_EQ, 1);
+
+ done:
+  UNMOCK(connection_or_change_state);
+  UNMOCK(connection_tls_start_handshake);
+  tor_free(buf);
+}
+
 static node_t test_node;
 
 static node_t *
@@ -882,22 +964,39 @@ test_failed_orconn_tracker(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define CONNECTION_TESTCASE(name, fork, setup)                           \
   { #name, test_conn_##name, fork, &setup, NULL }
 
-/* where arg is a string. */
-#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg)                  \
-  { #name "_" arg, test_conn_##name, fork, &setup, (void *)arg }
+#define STR(x) #x
+/* where arg is an expression (constant, variable, compound expression) */
+#define CONNECTION_TESTCASE_ARG(name, fork, setup, arg)                 \
+  { #name "_" STR(x),                                                   \
+      test_conn_##name,                                                 \
+      fork,                                                             \
+      &setup,                                                           \
+      (void *)arg }
+#endif /* !defined(COCCI) */
+
+static const unsigned int PROXY_CONNECT_ARG = PROXY_CONNECT;
+static const unsigned int PROXY_HAPROXY_ARG = PROXY_HAPROXY;
 
 struct testcase_t connection_tests[] = {
   CONNECTION_TESTCASE(get_basic, TT_FORK, test_conn_get_basic_st),
   CONNECTION_TESTCASE(get_rend,  TT_FORK, test_conn_get_rend_st),
   CONNECTION_TESTCASE(get_rsrc,  TT_FORK, test_conn_get_rsrc_st),
+
   CONNECTION_TESTCASE_ARG(download_status,  TT_FORK,
                           test_conn_download_status_st, "microdesc"),
   CONNECTION_TESTCASE_ARG(download_status,  TT_FORK,
                           test_conn_download_status_st, "ns"),
-//CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair),
+
+  CONNECTION_TESTCASE_ARG(https_proxy_connect,   TT_FORK,
+                          test_conn_proxy_connect_st, &PROXY_CONNECT_ARG),
+  CONNECTION_TESTCASE_ARG(haproxy_proxy_connect, TT_FORK,
+                          test_conn_proxy_connect_st, &PROXY_HAPROXY_ARG),
+
+  //CONNECTION_TESTCASE(func_suffix, TT_FORK, setup_func_pair),
   { "failed_orconn_tracker", test_failed_orconn_tracker, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_connection.h b/src/test/test_connection.h
index 47a5599e5f..bf327c0a3d 100644
--- a/src/test/test_connection.h
+++ b/src/test/test_connection.h
@@ -1,13 +1,18 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_CONNECTION_H
+#define TOR_TEST_CONNECTION_H
+
 /** Some constants used by test_connection and helpers */
 #define TEST_CONN_FAMILY        (AF_INET)
 #define TEST_CONN_ADDRESS       "127.0.0.1"
+#define TEST_CONN_ADDRESS_2     "127.0.0.2"
 #define TEST_CONN_PORT          (12345)
 #define TEST_CONN_ADDRESS_PORT  "127.0.0.1:12345"
-#define TEST_CONN_FD_INIT 50
+#define TEST_CONN_FD_INIT       0x10000
 
 void test_conn_lookup_addr_helper(const char *address,
                                   int family, tor_addr_t *addr);
 
+#endif /* !defined(TOR_TEST_CONNECTION_H) */
diff --git a/src/test/test_conscache.c b/src/test/test_conscache.c
index 095ff09350..c805774fa3 100644
--- a/src/test/test_conscache.c
+++ b/src/test/test_conscache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/test_consdiff.c b/src/test/test_consdiff.c
index 682ba5b970..242e2f7818 100644
--- a/src/test/test_consdiff.c
+++ b/src/test/test_consdiff.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2014, Daniel Martí
- * Copyright (c) 2014-2019, The Tor Project, Inc. */
+ * Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFF_PRIVATE
@@ -14,6 +14,39 @@
 #define tt_str_eq_line(a,b) \
   tt_assert(line_str_eq((b),(a)))
 
+static int
+consensus_split_lines_(smartlist_t *out, const char *s, memarea_t *area)
+{
+  size_t len = strlen(s);
+  return consensus_split_lines(out, s, len, area);
+}
+
+static int
+consensus_compute_digest_(const char *cons,
+                          consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
+static int
+consensus_compute_digest_as_signed_(const char *cons,
+                                    consensus_digest_t *digest_out)
+{
+  size_t len = strlen(cons);
+  char *tmp = tor_memdup(cons, len);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  int r = consensus_compute_digest_as_signed(tmp, len, digest_out);
+  tor_free(tmp);
+  return r;
+}
+
 static void
 test_consdiff_smartlist_slice(void *arg)
 {
@@ -58,7 +91,7 @@ test_consdiff_smartlist_slice_string_pos(void *arg)
 
   /* Create a regular smartlist. */
   (void)arg;
-  consensus_split_lines(sl, "a\nd\nc\na\nb\n", area);
+  consensus_split_lines_(sl, "a\nd\nc\na\nb\n", area);
 
   /* See that smartlist_slice_string_pos respects the bounds of the slice. */
   sls = smartlist_slice(sl, 2, 5);
@@ -87,8 +120,8 @@ test_consdiff_lcs_lengths(void *arg)
   int e_lengths2[] = { 0, 1, 1, 2, 3, 4 };
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nc\nd\ne\n", area);
-  consensus_split_lines(sl2, "a\nc\nd\ni\ne\n", area);
+  consensus_split_lines_(sl1, "a\nb\nc\nd\ne\n", area);
+  consensus_split_lines_(sl2, "a\nc\nd\ni\ne\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -119,10 +152,10 @@ test_consdiff_trim_slices(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\nb\nb\nd\n", area);
-  consensus_split_lines(sl2, "a\nc\nc\nc\nd\n", area);
-  consensus_split_lines(sl3, "a\nb\nb\nb\na\n", area);
-  consensus_split_lines(sl4, "c\nb\nb\nb\nc\n", area);
+  consensus_split_lines_(sl1, "a\nb\nb\nb\nd\n", area);
+  consensus_split_lines_(sl2, "a\nc\nc\nc\nd\n", area);
+  consensus_split_lines_(sl3, "a\nb\nb\nb\na\n", area);
+  consensus_split_lines_(sl4, "c\nb\nb\nb\nc\n", area);
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
   sls3 = smartlist_slice(sl3, 0, -1);
@@ -165,8 +198,8 @@ test_consdiff_set_changed(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\nb\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\nb\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   /* Length of sls1 is 0. */
   sls1 = smartlist_slice(sl1, 0, 0);
@@ -240,8 +273,8 @@ test_consdiff_calc_changes(void *arg)
   memarea_t *area = memarea_new();
 
   (void)arg;
-  consensus_split_lines(sl1, "a\na\na\na\n", area);
-  consensus_split_lines(sl2, "a\na\na\na\n", area);
+  consensus_split_lines_(sl1, "a\na\na\na\n", area);
+  consensus_split_lines_(sl2, "a\na\na\na\n", area);
 
   sls1 = smartlist_slice(sl1, 0, -1);
   sls2 = smartlist_slice(sl2, 0, -1);
@@ -259,7 +292,7 @@ test_consdiff_calc_changes(void *arg)
   tt_assert(!bitarray_is_set(changed2, 3));
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "a\nb\na\nb\n", area);
+  consensus_split_lines_(sl2, "a\nb\na\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -282,7 +315,7 @@ test_consdiff_calc_changes(void *arg)
   bitarray_clear(changed1, 3);
 
   smartlist_clear(sl2);
-  consensus_split_lines(sl2, "b\nb\nb\nb\n", area);
+  consensus_split_lines_(sl2, "b\nb\nb\nb\n", area);
   tor_free(sls1);
   tor_free(sls2);
   sls1 = smartlist_slice(sl1, 0, -1);
@@ -610,8 +643,8 @@ test_consdiff_gen_ed_diff(void *arg)
   /* Test 'a', 'c' and 'd' together. See that it is done in reverse order. */
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
-  consensus_split_lines(cons2, "A\nC\nO\nE\nU\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons2, "A\nC\nO\nE\nU\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(7, OP_EQ, smartlist_len(diff));
@@ -627,8 +660,8 @@ test_consdiff_gen_ed_diff(void *arg)
 
   smartlist_clear(cons1);
   smartlist_clear(cons2);
-  consensus_split_lines(cons1, "B\n", area);
-  consensus_split_lines(cons2, "A\nB\n", area);
+  consensus_split_lines_(cons1, "B\n", area);
+  consensus_split_lines_(cons2, "A\nB\n", area);
   diff = gen_ed_diff(cons1, cons2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(3, OP_EQ, smartlist_len(diff));
@@ -656,7 +689,7 @@ test_consdiff_apply_ed_diff(void *arg)
   diff = smartlist_new();
   setup_capture_of_logs(LOG_WARN);
 
-  consensus_split_lines(cons1, "A\nB\nC\nD\nE\n", area);
+  consensus_split_lines_(cons1, "A\nB\nC\nD\nE\n", area);
 
   /* Command without range. */
   smartlist_add_linecpy(diff, area, "a");
@@ -829,7 +862,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_clear(diff);
 
   /* Test appending text, 'a'. */
-  consensus_split_lines(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
+  consensus_split_lines_(diff, "3a\nU\nO\n.\n0a\nV\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(8, OP_EQ, smartlist_len(cons2));
@@ -846,7 +879,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test deleting text, 'd'. */
-  consensus_split_lines(diff, "4d\n1,2d\n", area);
+  consensus_split_lines_(diff, "4d\n1,2d\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(2, OP_EQ, smartlist_len(cons2));
@@ -857,7 +890,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test changing text, 'c'. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n1,2c\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(5, OP_EQ, smartlist_len(cons2));
@@ -871,7 +904,7 @@ test_consdiff_apply_ed_diff(void *arg)
   smartlist_free(cons2);
 
   /* Test 'a', 'd' and 'c' together. */
-  consensus_split_lines(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
+  consensus_split_lines_(diff, "4c\nT\nX\n.\n2d\n0a\nM\n.\n", area);
   cons2 = apply_ed_diff(cons1, diff, 0);
   tt_ptr_op(NULL, OP_NE, cons2);
   tt_int_op(6, OP_EQ, smartlist_len(cons2));
@@ -918,12 +951,12 @@ test_consdiff_gen_diff(void *arg)
       );
 
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons2_str, &digests2));
+      consensus_compute_digest_(cons2_str, &digests2));
 
-  consensus_split_lines(cons1, cons1_str, area);
-  consensus_split_lines(cons2, cons2_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
+  consensus_split_lines_(cons2, cons2_str, area);
 
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_EQ, diff);
@@ -937,9 +970,9 @@ test_consdiff_gen_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest_as_signed(cons1_str, &digests1));
+      consensus_compute_digest_as_signed_(cons1_str, &digests1));
   smartlist_clear(cons1);
-  consensus_split_lines(cons1, cons1_str, area);
+  consensus_split_lines_(cons1, cons1_str, area);
   diff = consdiff_gen_diff(cons1, cons2, &digests1, &digests2, area);
   tt_ptr_op(NULL, OP_NE, diff);
   tt_int_op(11, OP_EQ, smartlist_len(diff));
@@ -991,13 +1024,13 @@ test_consdiff_apply_diff(void *arg)
       "directory-signature foo bar\nbar\n"
       );
   tt_int_op(0, OP_EQ,
-      consensus_compute_digest(cons1_str, &digests1));
-  consensus_split_lines(cons1, cons1_str, area);
+      consensus_compute_digest_(cons1_str, &digests1));
+  consensus_split_lines_(cons1, cons1_str, area);
 
   /* diff doesn't have enough lines. */
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("too short")
+  expect_single_log_msg_containing("too short");
 
   /* first line doesn't match format-version string. */
   smartlist_add_linecpy(diff, area, "foo-bar");
@@ -1005,7 +1038,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("format is not known")
+  expect_single_log_msg_containing("format is not known");
 
   /* The first word of the second header line is not "hash". */
   smartlist_clear(diff);
@@ -1015,7 +1048,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("does not include the necessary digests")
+  expect_single_log_msg_containing("does not include the necessary digests");
 
   /* Wrong number of words after "hash". */
   smartlist_clear(diff);
@@ -1024,7 +1057,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("does not include the necessary digests")
+  expect_single_log_msg_containing("does not include the necessary digests");
 
   /* base16 digests do not have the expected length. */
   smartlist_clear(diff);
@@ -1034,7 +1067,7 @@ test_consdiff_apply_diff(void *arg)
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
   expect_single_log_msg_containing("includes base16-encoded digests of "
-                                   "incorrect size")
+                                   "incorrect size");
 
   /* base16 digests contain non-base16 characters. */
   smartlist_clear(diff);
@@ -1045,7 +1078,7 @@ test_consdiff_apply_diff(void *arg)
   mock_clean_saved_logs();
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
-  expect_single_log_msg_containing("includes malformed digests")
+  expect_single_log_msg_containing("includes malformed digests");
 
   /* Invalid ed diff.
    * As tested in apply_ed_diff, but check that apply_diff does return NULL if
@@ -1062,7 +1095,7 @@ test_consdiff_apply_diff(void *arg)
   cons2 = consdiff_apply_diff(cons1, diff, &digests1);
   tt_ptr_op(NULL, OP_EQ, cons2);
   expect_single_log_msg_containing("because an ed command was missing a line "
-                                   "number")
+                                   "number");
 
   /* Base consensus doesn't match its digest as found in the diff. */
   smartlist_clear(diff);
@@ -1182,4 +1215,3 @@ struct testcase_t consdiff_tests[] = {
   CONSDIFF_LEGACY(apply_diff),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_consdiffmgr.c b/src/test/test_consdiffmgr.c
index 254a5ba5d0..4bec6baeb0 100644
--- a/src/test/test_consdiffmgr.c
+++ b/src/test/test_consdiffmgr.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONSDIFFMGR_PRIVATE
@@ -21,6 +21,23 @@
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
+static char *
+consensus_diff_apply_(const char *c, const char *d)
+{
+  size_t c_len = strlen(c);
+  size_t d_len = strlen(d);
+  // We use memdup here to ensure that the input is NOT nul-terminated.
+  // This makes it likelier for us to spot bugs.
+  char *c_tmp = tor_memdup(c, c_len);
+  char *d_tmp = tor_memdup(d, d_len);
+  char *result = consensus_diff_apply(c_tmp, c_len, d_tmp, d_len);
+  tor_free(c_tmp);
+  tor_free(d_tmp);
+  return result;
+}
+
 // ============================== Setup/teardown the consdiffmgr
 // These functions get run before/after each test in this module
 
@@ -102,7 +119,7 @@ typedef struct fake_work_queue_ent_t {
   void (*reply_fn)(void *);
   void *arg;
 } fake_work_queue_ent_t;
-static struct workqueue_entry_s *
+static struct workqueue_entry_t *
 mock_cpuworker_queue_work(workqueue_priority_t prio,
                           enum workqueue_reply_t (*fn)(void *, void *),
                           void (*reply_fn)(void *),
@@ -118,7 +135,7 @@ mock_cpuworker_queue_work(workqueue_priority_t prio,
   ent->reply_fn = reply_fn;
   ent->arg = arg;
   smartlist_add(fake_cpuworker_queue, ent);
-  return (struct workqueue_entry_s *)ent;
+  return (struct workqueue_entry_t *)ent;
 }
 static int
 mock_cpuworker_run_work(void)
@@ -153,7 +170,8 @@ lookup_diff_from(consensus_cache_entry_t **out,
                  const char *str1)
 {
   uint8_t digest[DIGEST256_LEN];
-  if (router_get_networkstatus_v3_sha3_as_signed(digest, str1)<0) {
+  if (router_get_networkstatus_v3_sha3_as_signed(digest,
+                                                 str1, strlen(str1))<0) {
     TT_FAIL(("Unable to compute sha3-as-signed"));
     return CONSDIFF_NOT_FOUND;
   }
@@ -175,14 +193,15 @@ lookup_apply_and_verify_diff(consensus_flavor_t flav,
 
   consensus_cache_entry_incref(ent);
   size_t size;
-  char *diff_string = NULL;
-  int r = uncompress_or_copy(&diff_string, &size, ent);
+  const char *diff_string = NULL;
+  char *diff_owned = NULL;
+  int r = uncompress_or_set_ptr(&diff_string, &size, &diff_owned, ent);
   consensus_cache_entry_decref(ent);
   if (diff_string == NULL || r < 0)
     return -1;
 
-  char *applied = consensus_diff_apply(str1, diff_string);
-  tor_free(diff_string);
+  char *applied = consensus_diff_apply(str1, strlen(str1), diff_string, size);
+  tor_free(diff_owned);
   if (applied == NULL)
     return -1;
 
@@ -282,7 +301,8 @@ test_consdiffmgr_add(void *arg)
   (void) arg;
   time_t now = approx_time();
 
-  char *body = NULL;
+  const char *body = NULL;
+  char *body_owned = NULL;
 
   consensus_cache_entry_t *ent = NULL;
   networkstatus_t *ns_tmp = fake_ns_new(FLAV_NS, now);
@@ -324,7 +344,7 @@ test_consdiffmgr_add(void *arg)
   tt_assert(ent);
   consensus_cache_entry_incref(ent);
   size_t s;
-  r = uncompress_or_copy(&body, &s, ent);
+  r = uncompress_or_set_ptr(&body, &s, &body_owned, ent);
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(s, OP_EQ, 4);
   tt_mem_op(body, OP_EQ, "quux", 4);
@@ -337,7 +357,7 @@ test_consdiffmgr_add(void *arg)
   networkstatus_vote_free(ns_tmp);
   teardown_capture_of_logs();
   consensus_cache_entry_decref(ent);
-  tor_free(body);
+  tor_free(body_owned);
 }
 
 static void
@@ -370,7 +390,8 @@ test_consdiffmgr_make_diffs(void *arg)
   ns = fake_ns_new(FLAV_MICRODESC, now-3600);
   md_ns_body = fake_ns_body_new(FLAV_MICRODESC, now-3600);
   r = consdiffmgr_add_consensus(md_ns_body, ns);
-  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body);
+  router_get_networkstatus_v3_sha3_as_signed(md_ns_sha3, md_ns_body,
+                                             strlen(md_ns_body));
   networkstatus_vote_free(ns);
   tt_int_op(r, OP_EQ, 0);
 
@@ -414,7 +435,7 @@ test_consdiffmgr_make_diffs(void *arg)
   r = consensus_cache_entry_get_body(diff, &diff_body, &diff_size);
   tt_int_op(r, OP_EQ, 0);
   diff_text = tor_memdup_nulterm(diff_body, diff_size);
-  applied = consensus_diff_apply(md_ns_body, diff_text);
+  applied = consensus_diff_apply_(md_ns_body, diff_text);
   tt_assert(applied);
   tt_str_op(applied, OP_EQ, md_ns_body_2);
 
diff --git a/src/test/test_containers.c b/src/test/test_containers.c
index aedd2f7a89..6072148d1b 100644
--- a/src/test/test_containers.c
+++ b/src/test/test_containers.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -96,6 +96,30 @@ test_container_smartlist_basic(void *arg)
   tor_free(v555);
 }
 
+/** Test SMARTLIST_FOREACH_REVERSE_BEGIN loop macro */
+static void
+test_container_smartlist_foreach_reverse(void *arg)
+{
+  smartlist_t *sl = smartlist_new();
+  int i;
+
+  (void) arg;
+
+  /* Add integers to smartlist in increasing order */
+  for (i=0;i<100;i++) {
+    smartlist_add(sl, (void*)(uintptr_t)i);
+  }
+
+  /* Pop them out in reverse and test their value */
+  SMARTLIST_FOREACH_REVERSE_BEGIN(sl, void*, k) {
+    i--;
+    tt_ptr_op(k, OP_EQ, (void*)(uintptr_t)i);
+  } SMARTLIST_FOREACH_END(k);
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for smartlist-of-strings functionality. */
 static void
 test_container_smartlist_strings(void *arg)
@@ -582,6 +606,66 @@ test_container_smartlist_ints_eq(void *arg)
   smartlist_free(sl2);
 }
 
+static void
+test_container_smartlist_grow(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  int i;
+  const char *s[] = { "first", "2nd", "3rd" };
+
+  /* case 1: starting from empty. */
+  smartlist_grow(sl, 10);
+  tt_int_op(10, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 10; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 2: starting with a few elements, probably not reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 5);
+  tt_int_op(5, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  tt_ptr_op(smartlist_get(sl, 3), OP_EQ, NULL);
+  tt_ptr_op(smartlist_get(sl, 4), OP_EQ, NULL);
+
+  /* case 3: starting with a few elements, but reallocating. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 100);
+  tt_int_op(100, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+  for (i = 3; i < 100; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, NULL);
+  }
+
+  /* case 4: shrinking doesn't happen. */
+  smartlist_free(sl);
+  sl = smartlist_new();
+  smartlist_add(sl, (char*)s[0]);
+  smartlist_add(sl, (char*)s[1]);
+  smartlist_add(sl, (char*)s[2]);
+  smartlist_grow(sl, 1);
+  tt_int_op(3, OP_EQ, smartlist_len(sl));
+  for (i = 0; i < 3; ++i) {
+    tt_ptr_op(smartlist_get(sl, i), OP_EQ, s[i]);
+  }
+
+ done:
+  smartlist_free(sl);
+}
+
 /** Run unit tests for bitarray code */
 static void
 test_container_bitarray(void *arg)
@@ -922,6 +1006,10 @@ test_container_smartlist_remove(void *arg)
   tt_ptr_op(smartlist_get(sl, 1), OP_EQ, &array[2]);
   tt_ptr_op(smartlist_get(sl, 2), OP_EQ, &array[1]);
   tt_ptr_op(smartlist_get(sl, 3), OP_EQ, &array[2]);
+  /* Ordinary code should never look at this pointer; we're doing it here
+   * to make sure that we really cleared the pointer we removed.
+   */
+  tt_ptr_op(sl->list[4], OP_EQ, NULL);
 
  done:
   smartlist_free(sl);
@@ -1281,12 +1369,14 @@ test_container_smartlist_strings_eq(void *arg)
 struct testcase_t container_tests[] = {
   CONTAINER_LEGACY(smartlist_basic),
   CONTAINER_LEGACY(smartlist_strings),
+  CONTAINER_LEGACY(smartlist_foreach_reverse),
   CONTAINER_LEGACY(smartlist_overlap),
   CONTAINER_LEGACY(smartlist_digests),
   CONTAINER_LEGACY(smartlist_join),
   CONTAINER_LEGACY(smartlist_pos),
   CONTAINER(smartlist_remove, 0),
   CONTAINER(smartlist_ints_eq, 0),
+  CONTAINER(smartlist_grow, 0),
   CONTAINER_LEGACY(bitarray),
   CONTAINER_LEGACY(digestset),
   CONTAINER_LEGACY(strmap),
diff --git a/src/test/test_controller.c b/src/test/test_controller.c
index 5b406e159b..a69ec17db8 100644
--- a/src/test/test_controller.c
+++ b/src/test/test_controller.c
@@ -1,12 +1,19 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define CONTROL_PRIVATE
+#define CONTROL_CMD_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
 #include "core/or/or.h"
+#include "app/config/config.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
 #include "feature/client/bridges.h"
 #include "feature/control/control.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_getinfo.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/entrynodes.h"
+#include "feature/dircache/cached_dir_st.h"
+#include "feature/dircache/dirserv.h"
 #include "feature/hs/hs_common.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/rend/rendservice.h"
@@ -15,48 +22,257 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "lib/net/resolve.h"
+#include "lib/encoding/confline.h"
+#include "lib/encoding/kvline.h"
 
 #include "feature/control/control_connection_st.h"
+#include "feature/control/control_cmd_args_st.h"
 #include "feature/dirclient/download_status_st.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/node_st.h"
 
+typedef struct {
+  const char *input;
+  const char *expected_parse;
+  const char *expected_error;
+} parser_testcase_t;
+
+typedef struct {
+  const control_cmd_syntax_t *syntax;
+  size_t n_testcases;
+  const parser_testcase_t *testcases;
+} parse_test_params_t;
+
+static char *
+control_cmd_dump_args(const control_cmd_args_t *result)
+{
+  buf_t *buf = buf_new();
+  buf_add_string(buf, "{ args=[");
+  if (result->args) {
+    if (smartlist_len(result->args)) {
+        buf_add_string(buf, " ");
+    }
+    SMARTLIST_FOREACH_BEGIN(result->args, const char *, s) {
+      const bool last = (s_sl_idx == smartlist_len(result->args)-1);
+      buf_add_printf(buf, "%s%s ",
+                     escaped(s),
+                     last ? "" : ",");
+    } SMARTLIST_FOREACH_END(s);
+  }
+  buf_add_string(buf, "]");
+  if (result->cmddata) {
+    buf_add_string(buf, ", obj=");
+    buf_add_string(buf, escaped(result->cmddata));
+  }
+  if (result->kwargs) {
+    buf_add_string(buf, ", { ");
+    const config_line_t *line;
+    for (line = result->kwargs; line; line = line->next) {
+      const bool last = (line->next == NULL);
+      buf_add_printf(buf, "%s=%s%s ", line->key, escaped(line->value),
+                     last ? "" : ",");
+    }
+    buf_add_string(buf, "}");
+  }
+  buf_add_string(buf, " }");
+
+  char *encoded = buf_extract(buf, NULL);
+  buf_free(buf);
+  return encoded;
+}
+
+static void
+test_controller_parse_cmd(void *arg)
+{
+  const parse_test_params_t *params = arg;
+  control_cmd_args_t *result = NULL;
+  char *error = NULL;
+  char *encoded = NULL;
+
+  for (size_t i = 0; i < params->n_testcases; ++i) {
+    const parser_testcase_t *t = &params->testcases[i];
+    result = control_cmd_parse_args("EXAMPLE",
+                                    params->syntax,
+                                    strlen(t->input),
+                                    t->input,
+                                    &error);
+    // A valid test should expect exactly one parse or error.
+    tt_int_op((t->expected_parse == NULL), OP_NE,
+              (t->expected_error == NULL));
+    // We get a result or an error, not both.
+    tt_int_op((result == NULL), OP_EQ, (error != NULL));
+    // We got the one we expected.
+    tt_int_op((result == NULL), OP_EQ, (t->expected_parse == NULL));
+
+    if (result) {
+      encoded = control_cmd_dump_args(result);
+      tt_str_op(encoded, OP_EQ, t->expected_parse);
+    } else {
+      tt_str_op(error, OP_EQ, t->expected_error);
+    }
+
+    tor_free(error);
+    tor_free(encoded);
+    control_cmd_args_free(result);
+  }
+
+ done:
+  tor_free(error);
+  tor_free(encoded);
+  control_cmd_args_free(result);
+}
+
+#ifndef COCCI
+#define OK(inp, out) \
+  { inp "\r\n", out, NULL }
+#define ERR(inp, err) \
+  { inp "\r\n", NULL, err }
+
+#define TESTPARAMS(syntax, array)                \
+  { &syntax,                                     \
+      ARRAY_LENGTH(array),                       \
+      array }
+#endif /* !defined(COCCI) */
+
+static const parser_testcase_t one_to_three_tests[] = {
+   ERR("", "Need at least 1 argument(s)"),
+   ERR("   \t", "Need at least 1 argument(s)"),
+   OK("hello", "{ args=[ \"hello\" ] }"),
+   OK("hello world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("  hello  world      ", "{ args=[ \"hello\", \"world\" ] }"),
+   OK("hello there world", "{ args=[ \"hello\", \"there\", \"world\" ] }"),
+   ERR("why hello there world", "Cannot accept more than 3 argument(s)"),
+   ERR("hello\r\nworld.\r\n.", "Unexpected body"),
+};
+
+static const control_cmd_syntax_t one_to_three_syntax = {
+   .min_args=1, .max_args=3
+};
+
+static const parse_test_params_t parse_one_to_three_params =
+  TESTPARAMS( one_to_three_syntax, one_to_three_tests );
+
+// =
+static const parser_testcase_t no_args_one_obj_tests[] = {
+  ERR("Hi there!\r\n.", "Cannot accept more than 0 argument(s)"),
+  ERR("", "Empty body"),
+  OK("\r\n", "{ args=[], obj=\"\\n\" }"),
+  OK("\r\nHello world\r\n", "{ args=[], obj=\"Hello world\\n\\n\" }"),
+  OK("\r\nHello\r\nworld\r\n", "{ args=[], obj=\"Hello\\nworld\\n\\n\" }"),
+  OK("\r\nHello\r\n..\r\nworld\r\n",
+     "{ args=[], obj=\"Hello\\n.\\nworld\\n\\n\" }"),
+};
+static const control_cmd_syntax_t no_args_one_obj_syntax = {
+   .min_args=0, .max_args=0,
+   .want_cmddata=true,
+};
+static const parse_test_params_t parse_no_args_one_obj_params =
+  TESTPARAMS( no_args_one_obj_syntax, no_args_one_obj_tests );
+
+static const parser_testcase_t no_args_kwargs_tests[] = {
+  OK("", "{ args=[] }"),
+  OK(" ", "{ args=[] }"),
+  OK("hello there=world", "{ args=[], { hello=\"\", there=\"world\" } }"),
+  OK("hello there=world today",
+     "{ args=[], { hello=\"\", there=\"world\", today=\"\" } }"),
+  ERR("=Foo", "Cannot parse keyword argument(s)"),
+};
+static const control_cmd_syntax_t no_args_kwargs_syntax = {
+   .min_args=0, .max_args=0,
+   .accept_keywords=true,
+   .kvline_flags=KV_OMIT_VALS
+};
+static const parse_test_params_t parse_no_args_kwargs_params =
+  TESTPARAMS( no_args_kwargs_syntax, no_args_kwargs_tests );
+
+static const char *one_arg_kwargs_allow_keywords[] = {
+  "Hello", "world", NULL
+};
+static const parser_testcase_t one_arg_kwargs_tests[] = {
+  ERR("", "Need at least 1 argument(s)"),
+  OK("Hi", "{ args=[ \"Hi\" ] }"),
+  ERR("hello there=world", "Unrecognized keyword argument \"there\""),
+  OK("Hi HELLO=foo", "{ args=[ \"Hi\" ], { HELLO=\"foo\" } }"),
+  OK("Hi world=\"bar baz\" hello  ",
+     "{ args=[ \"Hi\" ], { world=\"bar baz\", hello=\"\" } }"),
+};
+static const control_cmd_syntax_t one_arg_kwargs_syntax = {
+   .min_args=1, .max_args=1,
+   .accept_keywords=true,
+   .allowed_keywords=one_arg_kwargs_allow_keywords,
+   .kvline_flags=KV_OMIT_VALS|KV_QUOTED,
+};
+static const parse_test_params_t parse_one_arg_kwargs_params =
+  TESTPARAMS( one_arg_kwargs_syntax, one_arg_kwargs_tests );
+
+static char *reply_str = NULL;
+/* Mock for control_write_reply that copies the string for inspection
+ * by tests */
+static void
+mock_control_write_reply(control_connection_t *conn, int code, int c,
+                                const char *s)
+{
+  (void)conn;
+  (void)code;
+  (void)c;
+  tor_free(reply_str);
+  reply_str = tor_strdup(s);
+}
+
 static void
 test_add_onion_helper_keyarg_v3(void *arg)
 {
   int ret, hs_version;
   add_onion_secret_key_t pk;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   const char *key_new_alg = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit ED25519-V3 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 0, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
+  /* Test "BEST" key generation (Assumes BEST = ED25519-V3). */
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
+  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
+                                &pk, &hs_version, NULL);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
+  tt_assert(pk.v3);
+  tt_str_op(key_new_alg, OP_EQ, "ED25519-V3");
+  tt_assert(key_new_blob);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
+
   /* Test discarding the private key. */
+  tor_free(reply_str);
+  tor_free(pk.v3); pk.v3 = NULL;
+  tor_free(key_new_blob);
   ret = add_onion_helper_keyarg("NEW:ED25519-V3", 1, &key_new_alg,
                                 &key_new_blob, &pk, &hs_version,
-                                &err_msg);
+                                NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
   tt_assert(pk.v3);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tor_free(pk.v3); pk.v3 = NULL;
   tor_free(key_new_blob);
 
@@ -76,9 +292,10 @@ test_add_onion_helper_keyarg_v3(void *arg)
 
     tor_asprintf(&key_blob, "ED25519-V3:%s", base64_sk);
     tt_assert(key_blob);
+    tor_free(reply_str);
     ret = add_onion_helper_keyarg(key_blob, 1, &key_new_alg,
                                   &key_new_blob, &pk, &hs_version,
-                                  &err_msg);
+                                  NULL);
     tor_free(key_blob);
     tt_int_op(ret, OP_EQ, 0);
     tt_int_op(hs_version, OP_EQ, HS_VERSION_THREE);
@@ -86,7 +303,7 @@ test_add_onion_helper_keyarg_v3(void *arg)
     tt_mem_op(pk.v3, OP_EQ, hex_sk, 64);
     tt_ptr_op(key_new_alg, OP_EQ, NULL);
     tt_ptr_op(key_new_blob, OP_EQ, NULL);
-    tt_ptr_op(err_msg, OP_EQ, NULL);
+    tt_ptr_op(reply_str, OP_EQ, NULL);
     tor_free(pk.v3); pk.v3 = NULL;
     tor_free(key_new_blob);
   }
@@ -94,7 +311,8 @@ test_add_onion_helper_keyarg_v3(void *arg)
  done:
   tor_free(pk.v3);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -105,72 +323,61 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_t *pk1 = NULL;
   const char *key_new_alg = NULL;
   char *key_new_blob = NULL;
-  char *err_msg = NULL;
   char *encoded = NULL;
   char *arg_str = NULL;
 
   (void) arg;
+  MOCK(control_write_reply, mock_control_write_reply);
 
   memset(&pk, 0, sizeof(pk));
 
   /* Test explicit RSA1024 key generation. */
+  tor_free(reply_str);
   ret = add_onion_helper_keyarg("NEW:RSA1024", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_str_op(key_new_alg, OP_EQ, "RSA1024");
   tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
-
-  /* Test "BEST" key generation (Assumes BEST = RSA1024). */
-  crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
-  tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
-  tt_assert(pk.v2);
-  tt_str_op(key_new_alg, OP_EQ, "RSA1024");
-  tt_assert(key_new_blob);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test discarding the private key. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_free(key_new_blob);
-  ret = add_onion_helper_keyarg("NEW:BEST", 1, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+  ret = add_onion_helper_keyarg("NEW:RSA1024", 1, &key_new_alg, &key_new_blob,
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
 
   /* Test generating a invalid key type. */
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   ret = add_onion_helper_keyarg("NEW:RSA512", 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a RSA1024 key. */
-  tor_free(err_msg);
+  tor_free(reply_str);
   pk1 = pk_generate(0);
   tt_int_op(0, OP_EQ, crypto_pk_base64_encode_private(pk1, &encoded));
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   tt_int_op(crypto_pk_cmp_keys(pk1, pk.v2), OP_EQ, 0);
 
   /* Test loading a invalid key type. */
@@ -179,36 +386,37 @@ test_add_onion_helper_keyarg_v2(void *arg)
   crypto_pk_free(pk.v2); pk.v2 = NULL;
   tor_asprintf(&arg_str, "RSA512:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                                &pk, &hs_version, &err_msg);
+                                &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
   /* Test loading a invalid key. */
   tor_free(arg_str);
   crypto_pk_free(pk.v2); pk.v2 = NULL;
-  tor_free(err_msg);
+  tor_free(reply_str);
   encoded[strlen(encoded)/2] = '\0';
   tor_asprintf(&arg_str, "RSA1024:%s", encoded);
   ret = add_onion_helper_keyarg(arg_str, 0, &key_new_alg, &key_new_blob,
-                               &pk, &hs_version, &err_msg);
+                               &pk, &hs_version, NULL);
   tt_int_op(ret, OP_EQ, -1);
   tt_int_op(hs_version, OP_EQ, HS_VERSION_TWO);
   tt_assert(!pk.v2);
   tt_ptr_op(key_new_alg, OP_EQ, NULL);
   tt_ptr_op(key_new_blob, OP_EQ, NULL);
-  tt_assert(err_msg);
+  tt_assert(reply_str);
 
  done:
   crypto_pk_free(pk1);
   crypto_pk_free(pk.v2);
   tor_free(key_new_blob);
-  tor_free(err_msg);
+  tor_free(reply_str);
   tor_free(encoded);
   tor_free(arg_str);
+  UNMOCK(control_write_reply);
 }
 
 static void
@@ -362,49 +570,52 @@ static void
 test_add_onion_helper_clientauth(void *arg)
 {
   rend_authorized_client_t *client = NULL;
-  char *err_msg = NULL;
   int created = 0;
 
   (void)arg;
 
+  MOCK(control_write_reply, mock_control_write_reply);
   /* Test "ClientName" only. */
-  client = add_onion_helper_clientauth("alice", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice", &created, NULL);
   tt_assert(client);
   tt_assert(created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test "ClientName:Blob" */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("alice:475hGBHPlq7Mc0cRZitK/B",
-                                       &created, &err_msg);
+                                       &created, NULL);
   tt_assert(client);
   tt_assert(!created);
-  tt_ptr_op(err_msg, OP_EQ, NULL);
+  tt_ptr_op(reply_str, OP_EQ, NULL);
   rend_authorized_client_free(client);
 
   /* Test invalid client names */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth("no*asterisks*allowed", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid auth cookie */
-  client = add_onion_helper_clientauth("alice:12345", &created, &err_msg);
+  tor_free(reply_str);
+  client = add_onion_helper_clientauth("alice:12345", &created, NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
   /* Test invalid syntax */
+  tor_free(reply_str);
   client = add_onion_helper_clientauth(":475hGBHPlq7Mc0cRZitK/B", &created,
-                                       &err_msg);
+                                       NULL);
   tt_ptr_op(client, OP_EQ, NULL);
-  tt_assert(err_msg);
-  tor_free(err_msg);
+  tt_assert(reply_str);
 
  done:
   rend_authorized_client_free(client);
-  tor_free(err_msg);
+  tor_free(reply_str);
+  UNMOCK(control_write_reply);
 }
 
 /* Mocks and data/variables used for GETINFO download status tests */
@@ -1485,6 +1696,138 @@ test_download_status_bridge(void *arg)
   return;
 }
 
+/** Mock cached consensus */
+static cached_dir_t *mock_ns_consensus_cache;
+static cached_dir_t *mock_microdesc_consensus_cache;
+
+/**  Mock the function that retrieves consensus from cache. These use a
+ * global variable so that they can be cleared from within the test.
+ * The actual code retains the pointer to the consensus data, but
+ * we are doing this here, to prevent memory leaks
+ * from within the tests */
+static cached_dir_t *
+mock_dirserv_get_consensus(const char *flavor_name)
+{
+  if (!strcmp(flavor_name, "ns")) {
+    mock_ns_consensus_cache = tor_malloc_zero(sizeof(cached_dir_t));
+    mock_ns_consensus_cache->dir = tor_strdup("mock_ns_consensus");
+    return mock_ns_consensus_cache;
+  } else {
+    mock_microdesc_consensus_cache = tor_malloc_zero(sizeof(cached_dir_t));
+    mock_microdesc_consensus_cache->dir = tor_strdup(
+                                            "mock_microdesc_consensus");
+    return mock_microdesc_consensus_cache;
+  }
+}
+
+/** Mock the function that retrieves consensuses
+ *  from a files in the directory. */
+static tor_mmap_t *
+mock_tor_mmap_file(const char* filename)
+{
+  tor_mmap_t *res;
+  res = tor_malloc_zero(sizeof(tor_mmap_t));
+  if (strstr(filename, "cached-consensus") != NULL) {
+    res->data = "mock_ns_consensus";
+  } else if (strstr(filename, "cached-microdesc-consensus") != NULL) {
+    res->data = "mock_microdesc_consensus";
+  } else {
+    res->data = ".";
+  }
+  res->size = strlen(res->data);
+  return res;
+}
+
+/** Mock the function that clears file data
+ * loaded into the memory */
+static int
+mock_tor_munmap_file(tor_mmap_t *handle)
+{
+  tor_free(handle);
+  return 0;
+}
+
+static void
+test_getinfo_helper_current_consensus_from_file(void *arg)
+{
+  /* We just need one of these to pass, it doesn't matter what's in it */
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+
+  MOCK(tor_mmap_file, mock_tor_mmap_file);
+  MOCK(tor_munmap_file, mock_tor_munmap_file);
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_ns_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tor_free(answer);
+  errmsg = NULL;
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus-microdesc",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_microdesc_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  errmsg = NULL;
+
+ done:
+  tor_free(answer);
+  UNMOCK(tor_mmap_file);
+  UNMOCK(tor_munmap_file);
+  return;
+}
+
+static void
+test_getinfo_helper_current_consensus_from_cache(void *arg)
+{
+  /* We just need one of these to pass, it doesn't matter what's in it */
+  control_connection_t dummy;
+  /* Get results out */
+  char *answer = NULL;
+  const char *errmsg = NULL;
+
+  (void)arg;
+  or_options_t *options = get_options_mutable();
+  options->FetchUselessDescriptors = 1;
+  MOCK(dirserv_get_consensus, mock_dirserv_get_consensus);
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_ns_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tor_free(answer);
+  tor_free(mock_ns_consensus_cache->dir);
+  tor_free(mock_ns_consensus_cache);
+  errmsg = NULL;
+
+  getinfo_helper_dir(&dummy,
+                     "dir/status-vote/current/consensus-microdesc",
+                     &answer,
+                     &errmsg);
+  tt_str_op(answer, OP_EQ, "mock_microdesc_consensus");
+  tt_ptr_op(errmsg, OP_EQ, NULL);
+  tor_free(mock_microdesc_consensus_cache->dir);
+  tor_free(answer);
+  errmsg = NULL;
+
+ done:
+  options->FetchUselessDescriptors = 0;
+  tor_free(answer);
+  tor_free(mock_microdesc_consensus_cache);
+  UNMOCK(dirserv_get_consensus);
+  return;
+}
+
 /** Set timeval to a mock date and time. This is necessary
  * to make tor_gettimeofday() mockable. */
 static void
@@ -1543,7 +1886,7 @@ test_current_time(void *arg)
 static size_t n_nodelist_get_list = 0;
 static smartlist_t *nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 mock_nodelist_get_list(void)
 {
   n_nodelist_get_list++;
@@ -1614,7 +1957,172 @@ test_getinfo_md_all(void *arg)
   return;
 }
 
+static smartlist_t *reply_strs;
+
+static void
+mock_control_write_reply_list(control_connection_t *conn, int code, int c,
+                              const char *s)
+{
+  (void)conn;
+  /* To make matching easier, don't append "\r\n" */
+  smartlist_add_asprintf(reply_strs, "%03d%c%s", code, c, s);
+}
+
+static void
+test_control_reply(void *arg)
+{
+  (void)arg;
+  smartlist_t *lines = smartlist_new();
+
+  MOCK(control_write_reply, mock_control_write_reply);
+
+  tor_free(reply_str);
+  control_reply_clear(lines);
+  control_reply_add_str(lines, 250, "FOO");
+  control_write_reply_lines(NULL, lines);
+  tt_str_op(reply_str, OP_EQ, "FOO");
+
+  tor_free(reply_str);
+  control_reply_clear(lines);
+  control_reply_add_done(lines);
+  control_write_reply_lines(NULL, lines);
+  tt_str_op(reply_str, OP_EQ, "OK");
+
+  tor_free(reply_str);
+  control_reply_clear(lines);
+  UNMOCK(control_write_reply);
+  MOCK(control_write_reply, mock_control_write_reply_list);
+  reply_strs = smartlist_new();
+  control_reply_add_one_kv(lines, 250, 0, "A", "B");
+  control_reply_add_one_kv(lines, 250, 0, "C", "D");
+  control_write_reply_lines(NULL, lines);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 2);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250-A=B");
+  tt_str_op((char *)smartlist_get(reply_strs, 1), OP_EQ, "250 C=D");
+
+  control_reply_clear(lines);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+  control_reply_add_printf(lines, 250, "PROTOCOLINFO %d", 1);
+  control_reply_add_one_kv(lines, 250, KV_OMIT_VALS|KV_RAW, "AUTH", "");
+  control_reply_append_kv(lines, "METHODS", "COOKIE");
+  control_reply_append_kv(lines, "COOKIEFILE", escaped("/tmp/cookie"));
+  control_reply_add_done(lines);
+  control_write_reply_lines(NULL, lines);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 3);
+  tt_str_op((char *)smartlist_get(reply_strs, 0),
+            OP_EQ, "250-PROTOCOLINFO 1");
+  tt_str_op((char *)smartlist_get(reply_strs, 1),
+            OP_EQ, "250-AUTH METHODS=COOKIE COOKIEFILE=\"/tmp/cookie\"");
+  tt_str_op((char *)smartlist_get(reply_strs, 2),
+            OP_EQ, "250 OK");
+
+ done:
+  UNMOCK(control_write_reply);
+  tor_free(reply_str);
+  control_reply_free(lines);
+  if (reply_strs)
+    SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_free(reply_strs);
+  return;
+}
+
+static void
+test_control_getconf(void *arg)
+{
+  (void)arg;
+  control_connection_t conn;
+  char *args = NULL;
+  int r = -1;
+
+  memset(&conn, 0, sizeof(conn));
+  conn.current_cmd = tor_strdup("GETCONF");
+
+  MOCK(control_write_reply, mock_control_write_reply_list);
+  reply_strs = smartlist_new();
+
+  args = tor_strdup("");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250 OK");
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+  tor_free(args);
+
+  args = tor_strdup("NoSuch");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ,
+            "552 Unrecognized configuration key \"NoSuch\"");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("NoSuch1 NoSuch2");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 2);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ,
+            "552-Unrecognized configuration key \"NoSuch1\"");
+  tt_str_op((char *)smartlist_get(reply_strs, 1), OP_EQ,
+            "552 Unrecognized configuration key \"NoSuch2\"");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("ControlPort NoSuch");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  /* Valid keys ignored if there are any invalid ones */
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ,
+            "552 Unrecognized configuration key \"NoSuch\"");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("ClientOnly");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 1);
+  /* According to config.c, this is an exception for the unit tests */
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250 ClientOnly=0");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+  args = tor_strdup("BridgeRelay ClientOnly");
+  r = handle_control_command(&conn, (uint32_t)strlen(args), args);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(smartlist_len(reply_strs), OP_EQ, 2);
+  /* Change if config.c changes BridgeRelay default (unlikely) */
+  tt_str_op((char *)smartlist_get(reply_strs, 0), OP_EQ, "250-BridgeRelay=0");
+  tt_str_op((char *)smartlist_get(reply_strs, 1), OP_EQ, "250 ClientOnly=0");
+  tor_free(args);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_clear(reply_strs);
+
+ done:
+  tor_free(conn.current_cmd);
+  tor_free(args);
+  UNMOCK(control_write_reply);
+  SMARTLIST_FOREACH(reply_strs, char *, p, tor_free(p));
+  smartlist_free(reply_strs);
+}
+
+#ifndef COCCI
+#define PARSER_TEST(type)                                             \
+  { "parse/" #type, test_controller_parse_cmd, 0, &passthrough_setup, \
+      (void*)&parse_ ## type ## _params }
+#endif
+
 struct testcase_t controller_tests[] = {
+  PARSER_TEST(one_to_three),
+  PARSER_TEST(no_args_one_obj),
+  PARSER_TEST(no_args_kwargs),
+  PARSER_TEST(one_arg_kwargs),
   { "add_onion_helper_keyarg_v2", test_add_onion_helper_keyarg_v2, 0,
     NULL, NULL },
   { "add_onion_helper_keyarg_v3", test_add_onion_helper_keyarg_v3, 0,
@@ -1626,11 +2134,17 @@ struct testcase_t controller_tests[] = {
     NULL },
   { "download_status_consensus", test_download_status_consensus, 0, NULL,
     NULL },
+  {"getinfo_helper_current_consensus_from_cache",
+   test_getinfo_helper_current_consensus_from_cache, 0, NULL, NULL },
+  {"getinfo_helper_current_consensus_from_file",
+   test_getinfo_helper_current_consensus_from_file, 0, NULL, NULL },
   { "download_status_cert", test_download_status_cert, 0, NULL,
     NULL },
   { "download_status_desc", test_download_status_desc, 0, NULL, NULL },
   { "download_status_bridge", test_download_status_bridge, 0, NULL, NULL },
   { "current_time", test_current_time, 0, NULL, NULL },
   { "getinfo_md_all", test_getinfo_md_all, 0, NULL, NULL },
+  { "control_reply", test_control_reply, 0, NULL, NULL },
+  { "control_getconf", test_control_getconf, 0, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_controller_events.c b/src/test/test_controller_events.c
index 70d36e53d4..60dfbd630a 100644
--- a/src/test/test_controller_events.c
+++ b/src/test/test_controller_events.c
@@ -1,19 +1,30 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define OCIRC_EVENT_PRIVATE
+#define ORCONN_EVENT_PRIVATE
+#include "app/main/subsysmgr.h"
 #include "core/or/or.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/or/circuitlist.h"
+#include "core/or/ocirc_event.h"
+#include "core/or/orconn_event.h"
 #include "core/mainloop/connection.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
 
+#include "core/or/entry_connection_st.h"
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
+#include "core/or/socks_request_st.h"
 
 static void
 add_testing_cell_stats_entry(circuit_t *circ, uint8_t command,
@@ -351,10 +362,10 @@ test_cntev_dirboot_defer_desc(void *arg)
   /* This event should get deferred */
   control_event_boot_dir(BOOTSTRAP_STATUS_REQUESTING_DESCRIPTORS, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
   assert_bootmsg("45 TAG=requesting_descriptors");
@@ -374,29 +385,343 @@ test_cntev_dirboot_defer_orconn(void *arg)
   control_event_bootstrap(BOOTSTRAP_STATUS_STARTING, 0);
   assert_bootmsg("0 TAG=starting");
   /* This event should get deferred */
-  control_event_boot_dir(BOOTSTRAP_STATUS_CONN_OR, 0);
+  control_event_boot_dir(BOOTSTRAP_STATUS_ENOUGH_DIRINFO, 0);
   assert_bootmsg("0 TAG=starting");
-  control_event_bootstrap(BOOTSTRAP_STATUS_CONN_DIR, 0);
-  assert_bootmsg("5 TAG=conn_dir");
+  control_event_bootstrap(BOOTSTRAP_STATUS_CONN, 0);
+  assert_bootmsg("5 TAG=conn");
   control_event_bootstrap(BOOTSTRAP_STATUS_HANDSHAKE, 0);
-  assert_bootmsg("10 TAG=handshake_dir");
+  assert_bootmsg("14 TAG=handshake");
   /* The deferred event should appear */
   control_event_boot_first_orconn();
-  assert_bootmsg("80 TAG=conn_or");
+  assert_bootmsg("75 TAG=enough_dirinfo");
  done:
   tor_free(saved_event_str);
   UNMOCK(queue_control_event_string);
 }
 
-#define TEST(name, flags)                                               \
+static void
+test_cntev_signal(void *arg)
+{
+  (void)arg;
+  int rv;
+
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+
+  /* Nothing is listening for signals, so no event should be queued. */
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+
+  /* Now try with signals included in the event mask. */
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_GOT_SIGNAL));
+  rv = control_event_signal(SIGHUP);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL RELOAD\r\n");
+
+  rv = control_event_signal(SIGACTIVE);
+  tt_int_op(0, OP_EQ, rv);
+  tt_str_op(saved_event_str, OP_EQ, "650 SIGNAL ACTIVE\r\n");
+
+  /* Try a signal that doesn't exist. */
+  setup_full_capture_of_logs(LOG_WARN);
+  tor_free(saved_event_str);
+  rv = control_event_signal(99999);
+  tt_int_op(-1, OP_EQ, rv);
+  tt_ptr_op(saved_event_str, OP_EQ, NULL);
+  expect_single_log_msg_containing("Unrecognized signal 99999");
+
+ done:
+  tor_free(saved_event_str);
+ teardown_capture_of_logs();
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+setup_orconn_state(orconn_state_msg_t *msg, uint64_t gid, uint64_t chan,
+                   int proxy_type)
+{
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->proxy_type = proxy_type;
+}
+
+static void
+send_orconn_state(const orconn_state_msg_t *msg_in, uint8_t state)
+{
+  orconn_state_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  *msg = *msg_in;
+  msg->state = state;
+  orconn_state_publish(msg);
+}
+
+static void
+send_ocirc_chan(uint32_t gid, uint64_t chan, bool onehop)
+{
+  ocirc_chan_msg_t *msg = tor_malloc(sizeof(*msg));
+
+  msg->gid = gid;
+  msg->chan = chan;
+  msg->onehop = onehop;
+  ocirc_chan_publish(msg);
+}
+
+static void
+test_cntev_orconn_state(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_NONE);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  send_ocirc_chan(1, 1, true);
+  assert_bootmsg("5 TAG=conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  /* It doesn't know it's an origin circuit yet */
+  assert_bootmsg("15 TAG=handshake_done");
+  send_ocirc_chan(2, 2, false);
+  assert_bootmsg("80 TAG=ap_conn");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("85 TAG=ap_conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("89 TAG=ap_handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("90 TAG=ap_handshake_done");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_pt(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_PLUGGABLE);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("1 TAG=conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("2 TAG=conn_done_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("76 TAG=ap_conn_pt");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("77 TAG=ap_conn_done_pt");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_orconn_state_proxy(void *arg)
+{
+  orconn_state_msg_t conn;
+  memset(&conn, 0, sizeof(conn));
+
+  (void)arg;
+  MOCK(queue_control_event_string, mock_queue_control_event_string);
+  control_testing_set_global_event_mask(EVENT_MASK_(EVENT_STATUS_CLIENT));
+  setup_orconn_state(&conn, 1, 1, PROXY_CONNECT);
+  send_ocirc_chan(1, 1, true);
+
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("3 TAG=conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("4 TAG=conn_done_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_TLS_HANDSHAKING);
+  assert_bootmsg("10 TAG=conn_done");
+  send_orconn_state(&conn, OR_CONN_STATE_OR_HANDSHAKING_V3);
+  assert_bootmsg("14 TAG=handshake");
+  send_orconn_state(&conn, OR_CONN_STATE_OPEN);
+  assert_bootmsg("15 TAG=handshake_done");
+
+  send_ocirc_chan(2, 2, false);
+  conn.gid = 2;
+  conn.chan = 2;
+  send_orconn_state(&conn, OR_CONN_STATE_CONNECTING);
+  assert_bootmsg("78 TAG=ap_conn_proxy");
+  send_orconn_state(&conn, OR_CONN_STATE_PROXY_HANDSHAKING);
+  assert_bootmsg("79 TAG=ap_conn_done_proxy");
+
+ done:
+  tor_free(saved_event_str);
+  UNMOCK(queue_control_event_string);
+}
+
+static void
+test_cntev_format_stream(void *arg)
+{
+  entry_connection_t *ec = NULL;
+  char *conndesc = NULL;
+  (void)arg;
+
+  ec = entry_connection_new(CONN_TYPE_AP, AF_INET);
+
+  char *username = tor_strdup("jeremy");
+  char *password = tor_strdup("letmein");
+  ec->socks_request->username = username; // steal reference
+  ec->socks_request->usernamelen = strlen(username);
+  ec->socks_request->password = password; // steal reference
+  ec->socks_request->passwordlen = strlen(password);
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "SOCKS_USERNAME=\"jeremy\""));
+  tt_assert(strstr(conndesc, "SOCKS_PASSWORD=\"letmein\""));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_LISTENER;
+  ec->socks_request->socks_version = 4;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=SOCKS4"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_LISTENER;
+  ec->socks_request->socks_version = 5;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=SOCKS5"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_LISTENER;
+  ec->socks_request->socks_version = 6;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=UNKNOWN"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_TRANS_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=TRANS"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_NATD_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=NATD"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_DNS_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=DNS"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_AP_HTTP_CONNECT_LISTENER;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=HTTPCONNECT"));
+  tor_free(conndesc);
+
+  ec->socks_request->listener_type = CONN_TYPE_OR;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "CLIENT_PROTOCOL=UNKNOWN"));
+  tor_free(conndesc);
+
+  ec->nym_epoch = 1337;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "NYM_EPOCH=1337"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.session_group = 4321;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "SESSION_GROUP=4321"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_DESTPORT;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=DESTPORT"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=DESTPORT,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_DESTADDR;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=DESTADDR"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=DESTADDR,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_SOCKSAUTH;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=SOCKS_USERNAME,SOCKS_PASSWORD"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=SOCKS_USERNAME,SOCKS_PASSWORD,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_CLIENTPROTO;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=CLIENT_PROTOCOL"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=CLIENT_PROTOCOL,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_CLIENTADDR;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=CLIENTADDR"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=CLIENTADDR,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_SESSIONGRP;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=SESSION_GROUP"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=SESSION_GROUP,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_NYM_EPOCH;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc, "ISO_FIELDS=NYM_EPOCH"));
+  tt_assert(!strstr(conndesc, "ISO_FIELDS=NYM_EPOCH,"));
+  tor_free(conndesc);
+
+  ec->entry_cfg.isolation_flags = ISO_DESTPORT | ISO_SOCKSAUTH | ISO_NYM_EPOCH;
+  conndesc = entry_connection_describe_status_for_controller(ec);
+  tt_assert(strstr(conndesc,
+    "ISO_FIELDS=DESTPORT,SOCKS_USERNAME,SOCKS_PASSWORD,NYM_EPOCH"));
+  tt_assert(!strstr(conndesc,
+    "ISO_FIELDS=DESTPORT,SOCKS_USERNAME,SOCKS_PASSWORD,NYM_EPOCH,"));
+
+ done:
+  tor_free(conndesc);
+  connection_free_minimal(ENTRY_TO_CONN(ec));
+}
+
+#define TEST(name, flags)                               \
   { #name, test_cntev_ ## name, flags, 0, NULL }
 
+#define T_PUBSUB(name, setup)                                           \
+  { #name, test_cntev_ ## name, TT_FORK, &helper_pubsub_setup, NULL }
+
 struct testcase_t controller_event_tests[] = {
   TEST(sum_up_cell_stats, TT_FORK),
   TEST(append_cell_stats, TT_FORK),
   TEST(format_cell_stats, TT_FORK),
   TEST(event_mask, TT_FORK),
-  TEST(dirboot_defer_desc, TT_FORK),
-  TEST(dirboot_defer_orconn, TT_FORK),
+  TEST(format_stream, TT_FORK),
+  TEST(signal, TT_FORK),
+  T_PUBSUB(dirboot_defer_desc, TT_FORK),
+  T_PUBSUB(dirboot_defer_orconn, TT_FORK),
+  T_PUBSUB(orconn_state, TT_FORK),
+  T_PUBSUB(orconn_state_pt, TT_FORK),
+  T_PUBSUB(orconn_state_proxy, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_crypto.c b/src/test/test_crypto.c
index 5af0cce130..0d75a212e9 100644
--- a/src/test/test_crypto.c
+++ b/src/test/test_crypto.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -29,10 +29,10 @@
 
 #if defined(ENABLE_OPENSSL)
 #include "lib/crypt_ops/compat_openssl.h"
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 #include <openssl/dh.h>
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 /** Run unit tests for Diffie-Hellman functionality. */
 static void
@@ -190,7 +190,7 @@ test_crypto_dh(void *arg)
     DH_get0_key(dh4, &pk, &sk);
 #else
     pk = dh4->pub_key;
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
     tt_assert(pk);
     tt_int_op(BN_num_bytes(pk), OP_LE, DH1024_KEY_LEN);
     tt_int_op(BN_num_bytes(pk), OP_GT, 0);
@@ -207,7 +207,7 @@ test_crypto_dh(void *arg)
     tt_int_op(s1len, OP_GT, 0);
     tt_mem_op(s1, OP_EQ, s2, s1len);
   }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
  done:
   crypto_dh_free(dh1);
@@ -219,7 +219,7 @@ test_crypto_dh(void *arg)
     DH_free(dh4);
   if (pubkey_tmp)
     BN_free(pubkey_tmp);
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 }
 
 static void
@@ -248,174 +248,12 @@ test_crypto_openssl_version(void *arg)
   tt_int_op(a, OP_GE, 0);
   tt_int_op(b, OP_GE, 0);
   tt_int_op(c, OP_GE, 0);
-#endif
+#endif /* defined(ENABLE_NSS) */
 
  done:
   ;
 }
 
-/** Run unit tests for our random number generation function and its wrappers.
- */
-static void
-test_crypto_rng(void *arg)
-{
-  int i, j, allok;
-  char data1[100], data2[100];
-  double d;
-  char *h=NULL;
-
-  /* Try out RNG. */
-  (void)arg;
-  tt_assert(! crypto_seed_rng());
-  crypto_rand(data1, 100);
-  crypto_rand(data2, 100);
-  tt_mem_op(data1,OP_NE, data2,100);
-  allok = 1;
-  for (i = 0; i < 100; ++i) {
-    uint64_t big;
-    char *host;
-    j = crypto_rand_int(100);
-    if (j < 0 || j >= 100)
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(1)<<40);
-    if (big >= (UINT64_C(1)<<40))
-      allok = 0;
-    big = crypto_rand_uint64(UINT64_C(5));
-    if (big >= 5)
-      allok = 0;
-    d = crypto_rand_double();
-    tt_assert(d >= 0);
-    tt_assert(d < 1.0);
-    host = crypto_random_hostname(3,8,"www.",".onion");
-    if (strcmpstart(host,"www.") ||
-        strcmpend(host,".onion") ||
-        strlen(host) < 13 ||
-        strlen(host) > 18)
-      allok = 0;
-    tor_free(host);
-  }
-
-  /* Make sure crypto_random_hostname clips its inputs properly. */
-  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
-  tt_assert(! strcmpstart(h,"www."));
-  tt_assert(! strcmpend(h,".onion"));
-  tt_int_op(63+4+6, OP_EQ, strlen(h));
-
-  tt_assert(allok);
- done:
-  tor_free(h);
-}
-
-static void
-test_crypto_rng_range(void *arg)
-{
-  int got_smallest = 0, got_largest = 0;
-  int i;
-
-  (void)arg;
-  for (i = 0; i < 1000; ++i) {
-    int x = crypto_rand_int_range(5,9);
-    tt_int_op(x, OP_GE, 5);
-    tt_int_op(x, OP_LT, 9);
-    if (x == 5)
-      got_smallest = 1;
-    if (x == 8)
-      got_largest = 1;
-  }
-  /* These fail with probability 1/10^603. */
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  got_smallest = got_largest = 0;
-  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
-  for (i = 0; i < 1000; ++i) {
-    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
-    tt_u64_op(x, OP_GE, ten_billion);
-    tt_u64_op(x, OP_LT, ten_billion+10);
-    if (x == ten_billion)
-      got_smallest = 1;
-    if (x == ten_billion+9)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
-
-  const time_t now = time(NULL);
-  for (i = 0; i < 2000; ++i) {
-    time_t x = crypto_rand_time_range(now, now+60);
-    tt_i64_op(x, OP_GE, now);
-    tt_i64_op(x, OP_LT, now+60);
-    if (x == now)
-      got_smallest = 1;
-    if (x == now+59)
-      got_largest = 1;
-  }
-
-  tt_assert(got_smallest);
-  tt_assert(got_largest);
- done:
-  ;
-}
-
-static void
-test_crypto_rng_strongest(void *arg)
-{
-  const char *how = arg;
-  int broken = 0;
-
-  if (how == NULL) {
-    ;
-  } else if (!strcmp(how, "nosyscall")) {
-    break_strongest_rng_syscall = 1;
-  } else if (!strcmp(how, "nofallback")) {
-    break_strongest_rng_fallback = 1;
-  } else if (!strcmp(how, "broken")) {
-    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
-  }
-
-#define N 128
-  uint8_t combine_and[N];
-  uint8_t combine_or[N];
-  int i, j;
-
-  memset(combine_and, 0xff, N);
-  memset(combine_or, 0, N);
-
-  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
-    uint8_t output[N];
-    memset(output, 0, N);
-    if (how == NULL) {
-      /* this one can't fail. */
-      crypto_strongest_rand(output, sizeof(output));
-    } else {
-      int r = crypto_strongest_rand_raw(output, sizeof(output));
-      if (r == -1) {
-        if (broken) {
-          goto done; /* we're fine. */
-        }
-        /* This function is allowed to break, but only if it always breaks. */
-        tt_int_op(i, OP_EQ, 0);
-        tt_skip();
-      } else {
-        tt_assert(! broken);
-      }
-    }
-    for (j = 0; j < N; ++j) {
-      combine_and[j] &= output[j];
-      combine_or[j] |= output[j];
-    }
-  }
-
-  for (j = 0; j < N; ++j) {
-    tt_int_op(combine_and[j], OP_EQ, 0);
-    tt_int_op(combine_or[j], OP_EQ, 0xff);
-  }
- done:
-  ;
-#undef N
-}
-
 /** Run unit tests for our AES128 functionality */
 static void
 test_crypto_aes128(void *arg)
@@ -551,7 +389,7 @@ test_crypto_aes128(void *arg)
                                    "\xff\xff\xff\xff\xff\xff\xff\xff"
                                    "\xff\xff\xff\xff\xff\xff\xff\xff");
   crypto_cipher_crypt_inplace(env1, data2, 64);
-  tt_assert(tor_mem_is_zero(data2, 64));
+  tt_assert(fast_mem_is_zero(data2, 64));
 
  done:
   tor_free(mem_op_hex_tmp);
@@ -1173,13 +1011,19 @@ test_crypto_sha3_xof(void *arg)
   crypto_xof_free(xof);
   memset(out, 0, sizeof(out));
 
+  /* Test one-function absorb/squeeze. */
+  crypto_xof(out, sizeof(out), msg, sizeof(msg));
+  test_memeq_hex(out, squeezed_hex);
+  memset(out, 0, sizeof(out));
+
   /* Test incremental absorb/squeeze. */
   xof = crypto_xof_new();
   tt_assert(xof);
   for (size_t i = 0; i < sizeof(msg); i++)
     crypto_xof_add_bytes(xof, msg + i, 1);
-  for (size_t i = 0; i < sizeof(out); i++)
+  for (size_t i = 0; i < sizeof(out); i++) {
     crypto_xof_squeeze_bytes(xof, out + i, 1);
+  }
   test_memeq_hex(out, squeezed_hex);
 
  done:
@@ -1903,13 +1747,13 @@ test_crypto_base32_decode(void *arg)
   /* Encode and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Encode, uppercase, and decode a random string. */
   base32_encode(encoded, 96 + 1, plain, 60);
   tor_strupper(encoded);
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_EQ, decoded, 60);
   /* Change encoded string and decode. */
   if (encoded[0] == 'A' || encoded[0] == 'a')
@@ -1917,12 +1761,12 @@ test_crypto_base32_decode(void *arg)
   else
     encoded[0] = 'A';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(res,OP_EQ, 0);
+  tt_int_op(res, OP_EQ, 60);
   tt_mem_op(plain,OP_NE, decoded, 60);
   /* Bad encodings. */
   encoded[0] = '!';
   res = base32_decode(decoded, 60, encoded, 96);
-  tt_int_op(0, OP_GT, res);
+  tt_int_op(res, OP_LT, 0);
 
  done:
   ;
@@ -2117,7 +1961,7 @@ test_crypto_curve25519_impl(void *arg)
                                 "e0544770bc7de853b38f9100489e3e79";
   const char e1e2k_expected[] = "cd6e8269104eb5aaee886bd2071fba88"
                                 "bd13861475516bc2cd2b6e005e805064";
-#else /* !(defined(SLOW_CURVE25519_TEST)) */
+#else /* !defined(SLOW_CURVE25519_TEST) */
   const int loop_max=200;
   const char e1_expected[]    = "bc7112cde03f97ef7008cad1bdc56be3"
                                 "c6a1037d74cceb3712e9206871dcf654";
@@ -2269,7 +2113,7 @@ test_crypto_curve25519_encode(void *arg)
 
   curve25519_secret_key_generate(&seckey, 0);
   curve25519_public_key_generate(&key1, &seckey);
-  tt_int_op(0, OP_EQ, curve25519_public_to_base64(buf, &key1));
+  curve25519_public_to_base64(buf, &key1);
   tt_int_op(CURVE25519_BASE64_PADDED_LEN, OP_EQ, strlen(buf));
 
   tt_int_op(0, OP_EQ, curve25519_public_from_base64(&key2, buf));
@@ -2328,7 +2172,7 @@ test_crypto_curve25519_persist(void *arg)
   tt_u64_op((uint64_t)st.st_size, OP_EQ,
             32+CURVE25519_PUBKEY_LEN+CURVE25519_SECKEY_LEN);
   tt_assert(fast_memeq(content, "== c25519v1: testing ==", taglen));
-  tt_assert(tor_mem_is_zero(content+taglen, 32-taglen));
+  tt_assert(fast_mem_is_zero(content+taglen, 32-taglen));
   cp = content + 32;
   tt_mem_op(keypair.seckey.secret_key,OP_EQ,
              cp,
@@ -2649,13 +2493,13 @@ test_crypto_ed25519_encode(void *arg)
 
   /* Test roundtrip. */
   tt_int_op(0, OP_EQ, ed25519_keypair_generate(&kp, 0));
-  tt_int_op(0, OP_EQ, ed25519_public_to_base64(buf, &kp.pubkey));
+  ed25519_public_to_base64(buf, &kp.pubkey);
   tt_int_op(ED25519_BASE64_LEN, OP_EQ, strlen(buf));
   tt_int_op(0, OP_EQ, ed25519_public_from_base64(&pk, buf));
   tt_mem_op(kp.pubkey.pubkey, OP_EQ, pk.pubkey, ED25519_PUBKEY_LEN);
 
   tt_int_op(0, OP_EQ, ed25519_sign(&sig1, (const uint8_t*)"ABC", 3, &kp));
-  tt_int_op(0, OP_EQ, ed25519_signature_to_base64(buf, &sig1));
+  ed25519_signature_to_base64(buf, &sig1);
   tt_int_op(0, OP_EQ, ed25519_signature_from_base64(&sig2, buf));
   tt_mem_op(sig1.sig, OP_EQ, sig2.sig, ED25519_SIG_LEN);
 
@@ -3165,6 +3009,7 @@ test_crypto_failure_modes(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define CRYPTO_LEGACY(name)                                            \
   { #name, test_crypto_ ## name , 0, NULL, NULL }
 
@@ -3175,18 +3020,10 @@ test_crypto_failure_modes(void *arg)
 #define ED25519_TEST(name, fl)                  \
   ED25519_TEST_ONE(name, (fl), "donna"),        \
   ED25519_TEST_ONE(name, (fl), "ref10")
+#endif /* !defined(COCCI) */
 
 struct testcase_t crypto_tests[] = {
   CRYPTO_LEGACY(formats),
-  CRYPTO_LEGACY(rng),
-  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
-  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
-  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nosyscall" },
-  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"nofallback" },
-  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
-    &passthrough_setup, (void*)"broken" },
   { "openssl_version", test_crypto_openssl_version, TT_FORK, NULL, NULL },
   { "aes_AES", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"aes" },
   { "aes_EVP", test_crypto_aes128, TT_FORK, &passthrough_setup, (void*)"evp" },
diff --git a/src/test/test_crypto_ope.c b/src/test/test_crypto_ope.c
index dc67c02676..119ebc114a 100644
--- a/src/test/test_crypto_ope.c
+++ b/src/test/test_crypto_ope.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_crypto_openssl.c b/src/test/test_crypto_openssl.c
index 42dc3f6be2..989f4a56ca 100644
--- a/src/test/test_crypto_openssl.c
+++ b/src/test/test_crypto_openssl.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_crypto_rng.c b/src/test/test_crypto_rng.c
new file mode 100644
index 0000000000..b0dc4c117c
--- /dev/null
+++ b/src/test/test_crypto_rng.c
@@ -0,0 +1,332 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#define CRYPTO_RAND_PRIVATE
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/crypt_ops/aes.h"
+#include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
+
+/** Run unit tests for our random number generation function and its wrappers.
+ */
+static void
+test_crypto_rng(void *arg)
+{
+  int i, j, allok;
+  char data1[100], data2[100];
+  double d;
+  char *h=NULL;
+
+  /* Try out RNG. */
+  (void)arg;
+  tt_assert(! crypto_seed_rng());
+  crypto_rand(data1, 100);
+  crypto_rand(data2, 100);
+  tt_mem_op(data1,OP_NE, data2,100);
+  allok = 1;
+  for (i = 0; i < 100; ++i) {
+    uint64_t big;
+    char *host;
+    j = crypto_rand_int(100);
+    if (j < 0 || j >= 100)
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(1)<<40);
+    if (big >= (UINT64_C(1)<<40))
+      allok = 0;
+    big = crypto_rand_uint64(UINT64_C(5));
+    if (big >= 5)
+      allok = 0;
+    d = crypto_rand_double();
+    tt_assert(d >= 0);
+    tt_assert(d < 1.0);
+    host = crypto_random_hostname(3,8,"www.",".onion");
+    if (strcmpstart(host,"www.") ||
+        strcmpend(host,".onion") ||
+        strlen(host) < 13 ||
+        strlen(host) > 18)
+      allok = 0;
+    tor_free(host);
+  }
+
+  /* Make sure crypto_random_hostname clips its inputs properly. */
+  h = crypto_random_hostname(20000, 9000, "www.", ".onion");
+  tt_assert(! strcmpstart(h,"www."));
+  tt_assert(! strcmpend(h,".onion"));
+  tt_int_op(63+4+6, OP_EQ, strlen(h));
+
+  tt_assert(allok);
+ done:
+  tor_free(h);
+}
+
+static void
+test_crypto_rng_range(void *arg)
+{
+  int got_smallest = 0, got_largest = 0;
+  int i;
+
+  (void)arg;
+  for (i = 0; i < 1000; ++i) {
+    int x = crypto_rand_int_range(5,9);
+    tt_int_op(x, OP_GE, 5);
+    tt_int_op(x, OP_LT, 9);
+    if (x == 5)
+      got_smallest = 1;
+    if (x == 8)
+      got_largest = 1;
+  }
+  /* These fail with probability 1/10^603. */
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  got_smallest = got_largest = 0;
+  const uint64_t ten_billion = 10 * ((uint64_t)1000000000000);
+  for (i = 0; i < 1000; ++i) {
+    uint64_t x = crypto_rand_uint64_range(ten_billion, ten_billion+10);
+    tt_u64_op(x, OP_GE, ten_billion);
+    tt_u64_op(x, OP_LT, ten_billion+10);
+    if (x == ten_billion)
+      got_smallest = 1;
+    if (x == ten_billion+9)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+
+  const time_t now = time(NULL);
+  for (i = 0; i < 2000; ++i) {
+    time_t x = crypto_rand_time_range(now, now+60);
+    tt_i64_op(x, OP_GE, now);
+    tt_i64_op(x, OP_LT, now+60);
+    if (x == now)
+      got_smallest = 1;
+    if (x == now+59)
+      got_largest = 1;
+  }
+
+  tt_assert(got_smallest);
+  tt_assert(got_largest);
+ done:
+  ;
+}
+
+static void
+test_crypto_rng_strongest(void *arg)
+{
+  const char *how = arg;
+  int broken = 0;
+
+  if (how == NULL) {
+    ;
+  } else if (!strcmp(how, "nosyscall")) {
+    break_strongest_rng_syscall = 1;
+  } else if (!strcmp(how, "nofallback")) {
+    break_strongest_rng_fallback = 1;
+  } else if (!strcmp(how, "broken")) {
+    broken = break_strongest_rng_syscall = break_strongest_rng_fallback = 1;
+  }
+
+#define N 128
+  uint8_t combine_and[N];
+  uint8_t combine_or[N];
+  int i, j;
+
+  memset(combine_and, 0xff, N);
+  memset(combine_or, 0, N);
+
+  for (i = 0; i < 100; ++i) { /* 2^-100 chances just don't happen. */
+    uint8_t output[N];
+    memset(output, 0, N);
+    if (how == NULL) {
+      /* this one can't fail. */
+      crypto_strongest_rand(output, sizeof(output));
+    } else {
+      int r = crypto_strongest_rand_raw(output, sizeof(output));
+      if (r == -1) {
+        if (broken) {
+          goto done; /* we're fine. */
+        }
+        /* This function is allowed to break, but only if it always breaks. */
+        tt_int_op(i, OP_EQ, 0);
+        tt_skip();
+      } else {
+        tt_assert(! broken);
+      }
+    }
+    for (j = 0; j < N; ++j) {
+      combine_and[j] &= output[j];
+      combine_or[j] |= output[j];
+    }
+  }
+
+  for (j = 0; j < N; ++j) {
+    tt_int_op(combine_and[j], OP_EQ, 0);
+    tt_int_op(combine_or[j], OP_EQ, 0xff);
+  }
+ done:
+  ;
+#undef N
+}
+
+static void
+test_crypto_rng_fast(void *arg)
+{
+  (void)arg;
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  tt_assert(rng);
+
+  /* Rudimentary black-block test to make sure that our prng outputs
+   * have all bits sometimes on and all bits sometimes off. */
+  uint64_t m1 = 0, m2 = ~(uint64_t)0;
+  const int N = 128;
+
+  for (int i=0; i < N; ++i) {
+    uint64_t v;
+    crypto_fast_rng_getbytes(rng, (void*)&v, sizeof(v));
+    m1 |= v;
+    m2 &= v;
+  }
+
+  tt_u64_op(m1, OP_EQ, ~(uint64_t)0);
+  tt_u64_op(m2, OP_EQ, 0);
+
+  /* Check range functions. */
+  int counts[5];
+  memset(counts, 0, sizeof(counts));
+  for (int i=0; i < N; ++i) {
+    unsigned u = crypto_fast_rng_get_uint(rng, 5);
+    tt_int_op(u, OP_GE, 0);
+    tt_int_op(u, OP_LT, 5);
+    counts[u]++;
+
+    uint64_t u64 = crypto_fast_rng_get_uint64(rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+
+    double d = crypto_fast_rng_get_double(rng);
+    tt_assert(d >= 0.0);
+    tt_assert(d < 1.0);
+  }
+
+  /* All values should have come up once. */
+  for (int i=0; i<5; ++i) {
+    tt_int_op(counts[i], OP_GT, 0);
+  }
+
+  /* per-thread rand_fast shouldn't crash or leak. */
+  crypto_fast_rng_t *t_rng = get_thread_fast_rng();
+  for (int i = 0; i < N; ++i) {
+    uint64_t u64 = crypto_fast_rng_get_uint64(t_rng, UINT64_C(1)<<40);
+    tt_u64_op(u64, OP_GE, 0);
+    tt_u64_op(u64, OP_LT, UINT64_C(1)<<40);
+  }
+
+ done:
+  crypto_fast_rng_free(rng);
+}
+
+static void
+test_crypto_rng_fast_whitebox(void *arg)
+{
+  (void)arg;
+  const size_t buflen = crypto_fast_rng_get_bytes_used_per_stream();
+  char *buf = tor_malloc_zero(buflen);
+  char *buf2 = tor_malloc_zero(buflen);
+  char *buf3 = NULL, *buf4 = NULL;
+
+  crypto_cipher_t *cipher = NULL, *cipher2 = NULL;
+  uint8_t seed[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed, 0, sizeof(seed));
+
+  /* Start with a prng with zero key and zero IV. */
+  crypto_fast_rng_t *rng = crypto_fast_rng_new_from_seed(seed);
+  tt_assert(rng);
+
+  /* We'll use a stream cipher to keep in sync */
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+
+  /* The first 48 bytes are used for the next seed -- let's make sure we have
+   * them.
+   */
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+
+  /* if we get 128 bytes, they should match the bytes from the aes256-counter
+   * stream, starting at position 48.
+   */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 128);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* Try that again, with an odd number of bytes. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 199);
+  memset(buf2, 0, 199);
+  crypto_cipher_crypt_inplace(cipher, buf2, 199);
+  tt_mem_op(buf, OP_EQ, buf2, 199);
+
+  /* Make sure that refilling works as expected: skip all but the last 5 bytes
+   * of this steam. */
+  size_t skip = buflen - (199+128) - 5;
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, skip);
+  crypto_cipher_crypt_inplace(cipher, buf2, skip);
+
+  /* Now get the next 128 bytes. The first 5 will come from this stream, and
+   * the next 5 will come from the stream keyed by the new value of 'seed'. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 128);
+  memset(buf2, 0, 128);
+  crypto_cipher_crypt_inplace(cipher, buf2, 5);
+  crypto_cipher_free(cipher);
+  cipher = crypto_cipher_new_with_iv_and_bits(seed, seed+32, 256);
+  memset(seed, 0, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed, sizeof(seed));
+  crypto_cipher_crypt_inplace(cipher, buf2+5, 128-5);
+  tt_mem_op(buf, OP_EQ, buf2, 128);
+
+  /* And check the next 7 bytes to make sure we didn't discard anything. */
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf, 7);
+  memset(buf2, 0, 7);
+  crypto_cipher_crypt_inplace(cipher, buf2, 7);
+  tt_mem_op(buf, OP_EQ, buf2, 7);
+
+  /* Now try the optimization for long outputs. */
+  buf3 = tor_malloc(65536);
+  crypto_fast_rng_getbytes(rng, (uint8_t*)buf3, 65536);
+
+  buf4 = tor_malloc_zero(65536);
+  uint8_t seed2[CRYPTO_FAST_RNG_SEED_LEN];
+  memset(seed2, 0, sizeof(seed2));
+  crypto_cipher_crypt_inplace(cipher, (char*)seed2, sizeof(seed2));
+  cipher2 = crypto_cipher_new_with_iv_and_bits(seed2, seed2+32, 256);
+  crypto_cipher_crypt_inplace(cipher2, buf4, 65536);
+  tt_mem_op(buf3, OP_EQ, buf4, 65536);
+
+ done:
+  crypto_fast_rng_free(rng);
+  crypto_cipher_free(cipher);
+  crypto_cipher_free(cipher2);
+  tor_free(buf);
+  tor_free(buf2);
+  tor_free(buf3);
+  tor_free(buf4);
+}
+
+struct testcase_t crypto_rng_tests[] = {
+  { "rng", test_crypto_rng, 0, NULL, NULL },
+  { "rng_range", test_crypto_rng_range, 0, NULL, NULL },
+  { "rng_strongest", test_crypto_rng_strongest, TT_FORK, NULL, NULL },
+  { "rng_strongest_nosyscall", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nosyscall" },
+  { "rng_strongest_nofallback", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"nofallback" },
+  { "rng_strongest_broken", test_crypto_rng_strongest, TT_FORK,
+    &passthrough_setup, (void*)"broken" },
+  { "fast", test_crypto_rng_fast, 0, NULL, NULL },
+  { "fast_whitebox", test_crypto_rng_fast_whitebox, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_crypto_slow.c b/src/test/test_crypto_slow.c
index e24aee8930..1702427b08 100644
--- a/src/test/test_crypto_slow.c
+++ b/src/test/test_crypto_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -109,7 +109,7 @@ run_s2k_tests(const unsigned flags, const unsigned type,
             secret_to_key_derivekey(buf3, sizeof(buf3), buf, speclen,
                                     pw1, strlen(pw1)));
   tt_mem_op(buf2, OP_EQ, buf3, sizeof(buf3));
-  tt_assert(!tor_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
+  tt_assert(!fast_mem_is_zero((char*)buf2+keylen, sizeof(buf2)-keylen));
 
  done:
   ;
@@ -342,7 +342,7 @@ test_crypto_scrypt_vectors(void *arg)
 #endif
 
   /* Test vectors from
-     http://tools.ietf.org/html/draft-josefsson-scrypt-kdf-00 section 11.
+     https://tools.ietf.org/html/draft-josefsson-scrypt-kdf-00 section 11.
 
      Note that the names of 'r' and 'N' are switched in that section. Or
      possibly in libscrypt.
@@ -584,6 +584,7 @@ test_crypto_ed25519_fuzz_donna(void *arg)
   ;
 }
 
+#ifndef COCCI
 #define CRYPTO_LEGACY(name)                                            \
   { #name, test_crypto_ ## name , 0, NULL, NULL }
 
@@ -594,6 +595,7 @@ test_crypto_ed25519_fuzz_donna(void *arg)
 #define ED25519_TEST(name, fl)                  \
   ED25519_TEST_ONE(name, (fl), "donna"),        \
   ED25519_TEST_ONE(name, (fl), "ref10")
+#endif /* !defined(COCCI) */
 
 struct testcase_t slow_crypto_tests[] = {
   CRYPTO_LEGACY(s2k_rfc2440),
diff --git a/src/test/test_data.c b/src/test/test_data.c
index fe1190ea77..30c14fcfff 100644
--- a/src/test/test_data.c
+++ b/src/test/test_data.c
@@ -1,6 +1,6 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "test/test.h"
diff --git a/src/test/test_dir.c b/src/test/test_dir.c
index 0e44c47f3f..3a0b8237cb 100644
--- a/src/test/test_dir.c
+++ b/src/test/test_dir.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -8,10 +8,10 @@
 
 #define BWAUTH_PRIVATE
 #define CONFIG_PRIVATE
-#define CONTROL_PRIVATE
+#define CONTROL_GETINFO_PRIVATE
+#define DIRAUTH_SYS_PRIVATE
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
-#define DIRSERV_PRIVATE
 #define DIRVOTE_PRIVATE
 #define DLSTATUS_PRIVATE
 #define HIBERNATE_PRIVATE
@@ -26,14 +26,15 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/connection.h"
 #include "core/or/relay.h"
 #include "core/or/versions.h"
 #include "feature/client/bridges.h"
 #include "feature/client/entrynodes.h"
-#include "feature/control/control.h"
+#include "feature/control/control_getinfo.h"
 #include "feature/dirauth/bwauth.h"
+#include "feature/dirauth/dirauth_sys.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/dsigs_parse.h"
 #include "feature/dirauth/process_descs.h"
@@ -46,10 +47,11 @@
 #include "feature/dirclient/dlstatus.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dircommon/fp_pair.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/hibernate/hibernate.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
+#include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
 #include "feature/nodelist/nickname.h"
 #include "feature/nodelist/node_select.h"
@@ -71,13 +73,16 @@
 #include "lib/memarea/memarea.h"
 #include "lib/osinfo/uname.h"
 #include "test/log_test_helpers.h"
+#include "test/opts_test_helpers.h"
 #include "test/test.h"
 #include "test/test_dir_common.h"
 
 #include "core/or/addr_policy_st.h"
+#include "feature/dirauth/dirauth_options_st.h"
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/document_signature_st.h"
 #include "feature/nodelist/extrainfo_st.h"
+#include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/networkstatus_voter_info_st.h"
 #include "feature/dirauth/ns_detached_signatures_st.h"
@@ -91,8 +96,26 @@
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
 #endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
 
-#define NS_MODULE dir
+static networkstatus_t *
+networkstatus_parse_vote_from_string_(const char *s,
+                                      const char **eos_out,
+                                      enum networkstatus_type_t ns_type)
+{
+  size_t len = strlen(s);
+  // memdup so that it won't be nul-terminated.
+  char *tmp = tor_memdup(s, len);
+  networkstatus_t *result =
+    networkstatus_parse_vote_from_string(tmp, len, eos_out, ns_type);
+  if (eos_out && *eos_out) {
+    *eos_out = s + (*eos_out - tmp);
+  }
+  tor_free(tmp);
+  return result;
+}
 
 static void
 test_dir_nicknames(void *arg)
@@ -142,6 +165,269 @@ test_dir_nicknames(void *arg)
   ;
 }
 
+/* Allocate and return a new routerinfo, with the fields set from the
+ * arguments to this function.
+ *
+ * Also sets:
+ *  - random RSA identity and onion keys,
+ *  - the platform field using get_platform_str(), and
+ *  - supports_tunnelled_dir_requests to 1.
+ *
+ * If rsa_onion_keypair_out is not NULL, it is set to the onion keypair.
+ * The caller must free this keypair.
+ */
+static routerinfo_t *
+basic_routerinfo_new(const char *nickname, uint32_t ipv4_addr,
+                     uint16_t or_port, uint16_t dir_port,
+                     uint32_t bandwidthrate, uint32_t bandwidthburst,
+                     uint32_t bandwidthcapacity,
+                     time_t published_on,
+                     crypto_pk_t **rsa_onion_keypair_out)
+{
+  char platform[256];
+
+  tor_assert(nickname);
+
+  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
+  /* These keys are random: idx is ignored. */
+  pk1 = pk_generate(0);
+  pk2 = pk_generate(1);
+
+  tor_assert(pk1);
+  tor_assert(pk2);
+
+  get_platform_str(platform, sizeof(platform));
+
+  routerinfo_t *r1 = tor_malloc_zero(sizeof(routerinfo_t));
+
+  r1->nickname = tor_strdup(nickname);
+  r1->platform = tor_strdup(platform);
+
+  r1->addr = ipv4_addr;
+  r1->or_port = or_port;
+  r1->dir_port = dir_port;
+  r1->supports_tunnelled_dir_requests = 1;
+
+  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
+  r1->identity_pkey = pk2;
+
+  r1->bandwidthrate = bandwidthrate;
+  r1->bandwidthburst = bandwidthburst;
+  r1->bandwidthcapacity = bandwidthcapacity;
+
+  r1->cache_info.published_on = published_on;
+
+  if (rsa_onion_keypair_out) {
+    *rsa_onion_keypair_out = pk1;
+  } else {
+    crypto_pk_free(pk1);
+  }
+
+  return r1;
+}
+
+/* Allocate and return a new string containing a "router" line for r1. */
+static char *
+get_new_router_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "router %s %s %d 0 %d\n",
+               r1->nickname, fmt_addr32(r1->addr),
+               r1->or_port, r1->dir_port);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "platform" line for the
+ * current Tor version and OS. */
+static char *
+get_new_platform_line(void)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "platform Tor %s on %s\n",
+               VERSION, get_uname());
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "published" line for r1.
+ * r1->cache_info.published_on must be between 0 and 59 seconds. */
+static char *
+get_new_published_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_assert(r1->cache_info.published_on >= 0);
+  tor_assert(r1->cache_info.published_on <= 59);
+
+  tor_asprintf(&line,
+               "published 1970-01-01 00:00:%02u\n",
+               (unsigned)r1->cache_info.published_on);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "fingerprint" line for r1. */
+static char *
+get_new_fingerprint_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+  char fingerprint[FINGERPRINT_LEN+1];
+
+  tor_assert(r1);
+
+  tor_assert(!crypto_pk_get_fingerprint(r1->identity_pkey, fingerprint, 1));
+  tor_assert(strlen(fingerprint) > 0);
+
+  tor_asprintf(&line,
+               "fingerprint %s\n",
+               fingerprint);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "uptime" line with uptime t.
+ *
+ * You should pass a hard-coded value to this function, because even if we made
+ * it reflect uptime, that still wouldn't make it right, because the two
+ * descriptors might be made on different seconds.
+ */
+static char *
+get_new_uptime_line(time_t t)
+{
+  char *line = NULL;
+
+  tor_asprintf(&line,
+               "uptime %u\n",
+               (unsigned)t);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing an "bandwidth" line for r1.
+ */
+static char *
+get_new_bandwidth_line(const routerinfo_t *r1)
+{
+  char *line = NULL;
+
+  tor_assert(r1);
+
+  tor_asprintf(&line,
+               "bandwidth %u %u %u\n",
+               r1->bandwidthrate,
+               r1->bandwidthburst,
+               r1->bandwidthcapacity);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a key_name block for the
+ * RSA key pk1.
+ */
+static char *
+get_new_rsa_key_block(const char *key_name, crypto_pk_t *pk1)
+{
+  char *block = NULL;
+  char *pk1_str = NULL;
+  size_t pk1_str_len = 0;
+
+  tor_assert(key_name);
+  tor_assert(pk1);
+
+  tor_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
+                                                   &pk1_str_len));
+  tor_assert(pk1_str);
+  tor_assert(pk1_str_len);
+
+  tor_asprintf(&block,
+               "%s\n%s",
+               key_name,
+               pk1_str);
+  tor_free(pk1_str);
+
+  tor_assert(block);
+  return block;
+}
+
+/* Allocate and return a new string containing an "onion-key" block for the
+ * router r1.
+ */
+static char *
+get_new_onion_key_block(const routerinfo_t *r1)
+{
+  char *block = NULL;
+  tor_assert(r1);
+  crypto_pk_t *pk_tmp = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                  r1->onion_pkey_len);
+  block = get_new_rsa_key_block("onion-key", pk_tmp);
+  crypto_pk_free(pk_tmp);
+  return block;
+}
+
+/* Allocate and return a new string containing an "signing-key" block for the
+ * router r1.
+ */
+static char *
+get_new_signing_key_block(const routerinfo_t *r1)
+{
+  tor_assert(r1);
+  return get_new_rsa_key_block("signing-key", r1->identity_pkey);
+}
+
+/* Allocate and return a new string containing an "ntor-onion-key" line for
+ * the curve25519 public key ntor_onion_pubkey.
+ */
+static char *
+get_new_ntor_onion_key_line(const curve25519_public_key_t *ntor_onion_pubkey)
+{
+  char *line = NULL;
+  char cert_buf[256];
+  int rv = 0;
+
+  tor_assert(ntor_onion_pubkey);
+
+  rv = base64_encode(cert_buf, sizeof(cert_buf),
+                     (const char*)ntor_onion_pubkey->public_key, 32,
+                     BASE64_ENCODE_MULTILINE);
+  tor_assert(rv > 0);
+  tor_assert(strlen(cert_buf) > 0);
+
+  tor_asprintf(&line,
+               "ntor-onion-key %s",
+               cert_buf);
+  tor_assert(line);
+
+  return line;
+}
+
+/* Allocate and return a new string containing a "bridge-distribution-request"
+ * line for options.
+ */
+static char *
+get_new_bridge_distribution_request_line(const or_options_t *options)
+{
+  if (options->BridgeRelay) {
+    return tor_strdup("bridge-distribution-request any\n");
+  } else {
+    return tor_strdup("");
+  }
+}
+
 static smartlist_t *mocked_configured_ports = NULL;
 
 /** Returns mocked_configured_ports */
@@ -151,71 +437,509 @@ mock_get_configured_ports(void)
   return mocked_configured_ports;
 }
 
-/** Run unit tests for router descriptor generation logic. */
+static tor_cert_t *
+mock_tor_cert_dup_null(const tor_cert_t *cert)
+{
+  (void)cert;
+  return NULL;
+}
+
+static crypto_pk_t *mocked_server_identitykey = NULL;
+
+/* Returns mocked_server_identitykey with no checks. */
+static crypto_pk_t *
+mock_get_server_identity_key(void)
+{
+  return mocked_server_identitykey;
+}
+
+static crypto_pk_t *mocked_onionkey = NULL;
+
+/* Returns mocked_onionkey with no checks. */
+static crypto_pk_t *
+mock_get_onion_key(void)
+{
+  return mocked_onionkey;
+}
+
+static routerinfo_t *mocked_routerinfo = NULL;
+
+/* Returns 0 and sets ri_out to mocked_routerinfo.
+ * ri_out must not be NULL. There are no other checks. */
+static int
+mock_router_build_fresh_unsigned_routerinfo(routerinfo_t **ri_out)
+{
+  tor_assert(ri_out);
+  *ri_out = mocked_routerinfo;
+  return 0;
+}
+
+static ed25519_keypair_t *mocked_master_signing_key = NULL;
+
+/* Returns mocked_master_signing_key with no checks. */
+static const ed25519_keypair_t *
+mock_get_master_signing_keypair(void)
+{
+  return mocked_master_signing_key;
+}
+
+static struct tor_cert_st *mocked_signing_key_cert = NULL;
+
+/* Returns mocked_signing_key_cert with no checks. */
+static const struct tor_cert_st *
+mock_get_master_signing_key_cert(void)
+{
+  return mocked_signing_key_cert;
+}
+
+static curve25519_keypair_t *mocked_curve25519_onion_key = NULL;
+
+/* Returns mocked_curve25519_onion_key with no checks. */
+static const curve25519_keypair_t *
+mock_get_current_curve25519_keypair(void)
+{
+  return mocked_curve25519_onion_key;
+}
+
+/* Unmock get_configured_ports() and free mocked_configured_ports. */
 static void
-test_dir_formats(void *arg)
+cleanup_mock_configured_ports(void)
+{
+  UNMOCK(get_configured_ports);
+
+  if (mocked_configured_ports) {
+    SMARTLIST_FOREACH(mocked_configured_ports, port_cfg_t *, p, tor_free(p));
+    smartlist_free(mocked_configured_ports);
+  }
+}
+
+/* Mock get_configured_ports() with a list containing or_port and dir_port.
+ * If a port is 0, don't set it.
+ * Only sets the minimal data required for the tests to pass. */
+static void
+setup_mock_configured_ports(uint16_t or_port, uint16_t dir_port)
+{
+  cleanup_mock_configured_ports();
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  MOCK(get_configured_ports, mock_get_configured_ports);
+  mocked_configured_ports = smartlist_new();
+
+  if (or_port) {
+    port_cfg_t *or_port_cfg = tor_malloc_zero(sizeof(*or_port_cfg));
+    or_port_cfg->type = CONN_TYPE_OR_LISTENER;
+    or_port_cfg->addr.family = AF_INET;
+    or_port_cfg->port = or_port;
+    smartlist_add(mocked_configured_ports, or_port_cfg);
+  }
+
+  if (dir_port) {
+    port_cfg_t *dir_port_cfg = tor_malloc_zero(sizeof(*dir_port_cfg));
+    dir_port_cfg->type = CONN_TYPE_DIR_LISTENER;
+    dir_port_cfg->addr.family = AF_INET;
+    dir_port_cfg->port = dir_port;
+    smartlist_add(mocked_configured_ports, dir_port_cfg);
+  }
+}
+
+/* Clean up the data structures and unmock the functions needed for generating
+ * a fresh descriptor. */
+static void
+cleanup_mocks_for_fresh_descriptor(void)
+{
+  tor_free(get_options_mutable()->Nickname);
+
+  mocked_server_identitykey = NULL;
+  UNMOCK(get_server_identity_key);
+
+  crypto_pk_free(mocked_onionkey);
+  UNMOCK(get_onion_key);
+}
+
+/* Mock the data structures and functions needed for generating a fresh
+ * descriptor.
+ *
+ * Sets options->Nickname from r1->nickname.
+ * Mocks get_server_identity_key() with r1->identity_pkey.
+ *
+ * If rsa_onion_keypair is not NULL, it is used to mock get_onion_key().
+ * Otherwise, the public key in r1->onion_pkey is used to mock get_onion_key().
+ */
+static void
+setup_mocks_for_fresh_descriptor(const routerinfo_t *r1,
+                                 crypto_pk_t *rsa_onion_keypair)
+{
+  cleanup_mocks_for_fresh_descriptor();
+
+  tor_assert(r1);
+
+  /* router_build_fresh_signed_extrainfo() requires options->Nickname */
+  get_options_mutable()->Nickname = tor_strdup(r1->nickname);
+
+  /* router_build_fresh_signed_extrainfo() requires get_server_identity_key().
+   * Use the same one as the call to router_dump_router_to_string() above.
+   */
+  mocked_server_identitykey = r1->identity_pkey;
+  MOCK(get_server_identity_key, mock_get_server_identity_key);
+
+  /* router_dump_and_sign_routerinfo_descriptor_body() requires
+   * get_onion_key(). Use the same one as r1.
+   */
+  if (rsa_onion_keypair) {
+    mocked_onionkey = crypto_pk_dup_key(rsa_onion_keypair);
+  } else {
+    mocked_onionkey = router_get_rsa_onion_pkey(r1->onion_pkey,
+                                                r1->onion_pkey_len);
+  }
+  MOCK(get_onion_key, mock_get_onion_key);
+}
+
+/* Set options based on arg.
+ *
+ * b: BridgeRelay 1
+ * e: ExtraInfoStatistics 1
+ * s: sets all the individual statistics options to 1
+ *
+ * Always sets AssumeReachable to 1.
+ *
+ * Does not set ServerTransportPlugin, because it's parsed before use.
+ *
+ * Does not set BridgeRecordUsageByCountry, because the tests don't have access
+ * to a GeoIPFile or GeoIPv6File. */
+static void
+setup_dir_formats_options(const char *arg, or_options_t *options)
+{
+  /* Skip reachability checks for DirPort, ORPort, and tunnelled-dir-server */
+  options->AssumeReachable = 1;
+
+  if (strchr(arg, 'b')) {
+    options->BridgeRelay = 1;
+  }
+
+  if (strchr(arg, 'e')) {
+    options->ExtraInfoStatistics = 1;
+  }
+
+  if (strchr(arg, 's')) {
+    options->DirReqStatistics = 1;
+    options->HiddenServiceStatistics = 1;
+    options->EntryStatistics = 1;
+    options->CellStatistics = 1;
+    options->ExitPortStatistics = 1;
+    options->ConnDirectionStatistics = 1;
+    options->PaddingStatistics = 1;
+  }
+}
+
+/* Check that routerinfos r1 and rp1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_ROUTERINFO_CONSISTENCY(r1, rp1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(rp1); \
+  tt_int_op(rp1->addr,OP_EQ, r1->addr); \
+  tt_int_op(rp1->or_port,OP_EQ, r1->or_port); \
+  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port); \
+  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate); \
+  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst); \
+  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity); \
+  crypto_pk_t *rp1_onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey, \
+                                                      rp1->onion_pkey_len); \
+  crypto_pk_t *r1_onion_pkey = router_get_rsa_onion_pkey(r1->onion_pkey, \
+                                                      r1->onion_pkey_len); \
+  tt_int_op(crypto_pk_cmp_keys(rp1_onion_pkey, r1_onion_pkey), OP_EQ, 0); \
+  crypto_pk_free(rp1_onion_pkey); \
+  crypto_pk_free(r1_onion_pkey); \
+  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, r1->identity_pkey), \
+            OP_EQ, 0); \
+  tt_int_op(rp1->supports_tunnelled_dir_requests, OP_EQ, \
+            r1->supports_tunnelled_dir_requests); \
+STMT_END
+
+/* Check that routerinfo r1 and extrainfo e1 are consistent.
+ * Only performs some basic checks.
+ */
+#define CHECK_EXTRAINFO_CONSISTENCY(r1, e1) \
+STMT_BEGIN \
+  tt_assert(r1); \
+  tt_assert(e1); \
+\
+  tt_str_op(e1->nickname, OP_EQ, r1->nickname); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA-only
+ * router. Tor versions without ed25519 (0.2.6 and earlier) are no longer
+ * officially supported, but the authorities still accept their descriptors.
+ */
+static void
+test_dir_formats_rsa(void *arg)
 {
   char *buf = NULL;
-  char buf2[8192];
-  char platform[256];
-  char fingerprint[FINGERPRINT_LEN+1];
-  char *pk1_str = NULL, *pk2_str = NULL, *cp;
-  size_t pk1_str_len, pk2_str_len;
-  routerinfo_t *r1=NULL, *r2=NULL;
-  crypto_pk_t *pk1 = NULL, *pk2 = NULL;
-  routerinfo_t *rp1 = NULL, *rp2 = NULL;
-  addr_policy_t *ex1, *ex2;
-  routerlist_t *dir1 = NULL, *dir2 = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
   uint8_t *rsa_cc = NULL;
-  or_options_t *options = get_options_mutable();
-  const addr_policy_t *p;
-  time_t now = time(NULL);
-  port_cfg_t orport, dirport;
-  char cert_buf[256];
 
-  (void)arg;
-  pk1 = pk_generate(0);
-  pk2 = pk_generate(1);
+  routerinfo_t *r1 = NULL;
+  extrainfo_t *e1 = NULL;
+  routerinfo_t *rp1 = NULL;
+  extrainfo_t *ep1 = NULL;
 
-  tt_assert(pk1 && pk2);
+  smartlist_t *chunks = NULL;
+  const char *msg = NULL;
+  int rv = -1;
+
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
 
   hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
 
-  get_platform_str(platform, sizeof(platform));
-  r1 = tor_malloc_zero(sizeof(routerinfo_t));
-  r1->addr = 0xc0a80001u; /* 192.168.0.1 */
-  r1->cache_info.published_on = 0;
-  r1->or_port = 9000;
-  r1->dir_port = 9003;
-  r1->supports_tunnelled_dir_requests = 1;
-  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
-  r1->ipv6_orport = 9999;
-  router_set_rsa_onion_pkey(pk1, &r1->onion_pkey, &r1->onion_pkey_len);
-  /* Fake just enough of an ntor key to get by */
+  /* r1 is a minimal, RSA-only descriptor, with DirPort and IPv6 */
+  r1 = basic_routerinfo_new("Magri", 0xc0a80001u /* 192.168.0.1 */,
+                            9000, 9003,
+                            1000, 5000, 10000,
+                            0,
+                            NULL);
+
+ /* Fake just enough of an ntor key to get by */
   curve25519_keypair_t r1_onion_keypair;
   curve25519_keypair_generate(&r1_onion_keypair, 0);
   r1->onion_curve25519_pkey = tor_memdup(&r1_onion_keypair.pubkey,
                                          sizeof(curve25519_public_key_t));
-  r1->identity_pkey = crypto_pk_dup_key(pk2);
-  r1->bandwidthrate = 1000;
-  r1->bandwidthburst = 5000;
-  r1->bandwidthcapacity = 10000;
+
+  /* Now add IPv6 */
+  tor_addr_parse(&r1->ipv6_addr, "1:2:3:4::");
+  r1->ipv6_orport = 9999;
+
   r1->exit_policy = NULL;
-  r1->nickname = tor_strdup("Magri");
-  r1->platform = tor_strdup(platform);
 
-  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
-  ex1->policy_type = ADDR_POLICY_ACCEPT;
-  tor_addr_from_ipv4h(&ex1->addr, 0);
-  ex1->maskbits = 0;
-  ex1->prt_min = ex1->prt_max = 80;
-  ex2->policy_type = ADDR_POLICY_REJECT;
-  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
-  ex2->maskbits = 8;
-  ex2->prt_min = ex2->prt_max = 24;
-  r2 = tor_malloc_zero(sizeof(routerinfo_t));
-  r2->addr = 0x0a030201u; /* 10.3.2.1 */
+  /* XXXX+++ router_dump_to_string should really take this from ri. */
+  options->ContactInfo = tor_strdup("Magri White "
+                                    "<magri@elsewhere.example.com>");
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  tor_free(options->ContactInfo);
+  cleanup_mock_configured_ports();
+
+  /* Synthesise a router descriptor, without the signature */
+  chunks = smartlist_new();
+
+  smartlist_add(chunks, get_new_router_line(r1));
+  smartlist_add_strdup(chunks, "or-address [1:2:3:4::]:9999\n");
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r1));
+  smartlist_add(chunks, get_new_fingerprint_line(r1));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r1));
+
+  smartlist_add(chunks, get_new_onion_key_block(r1));
+  smartlist_add(chunks, get_new_signing_key_block(r1));
+
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
+
+  smartlist_add_strdup(chunks, "contact Magri White "
+                               "<magri@elsewhere.example.com>\n");
+
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r1_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "reject *:*\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
+
+  smartlist_add_strdup(chunks, "router-signature\n");
+
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+                             * twice */
+
+  tt_str_op(buf,OP_EQ, buf2);
+  tor_free(buf);
+
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  buf = router_dump_router_to_string(r1, r1->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
+
+  cleanup_mock_configured_ports();
+
+  /* Now, try to parse buf */
+  cp = buf;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  tor_free(buf);
+  routerinfo_free(rp1);
+
+  /* Test extrainfo creation.
+   * We avoid calling router_build_fresh_unsigned_routerinfo(), because it's
+   * too complex. Instead, we re-use the manually-created routerinfos.
+   */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r1, NULL);
+
+  /* router_build_fresh_signed_extrainfo() passes the result of
+   * get_master_signing_key_cert() directly to tor_cert_dup(), which fails on
+   * NULL. But we want a NULL ei->cache_info.signing_key_cert to test the
+   * non-ed key path.
+   */
+  MOCK(tor_cert_dup, mock_tor_cert_dup_null);
+
+  /* Fake just enough of an ORPort and DirPort to get by */
+  setup_mock_configured_ports(r1->or_port, r1->dir_port);
+
+  /* Test some of the low-level static functions. */
+  e1 = router_build_fresh_signed_extrainfo(r1);
+  tt_assert(e1);
+  router_update_routerinfo_from_extrainfo(r1, e1);
+  rv = router_dump_and_sign_routerinfo_descriptor_body(r1);
+  tt_assert(rv == 0);
+  msg = "";
+  rv = routerinfo_incompatible_with_extrainfo(r1->identity_pkey, e1,
+                                              &r1->cache_info, &msg);
+  /* If they are incompatible, fail and show the msg string */
+  tt_str_op(msg, OP_EQ, "");
+  tt_assert(rv == 0);
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  UNMOCK(tor_cert_dup);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, e1);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r1->cache_info.signed_descriptor_body);
+  cp = r1->cache_info.signed_descriptor_body;
+  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r1, rp1);
+
+  tt_assert(rp1->policy_is_reject_star);
+
+  routerinfo_free(rp1);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e1->cache_info.signed_descriptor_body);
+  cp = e1->cache_info.signed_descriptor_body;
+  ep1 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r1, ep1);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep1);
+
+ done:
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->ContactInfo);
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r1);
+  routerinfo_free(rp1);
+
+  extrainfo_free(e1);
+  extrainfo_free(ep1);
+
+  tor_free(rsa_cc);
+
+  tor_free(buf);
+  tor_free(buf2);
+}
+
+/* Check that the exit policy in rp2 is as expected. */
+#define CHECK_PARSED_EXIT_POLICY(rp2) \
+STMT_BEGIN \
+  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2); \
+ \
+  p = smartlist_get(rp2->exit_policy, 0); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT); \
+  tt_assert(tor_addr_is_null(&p->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 0); \
+  tt_int_op(p->prt_min,OP_EQ, 80); \
+  tt_int_op(p->prt_max,OP_EQ, 80); \
+ \
+  p = smartlist_get(rp2->exit_policy, 1); \
+  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT); \
+  tt_assert(tor_addr_eq(&p->addr, &ex2->addr)); \
+  tt_int_op(p->maskbits,OP_EQ, 8); \
+  tt_int_op(p->prt_min,OP_EQ, 24); \
+  tt_int_op(p->prt_max,OP_EQ, 24); \
+STMT_END
+
+/** Run unit tests for router descriptor generation logic for a RSA + ed25519
+ * router.
+ */
+static void
+test_dir_formats_rsa_ed25519(void *arg)
+{
+  char *buf = NULL;
+  char *buf2 = NULL;
+  char *cp = NULL;
+
+  crypto_pk_t *r2_onion_pkey = NULL;
+  char cert_buf[256];
+  uint8_t *rsa_cc = NULL;
+  time_t now = time(NULL);
+
+  routerinfo_t *r2 = NULL;
+  extrainfo_t *e2 = NULL;
+  routerinfo_t *r2_out = NULL;
+  routerinfo_t *rp2 = NULL;
+  extrainfo_t *ep2 = NULL;
+  addr_policy_t *ex1, *ex2;
+  const addr_policy_t *p;
+
+  smartlist_t *chunks = NULL;
+  int rv = -1;
+
+  or_options_t *options = get_options_mutable();
+  setup_dir_formats_options((const char *)arg, options);
+
+  hibernate_set_state_for_testing_(HIBERNATE_STATE_LIVE);
+
+  /* r2 is a RSA + ed25519 descriptor, with an exit policy, but no DirPort or
+   * IPv6 */
+  r2 = basic_routerinfo_new("Fred", 0x0a030201u /* 10.3.2.1 */,
+                            9005, 0,
+                            3000, 3000, 3000,
+                            5,
+                            &r2_onion_pkey);
+
+  /* Fake just enough of an ntor key to get by */
+  curve25519_keypair_t r2_onion_keypair;
+  curve25519_keypair_generate(&r2_onion_keypair, 0);
+  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
+                                         sizeof(curve25519_public_key_t));
+
+  /* Now add relay ed25519 keys
+   * We can't use init_mock_ed_keys() here, because the keys are seeded */
   ed25519_keypair_t kp1, kp2;
   ed25519_secret_key_from_seed(&kp1.seckey,
                           (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
@@ -228,157 +952,78 @@ test_dir_formats(void *arg)
                                          &kp2.pubkey,
                                          now, 86400,
                                          CERT_FLAG_INCLUDE_SIGNING_KEY);
-  r2->platform = tor_strdup(platform);
-  r2->cache_info.published_on = 5;
-  r2->or_port = 9005;
-  r2->dir_port = 0;
-  r2->supports_tunnelled_dir_requests = 1;
-  router_set_rsa_onion_pkey(pk2, &r2->onion_pkey, &r2->onion_pkey_len);
-  curve25519_keypair_t r2_onion_keypair;
-  curve25519_keypair_generate(&r2_onion_keypair, 0);
-  r2->onion_curve25519_pkey = tor_memdup(&r2_onion_keypair.pubkey,
-                                         sizeof(curve25519_public_key_t));
-  r2->identity_pkey = crypto_pk_dup_key(pk1);
-  r2->bandwidthrate = r2->bandwidthburst = r2->bandwidthcapacity = 3000;
+
+  /* Now add an exit policy */
+  ex1 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex2 = tor_malloc_zero(sizeof(addr_policy_t));
+  ex1->policy_type = ADDR_POLICY_ACCEPT;
+  tor_addr_from_ipv4h(&ex1->addr, 0);
+  ex1->maskbits = 0;
+  ex1->prt_min = ex1->prt_max = 80;
+  ex2->policy_type = ADDR_POLICY_REJECT;
+  tor_addr_from_ipv4h(&ex2->addr, 18<<24);
+  ex2->maskbits = 8;
+  ex2->prt_min = ex2->prt_max = 24;
+
   r2->exit_policy = smartlist_new();
   smartlist_add(r2->exit_policy, ex1);
   smartlist_add(r2->exit_policy, ex2);
-  r2->nickname = tor_strdup("Fred");
-
-  tt_assert(!crypto_pk_write_public_key_to_string(pk1, &pk1_str,
-                                                    &pk1_str_len));
-  tt_assert(!crypto_pk_write_public_key_to_string(pk2 , &pk2_str,
-                                                    &pk2_str_len));
-
-  /* XXXX+++ router_dump_to_string should really take this from ri.*/
-  options->ContactInfo = tor_strdup("Magri White "
-                                    "<magri@elsewhere.example.com>");
-  /* Skip reachability checks for DirPort and tunnelled-dir-server */
-  options->AssumeReachable = 1;
-
-  /* Fake just enough of an ORPort and DirPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
 
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9000;
-  smartlist_add(mocked_configured_ports, &orport);
-
-  memset(&dirport, 0, sizeof(dirport));
-  dirport.type = CONN_TYPE_DIR_LISTENER;
-  dirport.addr.family = AF_INET;
-  dirport.port = 9003;
-  smartlist_add(mocked_configured_ports, &dirport);
-
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  tor_free(options->ContactInfo);
+  buf = router_dump_router_to_string(r2,
+                                     r2->identity_pkey, r2_onion_pkey,
+                                     &r2_onion_keypair, &kp2);
   tt_assert(buf);
 
-  strlcpy(buf2, "router Magri 192.168.0.1 9000 0 9003\n"
-          "or-address [1:2:3:4::]:9999\n"
-          "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:00\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk2, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-  /* XXX the "0" above is hard-coded, but even if we made it reflect
-   * uptime, that still wouldn't make it right, because the two
-   * descriptors might be made on different seconds... hm. */
-         "bandwidth 1000 5000 10000\n"
-          "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "contact Magri White <magri@elsewhere.example.com>\n",
-          sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r1_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "reject *:*\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\nrouter-signature\n", sizeof(buf2));
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
-                             * twice */
+  cleanup_mock_configured_ports();
 
-  tt_str_op(buf,OP_EQ, buf2);
-  tor_free(buf);
+  chunks = smartlist_new();
 
-  buf = router_dump_router_to_string(r1, pk2, NULL, NULL, NULL);
-  tt_assert(buf);
-  cp = buf;
-  rp1 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp1);
-  tt_int_op(rp1->addr,OP_EQ, r1->addr);
-  tt_int_op(rp1->or_port,OP_EQ, r1->or_port);
-  tt_int_op(rp1->dir_port,OP_EQ, r1->dir_port);
-  tt_int_op(rp1->bandwidthrate,OP_EQ, r1->bandwidthrate);
-  tt_int_op(rp1->bandwidthburst,OP_EQ, r1->bandwidthburst);
-  tt_int_op(rp1->bandwidthcapacity,OP_EQ, r1->bandwidthcapacity);
-  crypto_pk_t *onion_pkey = router_get_rsa_onion_pkey(rp1->onion_pkey,
-                                                      rp1->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk1), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp1->identity_pkey, pk2), OP_EQ, 0);
-  tt_assert(rp1->supports_tunnelled_dir_requests);
-  //tt_assert(rp1->exit_policy == NULL);
-  tor_free(buf);
+  /* Synthesise a router descriptor, without the signatures */
+  smartlist_add(chunks, get_new_router_line(r2));
 
-  strlcpy(buf2,
-          "router Fred 10.3.2.1 9005 0 0\n"
-          "identity-ed25519\n"
-          "-----BEGIN ED25519 CERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks,
+                       "identity-ed25519\n"
+                       "-----BEGIN ED25519 CERT-----\n");
   base64_encode(cert_buf, sizeof(cert_buf),
                 (const char*)r2->cache_info.signing_key_cert->encoded,
                 r2->cache_info.signing_key_cert->encoded_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END ED25519 CERT-----\n", sizeof(buf2));
-  strlcat(buf2, "master-key-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END ED25519 CERT-----\n");
+
+  smartlist_add_strdup(chunks, "master-key-ed25519 ");
   {
     char k[ED25519_BASE64_LEN+1];
-    tt_int_op(ed25519_public_to_base64(k,
-                          &r2->cache_info.signing_key_cert->signing_key),
-              OP_GE, 0);
-    strlcat(buf2, k, sizeof(buf2));
-    strlcat(buf2, "\n", sizeof(buf2));
+    ed25519_public_to_base64(k, &r2->cache_info.signing_key_cert->signing_key);
+    smartlist_add_strdup(chunks, k);
+    smartlist_add_strdup(chunks, "\n");
   }
-  strlcat(buf2, "platform Tor "VERSION" on ", sizeof(buf2));
-  strlcat(buf2, get_uname(), sizeof(buf2));
-  strlcat(buf2, "\n"
-          "published 1970-01-01 00:00:05\n"
-          "fingerprint ", sizeof(buf2));
-  tt_assert(!crypto_pk_get_fingerprint(pk1, fingerprint, 1));
-  strlcat(buf2, fingerprint, sizeof(buf2));
-  strlcat(buf2, "\nuptime 0\n"
-          "bandwidth 3000 3000 3000\n", sizeof(buf2));
-  strlcat(buf2, "onion-key\n", sizeof(buf2));
-  strlcat(buf2, pk2_str, sizeof(buf2));
-  strlcat(buf2, "signing-key\n", sizeof(buf2));
-  strlcat(buf2, pk1_str, sizeof(buf2));
+
+  smartlist_add(chunks, get_new_platform_line());
+  smartlist_add(chunks, get_new_published_line(r2));
+  smartlist_add(chunks, get_new_fingerprint_line(r2));
+
+  smartlist_add(chunks, get_new_uptime_line(0));
+  smartlist_add(chunks, get_new_bandwidth_line(r2));
+
+  smartlist_add(chunks, get_new_onion_key_block(r2));
+  smartlist_add(chunks, get_new_signing_key_block(r2));
+
   int rsa_cc_len;
-  rsa_cc = make_tap_onion_key_crosscert(pk2,
+  rsa_cc = make_tap_onion_key_crosscert(r2_onion_pkey,
                                         &kp1.pubkey,
-                                        pk1,
+                                        r2->identity_pkey,
                                         &rsa_cc_len);
   tt_assert(rsa_cc);
   base64_encode(cert_buf, sizeof(cert_buf), (char*)rsa_cc, rsa_cc_len,
                 BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, "onion-key-crosscert\n"
-          "-----BEGIN CROSSCERT-----\n", sizeof(buf2));
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "-----END CROSSCERT-----\n", sizeof(buf2));
+  smartlist_add_strdup(chunks, "onion-key-crosscert\n"
+                       "-----BEGIN CROSSCERT-----\n");
+  smartlist_add_strdup(chunks, cert_buf);
+  smartlist_add_strdup(chunks, "-----END CROSSCERT-----\n");
   int ntor_cc_sign;
   {
     tor_cert_t *ntor_cc = NULL;
@@ -393,112 +1038,165 @@ test_dir_formats(void *arg)
                 BASE64_ENCODE_MULTILINE);
     tor_cert_free(ntor_cc);
   }
-  tor_snprintf(buf2+strlen(buf2), sizeof(buf2)-strlen(buf2),
+  smartlist_add_asprintf(chunks,
                "ntor-onion-key-crosscert %d\n"
                "-----BEGIN ED25519 CERT-----\n"
                "%s"
                "-----END ED25519 CERT-----\n", ntor_cc_sign, cert_buf);
 
-  strlcat(buf2, "hidden-service-dir\n", sizeof(buf2));
-  strlcat(buf2, "ntor-onion-key ", sizeof(buf2));
-  base64_encode(cert_buf, sizeof(cert_buf),
-                (const char*)r2_onion_keypair.pubkey.public_key, 32,
-                BASE64_ENCODE_MULTILINE);
-  strlcat(buf2, cert_buf, sizeof(buf2));
-  strlcat(buf2, "accept *:80\nreject 18.0.0.0/8:24\n", sizeof(buf2));
-  strlcat(buf2, "tunnelled-dir-server\n", sizeof(buf2));
-  strlcat(buf2, "router-sig-ed25519 ", sizeof(buf2));
+  smartlist_add_strdup(chunks, "hidden-service-dir\n");
 
-  /* Fake just enough of an ORPort to get by */
-  MOCK(get_configured_ports, mock_get_configured_ports);
-  mocked_configured_ports = smartlist_new();
+  smartlist_add(chunks, get_new_bridge_distribution_request_line(options));
+  smartlist_add(chunks, get_new_ntor_onion_key_line(&r2_onion_keypair.pubkey));
+  smartlist_add_strdup(chunks, "accept *:80\nreject 18.0.0.0/8:24\n");
+  smartlist_add_strdup(chunks, "tunnelled-dir-server\n");
 
-  memset(&orport, 0, sizeof(orport));
-  orport.type = CONN_TYPE_OR_LISTENER;
-  orport.addr.family = AF_INET;
-  orport.port = 9005;
-  smartlist_add(mocked_configured_ports, &orport);
+  smartlist_add_strdup(chunks, "router-sig-ed25519 ");
 
-  buf = router_dump_router_to_string(r2, pk1, pk2, &r2_onion_keypair, &kp2);
-  tt_assert(buf);
-  buf[strlen(buf2)] = '\0'; /* Don't compare the sig; it's never the same
+  size_t len_out = 0;
+  buf2 = smartlist_join_strings(chunks, "", 0, &len_out);
+  SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+  smartlist_free(chunks);
+
+  tt_assert(len_out > 0);
+
+  buf[strlen(buf2)] = '\0'; /* Don't compare either sig; they're never the same
                              * twice */
 
   tt_str_op(buf, OP_EQ, buf2);
   tor_free(buf);
 
-  buf = router_dump_router_to_string(r2, pk1, NULL, NULL, NULL);
+  setup_mock_configured_ports(r2->or_port, 0);
 
-  UNMOCK(get_configured_ports);
-  smartlist_free(mocked_configured_ports);
-  mocked_configured_ports = NULL;
+  buf = router_dump_router_to_string(r2, r2->identity_pkey, NULL, NULL, NULL);
+  tt_assert(buf);
 
-  /* Reset for later */
+  cleanup_mock_configured_ports();
+
+  /* Now, try to parse buf */
   cp = buf;
   rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
-  tt_assert(rp2);
-  tt_int_op(rp2->addr,OP_EQ, r2->addr);
-  tt_int_op(rp2->or_port,OP_EQ, r2->or_port);
-  tt_int_op(rp2->dir_port,OP_EQ, r2->dir_port);
-  tt_int_op(rp2->bandwidthrate,OP_EQ, r2->bandwidthrate);
-  tt_int_op(rp2->bandwidthburst,OP_EQ, r2->bandwidthburst);
-  tt_int_op(rp2->bandwidthcapacity,OP_EQ, r2->bandwidthcapacity);
+
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
   tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
              r2->onion_curve25519_pkey->public_key,
              CURVE25519_PUBKEY_LEN);
-  onion_pkey = router_get_rsa_onion_pkey(rp2->onion_pkey,
-                                         rp2->onion_pkey_len);
-  tt_int_op(crypto_pk_cmp_keys(onion_pkey, pk2), OP_EQ, 0);
-  crypto_pk_free(onion_pkey);
-  tt_int_op(crypto_pk_cmp_keys(rp2->identity_pkey, pk1), OP_EQ, 0);
-  tt_assert(rp2->supports_tunnelled_dir_requests);
-
-  tt_int_op(smartlist_len(rp2->exit_policy),OP_EQ, 2);
-
-  p = smartlist_get(rp2->exit_policy, 0);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_ACCEPT);
-  tt_assert(tor_addr_is_null(&p->addr));
-  tt_int_op(p->maskbits,OP_EQ, 0);
-  tt_int_op(p->prt_min,OP_EQ, 80);
-  tt_int_op(p->prt_max,OP_EQ, 80);
-
-  p = smartlist_get(rp2->exit_policy, 1);
-  tt_int_op(p->policy_type,OP_EQ, ADDR_POLICY_REJECT);
-  tt_assert(tor_addr_eq(&p->addr, &ex2->addr));
-  tt_int_op(p->maskbits,OP_EQ, 8);
-  tt_int_op(p->prt_min,OP_EQ, 24);
-  tt_int_op(p->prt_max,OP_EQ, 24);
-
-#if 0
-  /* Okay, now for the directories. */
-  {
-    fingerprint_list = smartlist_new();
-    crypto_pk_get_fingerprint(pk2, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
-    crypto_pk_get_fingerprint(pk1, buf, 1);
-    add_fingerprint_to_dir(buf, fingerprint_list, 0);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  tor_free(buf);
+  routerinfo_free(rp2);
+
+  /* Test extrainfo creation. */
+
+  /* Set up standard mocks and data */
+  setup_mocks_for_fresh_descriptor(r2, r2_onion_pkey);
+
+  /* router_build_fresh_descriptor() requires
+   * router_build_fresh_unsigned_routerinfo(), but the implementation is
+   * too complex. Instead, we re-use r2.
+   */
+  mocked_routerinfo = r2;
+  MOCK(router_build_fresh_unsigned_routerinfo,
+       mock_router_build_fresh_unsigned_routerinfo);
+
+  /* r2 uses ed25519, so we need to mock the ed key functions */
+  mocked_master_signing_key = &kp2;
+  MOCK(get_master_signing_keypair, mock_get_master_signing_keypair);
+
+  mocked_signing_key_cert = r2->cache_info.signing_key_cert;
+  MOCK(get_master_signing_key_cert, mock_get_master_signing_key_cert);
+
+  mocked_curve25519_onion_key = &r2_onion_keypair;
+  MOCK(get_current_curve25519_keypair, mock_get_current_curve25519_keypair);
+
+  /* Fake just enough of an ORPort to get by */
+  setup_mock_configured_ports(r2->or_port, 0);
+
+  /* Test the high-level interface. */
+  rv = router_build_fresh_descriptor(&r2_out, &e2);
+  if (rv < 0) {
+    /* router_build_fresh_descriptor() frees r2 on failure. */
+    r2 = NULL;
+    /* Get rid of an alias to rp2 */
+    r2_out = NULL;
   }
+  tt_assert(rv == 0);
+  tt_assert(r2_out);
+  tt_assert(e2);
+  /* Guaranteed by mock_router_build_fresh_unsigned_routerinfo() */
+  tt_ptr_op(r2_out, OP_EQ, r2);
+  /* Get rid of an alias to r2 */
+  r2_out = NULL;
+
+  /* Now cleanup */
+  cleanup_mocks_for_fresh_descriptor();
+
+  mocked_routerinfo = NULL;
+  UNMOCK(router_build_fresh_unsigned_routerinfo);
+  mocked_master_signing_key = NULL;
+  UNMOCK(get_master_signing_keypair);
+  mocked_signing_key_cert = NULL;
+  UNMOCK(get_master_signing_key_cert);
+  mocked_curve25519_onion_key = NULL;
+  UNMOCK(get_current_curve25519_keypair);
+
+  cleanup_mock_configured_ports();
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, e2);
+
+  /* Test that the signed ri is parseable */
+  tt_assert(r2->cache_info.signed_descriptor_body);
+  cp = r2->cache_info.signed_descriptor_body;
+  rp2 = router_parse_entry_from_string((const char*)cp,NULL,1,0,NULL,NULL);
 
-#endif /* 0 */
-  dirserv_free_fingerprint_list();
+  CHECK_ROUTERINFO_CONSISTENCY(r2, rp2);
+
+  tt_mem_op(rp2->onion_curve25519_pkey->public_key,OP_EQ,
+            r2->onion_curve25519_pkey->public_key,
+            CURVE25519_PUBKEY_LEN);
+
+  CHECK_PARSED_EXIT_POLICY(rp2);
+
+  routerinfo_free(rp2);
+
+  /* Test that the signed ei is parseable */
+  tt_assert(e2->cache_info.signed_descriptor_body);
+  cp = e2->cache_info.signed_descriptor_body;
+  ep2 = extrainfo_parse_entry_from_string((const char*)cp,NULL,1,NULL,NULL);
+
+  CHECK_EXTRAINFO_CONSISTENCY(r2, ep2);
+
+  /* In future tests, we could check the actual extrainfo statistics. */
+
+  extrainfo_free(ep2);
 
  done:
-  if (r1)
-    routerinfo_free(r1);
-  if (r2)
-    routerinfo_free(r2);
-  if (rp2)
-    routerinfo_free(rp2);
+  dirserv_free_fingerprint_list();
+
+  tor_free(options->Nickname);
+
+  cleanup_mock_configured_ports();
+  cleanup_mocks_for_fresh_descriptor();
+
+  if (chunks) {
+    SMARTLIST_FOREACH(chunks, char *, s, tor_free(s));
+    smartlist_free(chunks);
+  }
+
+  routerinfo_free(r2);
+  routerinfo_free(r2_out);
+  routerinfo_free(rp2);
+
+  extrainfo_free(e2);
+  extrainfo_free(ep2);
 
   tor_free(rsa_cc);
+  crypto_pk_free(r2_onion_pkey);
+
   tor_free(buf);
-  tor_free(pk1_str);
-  tor_free(pk2_str);
-  if (pk1) crypto_pk_free(pk1);
-  if (pk2) crypto_pk_free(pk2);
-  if (rp1) routerinfo_free(rp1);
-  tor_free(dir1); /* XXXX And more !*/
-  tor_free(dir2); /* And more !*/
+  tor_free(buf2);
 }
 
 #include "failing_routerdescs.inc"
@@ -1767,7 +2465,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, "", 0);
   setup_capture_of_logs(LOG_WARN);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
   expect_log_msg("Empty bandwidth file\n");
   teardown_capture_of_logs();
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1783,7 +2482,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(-1, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                        bw_file_headers));
+                                                        bw_file_headers,
+                                                        NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op("", OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1794,7 +2495,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, header_lines_v100, 0);
   bw_file_headers = smartlist_new();
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1807,7 +2510,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1818,7 +2523,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   tor_asprintf(&content, "%s%s", header_lines_v100, relay_lines_v100);
   write_str_to_file(fname, content, 0);
   tor_free(content);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
 
   /* Test bandwidth file including v1.1.0 bandwidth headers and
    * v1.0.0 relay lines. bw_file_headers will contain the v1.1.0 headers. */
@@ -1828,7 +2534,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1844,7 +2552,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1861,7 +2571,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v100, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1872,7 +2584,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110_no_terminator, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1883,7 +2597,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   bw_file_headers = smartlist_new();
   write_str_to_file(fname, header_lines_v110, 0);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1898,7 +2614,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1913,7 +2631,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1929,7 +2649,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_v110, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1946,7 +2668,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
   tt_str_op(bw_file_headers_str_bad, OP_EQ, bw_file_headers_str);
   SMARTLIST_FOREACH(bw_file_headers, char *, c, tor_free(c));
@@ -1964,7 +2688,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   bw_file_headers_str = smartlist_join_strings(bw_file_headers, " ", 0, NULL);
@@ -1985,7 +2711,9 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
   write_str_to_file(fname, content, 0);
   tor_free(content);
   tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL,
-                                                       bw_file_headers));
+                                                       bw_file_headers,
+                                                       NULL));
+
   tt_int_op(MAX_BW_FILE_HEADER_COUNT_IN_VOTE, OP_EQ,
             smartlist_len(bw_file_headers));
   /* force bw_file_headers to be bigger than
@@ -2014,7 +2742,8 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
@@ -2032,11 +2761,13 @@ test_dir_dirserv_read_measured_bandwidths(void *arg)
 
   /* Read the bandwidth file */
   setup_full_capture_of_logs(LOG_DEBUG);
-  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL));
+  tt_int_op(0, OP_EQ, dirserv_read_measured_bandwidths(fname, NULL, NULL,
+                                                       NULL));
   expect_log_msg_not_containing("Ignoring bandwidth file line");
   teardown_capture_of_logs();
 
  done:
+  unlink(fname);
   tor_free(fname);
   tor_free(header_lines_v100);
   tor_free(header_lines_v110_no_terminator);
@@ -2274,6 +3005,7 @@ test_dir_param_voting_lookup(void *arg)
   tt_int_op(99, OP_EQ,
             dirvote_get_intermediate_param_value(lst, "abcd", 1000));
 
+#ifndef ALL_BUGS_ARE_FATAL
   /* moomin appears twice. That's a bug. */
   tor_capture_bugs_(1);
   tt_int_op(-100, OP_EQ,
@@ -2291,7 +3023,7 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "jack", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(! ok)");
+            "!(!ok)");
   tor_end_capture_bugs_();
   /* electricity and opa aren't integers. */
   tor_capture_bugs_(1);
@@ -2299,7 +3031,7 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "electricity", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(! ok)");
+            "!(!ok)");
   tor_end_capture_bugs_();
 
   tor_capture_bugs_(1);
@@ -2307,8 +3039,9 @@ test_dir_param_voting_lookup(void *arg)
             dirvote_get_intermediate_param_value(lst, "opa", -100));
   tt_int_op(smartlist_len(tor_get_captured_bug_log_()), OP_EQ, 1);
   tt_str_op(smartlist_get(tor_get_captured_bug_log_(), 0), OP_EQ,
-            "!(! ok)");
+            "!(!ok)");
   tor_end_capture_bugs_();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
 
  done:
   SMARTLIST_FOREACH(lst, char *, cp, tor_free(cp));
@@ -2863,17 +3596,23 @@ test_a_networkstatus(
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   /* Parse certificates and keys. */
-  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert1 = mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
   tt_assert(cert1);
-  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                           strlen(AUTHORITY_CERT_2),
+                                           NULL);
   tt_assert(cert2);
-  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                           strlen(AUTHORITY_CERT_3),
+                                           NULL);
   tt_assert(cert3);
   sign_skey_1 = crypto_pk_new();
   sign_skey_2 = crypto_pk_new();
   sign_skey_3 = crypto_pk_new();
   sign_skey_leg1 = pk_generate(4);
-  voting_schedule_recalculate_timing(get_options(), now);
+  dirauth_sched_recalculate_timing(get_options(), now);
   sr_state_init(0, 0);
 
   tt_assert(!crypto_pk_read_private_key_from_string(sign_skey_1,
@@ -2969,7 +3708,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_NS);
   tt_assert(consensus_text);
-  con = networkstatus_parse_vote_from_string(consensus_text, NULL,
+  con = networkstatus_parse_vote_from_string_(consensus_text, NULL,
                                              NS_TYPE_CONSENSUS);
   tt_assert(con);
   //log_notice(LD_GENERAL, "<<%s>>\n<<%s>>\n<<%s>>\n",
@@ -2981,7 +3720,7 @@ test_a_networkstatus(
                                                    sign_skey_leg1,
                                                    FLAV_MICRODESC);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string_(consensus_text_md, NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -3080,13 +3819,13 @@ test_a_networkstatus(
     tt_assert(consensus_text3);
     tt_assert(consensus_text_md2);
     tt_assert(consensus_text_md3);
-    con2 = networkstatus_parse_vote_from_string(consensus_text2, NULL,
+    con2 = networkstatus_parse_vote_from_string_(consensus_text2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con3 = networkstatus_parse_vote_from_string(consensus_text3, NULL,
+    con3 = networkstatus_parse_vote_from_string_(consensus_text3, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md2 = networkstatus_parse_vote_from_string(consensus_text_md2, NULL,
+    con_md2 = networkstatus_parse_vote_from_string_(consensus_text_md2, NULL,
                                                 NS_TYPE_CONSENSUS);
-    con_md3 = networkstatus_parse_vote_from_string(consensus_text_md3, NULL,
+    con_md3 = networkstatus_parse_vote_from_string_(consensus_text_md3, NULL,
                                                 NS_TYPE_CONSENSUS);
     tt_assert(con2);
     tt_assert(con3);
@@ -3864,6 +4603,62 @@ mock_get_options(void)
   return mock_options;
 }
 
+/**
+ * Test dirauth_get_b64_digest_bw_file.
+ * This function should be near the other bwauth functions, but it needs
+ * mock_get_options, that is only defined here.
+ */
+
+static void
+test_dir_bwauth_bw_file_digest256(void *arg)
+{
+  (void)arg;
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  char *fname = tor_strdup(get_fname("V3BandwidthsFile"));
+  /* Initialize to a wrong digest. */
+  uint8_t digest[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+
+  /* Digest of an empty string. Initialize to a wrong digest. */
+  char digest_empty_str[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_empty_str, "", 0, DIGEST_SHA256);
+
+  /* Digest of the content. Initialize to a wrong digest. */
+  char digest_expected[DIGEST256_LEN] = "01234567890123456789abcdefghijkl";
+  crypto_digest256(digest_expected, content, strlen(content), DIGEST_SHA256);
+
+  /* When the bandwidth file can not be found. */
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_empty_str, DIGEST256_LEN);
+
+  /* When there is a timestamp but it is too old. */
+  write_str_to_file(fname, content, 0);
+  tt_int_op(-1, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  /* The digest will be correct. */
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+  update_approx_time(1541171221);
+
+  /* When there is a bandwidth file and it can be read. */
+  tt_int_op(0, OP_EQ,
+            dirserv_read_measured_bandwidths(fname,
+                                             NULL, NULL, digest));
+  tt_mem_op(digest, OP_EQ, digest_expected, DIGEST256_LEN);
+
+ done:
+  unlink(fname);
+  tor_free(fname);
+  update_approx_time(time(NULL));
+}
+
 static void
 reset_routerstatus(routerstatus_t *rs,
                    const char *hex_identity_digest,
@@ -3900,10 +4695,13 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   (void)arg;
 
   /* Init options */
+  dirauth_options_t *dirauth_options =
+    tor_malloc_zero(sizeof(dirauth_options_t));
+
   mock_options = tor_malloc(sizeof(or_options_t));
   reset_options(mock_options, &mock_get_options_calls);
-
   MOCK(get_options, mock_get_options);
+  dirauth_set_options(dirauth_options);
 
   /* Init routersets */
   routerset_t *routerset_all  = routerset_new();
@@ -3943,16 +4741,15 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check that "*" sets flags on all routers: Exit
    * Check the flags aren't being confused with each other */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_all;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_all;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_exit, OP_EQ, 1);
   tt_uint_op(rs_b->is_exit, OP_EQ, 1);
@@ -3965,18 +4762,17 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check that "*" sets flags on all routers: Guard & HSDir
    * Cover the remaining flags in one test */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteGuard = routerset_all;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_all;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_all;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_all;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_possible_guard, OP_EQ, 1);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
@@ -3989,20 +4785,19 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check routerset A sets all flags on router A,
    * but leaves router B unmodified */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_a;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
-  mock_options->TestingDirAuthVoteGuard = routerset_a;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_a;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_a;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_a;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_a;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_exit, OP_EQ, 1);
   tt_uint_op(rs_b->is_exit, OP_EQ, 0);
@@ -4013,21 +4808,21 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
 
   /* Check routerset A unsets all flags on router B when Strict is set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_a;
-  mock_options->TestingDirAuthVoteExitIsStrict = 1;
-  mock_options->TestingDirAuthVoteGuard = routerset_a;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 1;
-  mock_options->TestingDirAuthVoteHSDir = routerset_a;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteExit = routerset_a;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_a;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_a;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 1;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
   tt_uint_op(rs_b->is_exit, OP_EQ, 0);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
@@ -4035,21 +4830,21 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
 
   /* Check routerset A doesn't modify flags on router B without Strict set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_a;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
-  mock_options->TestingDirAuthVoteGuard = routerset_a;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_a;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_a;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_a;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_a;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
   tt_uint_op(rs_b->is_exit, OP_EQ, 1);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 1);
@@ -4058,21 +4853,21 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check the empty routerset zeroes all flags
    * on routers A & B with Strict set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_none;
-  mock_options->TestingDirAuthVoteExitIsStrict = 1;
-  mock_options->TestingDirAuthVoteGuard = routerset_none;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 1;
-  mock_options->TestingDirAuthVoteHSDir = routerset_none;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteExit = routerset_none;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_none;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 1;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_none;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 1;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
 
   tt_uint_op(rs_b->is_exit, OP_EQ, 0);
   tt_uint_op(rs_b->is_possible_guard, OP_EQ, 0);
@@ -4081,24 +4876,23 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
   /* Check the empty routerset doesn't modify any flags
    * on A or B without Strict set */
   reset_options(mock_options, &mock_get_options_calls);
+  memset(dirauth_options, 0, sizeof(*dirauth_options));
   reset_routerstatus(rs_a, ROUTER_A_ID_STR, ROUTER_A_IPV4);
   reset_routerstatus(rs_b, ROUTER_B_ID_STR, ROUTER_B_IPV4);
 
-  mock_options->TestingDirAuthVoteExit = routerset_none;
-  mock_options->TestingDirAuthVoteExitIsStrict = 0;
-  mock_options->TestingDirAuthVoteGuard = routerset_none;
-  mock_options->TestingDirAuthVoteGuardIsStrict = 0;
-  mock_options->TestingDirAuthVoteHSDir = routerset_none;
-  mock_options->TestingDirAuthVoteHSDirIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteExit = routerset_none;
+  dirauth_options->TestingDirAuthVoteExitIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteGuard = routerset_none;
+  dirauth_options->TestingDirAuthVoteGuardIsStrict = 0;
+  dirauth_options->TestingDirAuthVoteHSDir = routerset_none;
+  dirauth_options->TestingDirAuthVoteHSDirIsStrict = 0;
 
   rs_b->is_exit = 1;
   rs_b->is_possible_guard = 1;
   rs_b->is_hs_dir = 1;
 
   dirserv_set_routerstatus_testing(rs_a);
-  tt_int_op(mock_get_options_calls, OP_EQ, 1);
   dirserv_set_routerstatus_testing(rs_b);
-  tt_int_op(mock_get_options_calls, OP_EQ, 2);
 
   tt_uint_op(rs_a->is_exit, OP_EQ, 0);
   tt_uint_op(rs_a->is_possible_guard, OP_EQ, 0);
@@ -4109,6 +4903,7 @@ test_dir_dirserv_set_routerstatus_testing(void *arg)
 
  done:
   tor_free(mock_options);
+  tor_free(dirauth_options);
   mock_options = NULL;
 
   UNMOCK(get_options);
@@ -4196,6 +4991,14 @@ test_dir_purpose_needs_anonymity_returns_true_by_default(void *arg)
 {
   (void)arg;
 
+#ifdef ALL_BUGS_ARE_FATAL
+  /* Coverity (and maybe clang analyser) complain that the code following
+   * tt_skip() is unconditionally unreachable. */
+#if !defined(__COVERITY__) && !defined(__clang_analyzer__)
+  tt_skip();
+#endif
+#endif /* defined(ALL_BUGS_ARE_FATAL) */
+
   tor_capture_bugs_(1);
   setup_full_capture_of_logs(LOG_WARN);
   tt_int_op(1, OP_EQ, purpose_needs_anonymity(0, 0, NULL));
@@ -4683,15 +5486,15 @@ test_dir_conn_purpose_to_string(void *data)
   teardown_capture_of_logs();
 }
 
-NS_DECL(int,
-public_server_mode, (const or_options_t *options));
+static int dir_tests_public_server_mode(const or_options_t *options);
+ATTR_UNUSED static int dir_tests_public_server_mode_called = 0;
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+dir_tests_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
-  if (CALLED(public_server_mode)++ == 0) {
+  if (dir_tests_public_server_mode_called++ == 0) {
     return 1;
   }
 
@@ -4705,13 +5508,14 @@ test_dir_should_use_directory_guards(void *data)
   char *errmsg = NULL;
   (void)data;
 
-  NS_MOCK(public_server_mode);
+  MOCK(public_server_mode,
+       dir_tests_public_server_mode);
 
   options = options_new();
   options_init(options);
 
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 1);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 1);
 
   options->UseEntryGuards = 1;
   options->DownloadExtraInfo = 0;
@@ -4719,41 +5523,41 @@ test_dir_should_use_directory_guards(void *data)
   options->FetchDirInfoExtraEarly = 0;
   options->FetchUselessDescriptors = 0;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 1);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 2);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 2);
 
   options->UseEntryGuards = 0;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 3);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 3);
   options->UseEntryGuards = 1;
 
   options->DownloadExtraInfo = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 4);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 4);
   options->DownloadExtraInfo = 0;
 
   options->FetchDirInfoEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 5);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 5);
   options->FetchDirInfoEarly = 0;
 
   options->FetchDirInfoExtraEarly = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 6);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 6);
   options->FetchDirInfoExtraEarly = 0;
 
   options->FetchUselessDescriptors = 1;
   tt_int_op(should_use_directory_guards(options), OP_EQ, 0);
-  tt_int_op(CALLED(public_server_mode), OP_EQ, 7);
+  tt_int_op(dir_tests_public_server_mode_called, OP_EQ, 7);
   options->FetchUselessDescriptors = 0;
 
   done:
-    NS_UNMOCK(public_server_mode);
+    UNMOCK(public_server_mode);
     or_options_free(options);
     tor_free(errmsg);
 }
 
-NS_DECL(void,
-directory_initiate_request, (directory_request_t *req));
+static void dir_tests_directory_initiate_request(directory_request_t *req);
+ATTR_UNUSED static int dir_tests_directory_initiate_request_called = 0;
 
 static void
 test_dir_should_not_init_request_to_ourselves(void *data)
@@ -4763,7 +5567,8 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   crypto_pk_t *key = pk_generate(2);
   (void) data;
 
-  NS_MOCK(directory_initiate_request);
+  MOCK(directory_initiate_request,
+       dir_tests_directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4778,15 +5583,15 @@ test_dir_should_not_init_request_to_ourselves(void *data)
   dir_server_add(ourself);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
 
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_request);
+    UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
     crypto_pk_free(key);
@@ -4800,7 +5605,8 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
                                 | MICRODESC_DIRINFO;
   (void) data;
 
-  NS_MOCK(directory_initiate_request);
+  MOCK(directory_initiate_request,
+       dir_tests_directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4811,14 +5617,14 @@ test_dir_should_not_init_request_to_dir_auths_without_v3_info(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 0);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(directory_initiate_request);
+    UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
@@ -4829,7 +5635,8 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_t *ds = NULL;
   (void) data;
 
-  NS_MOCK(directory_initiate_request);
+  MOCK(directory_initiate_request,
+       dir_tests_directory_initiate_request);
 
   clear_dir_servers();
   routerlist_free_all();
@@ -4840,23 +5647,23 @@ test_dir_should_init_request_to_dir_auths(void *data)
   dir_server_add(ds);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_STATUS_VOTE, 0, NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 1);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 1);
 
   directory_get_from_all_authorities(DIR_PURPOSE_FETCH_DETACHED_SIGNATURES, 0,
                                      NULL);
-  tt_int_op(CALLED(directory_initiate_request), OP_EQ, 2);
+  tt_int_op(dir_tests_directory_initiate_request_called, OP_EQ, 2);
 
   done:
-    NS_UNMOCK(directory_initiate_request);
+    UNMOCK(directory_initiate_request);
     clear_dir_servers();
     routerlist_free_all();
 }
 
 void
-NS(directory_initiate_request)(directory_request_t *req)
+dir_tests_directory_initiate_request(directory_request_t *req)
 {
   (void)req;
-  CALLED(directory_initiate_request)++;
+  dir_tests_directory_initiate_request_called++;
 }
 
 static void
@@ -6087,6 +6894,80 @@ test_dir_find_dl_min_delay(void* data)
 }
 
 static void
+test_dir_matching_flags(void *arg)
+{
+  (void) arg;
+  routerstatus_t *rs_noflags = NULL;
+  routerstatus_t *rs = NULL;
+  char *s = NULL;
+
+  smartlist_t *tokens = smartlist_new();
+  memarea_t *area = memarea_new();
+
+  int expected_val_when_unused = 0;
+
+  const char *ex_noflags =
+    "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 "
+       "192.168.0.1 9001 0\n"
+    "m thisoneislongerbecauseitisa256bitmddigest33\n"
+    "s\n";
+  const char *cp = ex_noflags;
+  rs_noflags = routerstatus_parse_entry_from_string(
+         area, &cp,
+         cp + strlen(cp),
+         tokens, NULL, NULL,
+         MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);
+  tt_assert(rs_noflags);
+
+#define FLAG(string, field) STMT_BEGIN {        \
+    tor_asprintf(&s,\
+                 "r example hereiswhereyouridentitygoes 2015-08-30 12:00:00 " \
+                 "192.168.0.1 9001 0\n"                                 \
+                 "m thisoneislongerbecauseitisa256bitmddigest33\n"      \
+                 "s %s\n", string);                                     \
+    cp = s;                                                             \
+    rs =  routerstatus_parse_entry_from_string(                         \
+      area, &cp,                                                        \
+      cp + strlen(cp),                                                  \
+      tokens, NULL, NULL,                                               \
+      MAX_SUPPORTED_CONSENSUS_METHOD, FLAV_MICRODESC);                  \
+    /* the field should usually be 0 when no flags are listed */        \
+    tt_int_op(rs_noflags->field, OP_EQ, expected_val_when_unused);      \
+    /* the field should be 1 when this flags islisted */                \
+    tt_int_op(rs->field, OP_EQ, 1);                                     \
+    tor_free(s);                                                        \
+    routerstatus_free(rs);                                              \
+} STMT_END
+
+  FLAG("Authority", is_authority);
+  FLAG("BadExit", is_bad_exit);
+  FLAG("Exit", is_exit);
+  FLAG("Fast", is_fast);
+  FLAG("Guard", is_possible_guard);
+  FLAG("HSDir", is_hs_dir);
+  FLAG("Stable", is_stable);
+  FLAG("StaleDesc", is_staledesc);
+  FLAG("V2Dir", is_v2_dir);
+
+  // These flags are assumed to be set whether they're declared or not.
+  expected_val_when_unused = 1;
+  FLAG("Running", is_flagged_running);
+  FLAG("Valid", is_valid);
+  expected_val_when_unused = 0;
+
+  // These flags are no longer used, but still parsed.
+  FLAG("Named", is_named);
+  FLAG("Unnamed", is_unnamed);
+
+ done:
+  tor_free(s);
+  routerstatus_free(rs);
+  routerstatus_free(rs_noflags);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+}
+
+static void
 test_dir_assumed_flags(void *arg)
 {
   (void)arg;
@@ -6101,9 +6982,10 @@ test_dir_assumed_flags(void *arg)
        "192.168.0.1 9001 0\n"
     "m thisoneislongerbecauseitisa256bitmddigest33\n"
     "s Fast Guard Stable\n";
+  const char *eos = str1 + strlen(str1);
 
   const char *cp = str1;
-  rs = routerstatus_parse_entry_from_string(area, &cp, tokens, NULL, NULL,
+  rs = routerstatus_parse_entry_from_string(area, &cp, eos, tokens, NULL, NULL,
                                             24, FLAV_MICRODESC);
   tt_assert(rs);
   tt_assert(rs->is_flagged_running);
@@ -6201,98 +7083,6 @@ test_dir_platform_str(void *arg)
   ;
 }
 
-static networkstatus_t *mock_networkstatus;
-
-static networkstatus_t *
-mock_networkstatus_get_latest_consensus_by_flavor(consensus_flavor_t f)
-{
-  (void)f;
-  return mock_networkstatus;
-}
-
-static void
-test_dir_networkstatus_consensus_has_ipv6(void *arg)
-{
-  (void)arg;
-
-  int has_ipv6 = 0;
-
-  /* Init options and networkstatus */
-  or_options_t our_options;
-  mock_options = &our_options;
-  reset_options(mock_options, &mock_get_options_calls);
-  MOCK(get_options, mock_get_options);
-
-  networkstatus_t our_networkstatus;
-  mock_networkstatus = &our_networkstatus;
-  memset(mock_networkstatus, 0, sizeof(*mock_networkstatus));
-  MOCK(networkstatus_get_latest_consensus_by_flavor,
-       mock_networkstatus_get_latest_consensus_by_flavor);
-
-  /* A live consensus */
-  mock_networkstatus->valid_after = time(NULL) - 3600;
-  mock_networkstatus->valid_until = time(NULL) + 3600;
-
-  /* Test the bounds for A lines in the NS consensus */
-  mock_options->UseMicrodescriptors = 0;
-
-  mock_networkstatus->consensus_method = MIN_SUPPORTED_CONSENSUS_METHOD;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  /* Test the bounds for A lines in the microdesc consensus */
-  mock_options->UseMicrodescriptors = 1;
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method = MAX_SUPPORTED_CONSENSUS_METHOD + 20;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 1;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS + 20;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS - 1;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(!has_ipv6);
-
-  /* Test the edge cases */
-  mock_options->UseMicrodescriptors = 1;
-  mock_networkstatus->consensus_method =
-      MIN_METHOD_FOR_A_LINES_IN_MICRODESC_CONSENSUS;
-
-  /* Reasonably live */
-  mock_networkstatus->valid_until = approx_time() - 60;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(has_ipv6);
-
-  /* Not reasonably live */
-  mock_networkstatus->valid_after = approx_time() - 24*60*60 - 3600;
-  mock_networkstatus->valid_until = approx_time() - 24*60*60 - 60;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(!has_ipv6);
-
-  /* NULL consensus */
-  mock_networkstatus = NULL;
-  has_ipv6 = networkstatus_consensus_has_ipv6(get_options());
-  tt_assert(!has_ipv6);
-
- done:
-  UNMOCK(get_options);
-  UNMOCK(networkstatus_get_latest_consensus_by_flavor);
-}
-
 static void
 test_dir_format_versions_list(void *arg)
 {
@@ -6344,6 +7134,301 @@ test_dir_format_versions_list(void *arg)
   teardown_capture_of_logs();
 }
 
+static void
+test_dir_add_fingerprint(void *arg)
+{
+  (void)arg;
+  authdir_config_t *list;
+  int ret;
+  ed25519_secret_key_t seckey;
+  ed25519_public_key_t pubkey_good, pubkey_bad;
+
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+
+  setup_capture_of_logs(LOG_WARN);
+
+  /* RSA test - successful */
+  ret = add_rsa_fingerprint_to_dir("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
+                                   list, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* RSA test - failure */
+  ret = add_rsa_fingerprint_to_dir("ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ",
+                                   list, 0);
+  tt_int_op(ret, OP_EQ, -1);
+
+  /* ed25519 test - successful */
+  ed25519_secret_key_generate(&seckey, 0);
+  ed25519_public_key_generate(&pubkey_good, &seckey);
+
+  ret = add_ed25519_to_dir(&pubkey_good, list, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* ed25519 test - failure */
+  digest256_from_base64((char *) pubkey_bad.pubkey, "gibberish");
+
+  ret = add_ed25519_to_dir(&pubkey_bad, list, 0);
+  tt_int_op(ret, OP_EQ, -1);
+
+ done:
+  teardown_capture_of_logs();
+  dirserv_free_fingerprint_list();
+}
+
+static void
+test_dir_dirserv_load_fingerprint_file(void *arg)
+{
+  (void)arg;
+  char *fname = tor_strdup(get_fname("approved-routers"));
+
+  // Neither RSA nor ed25519
+  const char *router_lines_invalid =
+    "!badexit notafingerprint";
+  const char *router_lines_too_long =
+    "!badexit thisisareallylongstringthatislongerthanafingerprint\n";
+  const char *router_lines_bad_fmt_str =
+    "!badexit ABCDEFGH|%1$p|%2$p|%3$p|%4$p|%5$p|%6$p\n";
+  const char *router_lines_valid_rsa =
+    "!badexit AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\n";
+  const char *router_lines_invalid_rsa =
+    "!badexit ZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZZ\n";
+  const char *router_lines_valid_ed25519 =
+    "!badexit wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+  const char *router_lines_invalid_ed25519 =
+    "!badexit --fLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx--\n";
+
+  // Test: Invalid Fingerprint (not RSA or ed25519)
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_invalid, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Very long string (longer than RSA or ed25519 key)
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_too_long, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Formt string exploit
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_bad_fmt_str, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Valid RSA
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_valid_rsa, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  teardown_capture_of_logs();
+
+  // Test: Invalid RSA
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_invalid_rsa, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+  // Test: Valid ed25519
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_valid_ed25519, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  teardown_capture_of_logs();
+
+  // Test: Invalid ed25519
+  setup_capture_of_logs(LOG_NOTICE);
+  write_str_to_file(fname, router_lines_invalid_ed25519, 0);
+  tt_int_op(dirserv_load_fingerprint_file(), OP_EQ, 0);
+  expect_log_msg_containing("Invalid fingerprint");
+  teardown_capture_of_logs();
+
+ done:
+  tor_free(fname);
+  dirserv_free_fingerprint_list();
+}
+
+#define RESET_FP_LIST(list) STMT_BEGIN \
+    dirserv_free_fingerprint_list(); \
+    authdir_init_fingerprint_list(); \
+    list = authdir_return_fingerprint_list(); \
+  STMT_END
+
+static void
+test_dir_dirserv_router_get_status(void *arg)
+{
+  authdir_config_t *list;
+  routerinfo_t *ri = NULL;
+  ed25519_keypair_t kp1, kp2;
+  char d[DIGEST_LEN];
+  char fp[HEX_DIGEST_LEN+1];
+  int ret;
+  const char *msg;
+  time_t now = time(NULL);
+
+  (void)arg;
+
+  crypto_pk_t *pk = pk_generate(0);
+
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+
+  /* Set up the routerinfo */
+  ri = tor_malloc_zero(sizeof(routerinfo_t));
+  ri->addr = 0xc0a80001u;
+  ri->or_port = 9001;
+  ri->platform = tor_strdup("0.4.0.1-alpha");
+  ri->nickname = tor_strdup("Jessica");
+  ri->identity_pkey = crypto_pk_dup_key(pk);
+
+  curve25519_keypair_t ri_onion_keypair;
+  curve25519_keypair_generate(&ri_onion_keypair, 0);
+  ri->onion_curve25519_pkey = tor_memdup(&ri_onion_keypair.pubkey,
+                                         sizeof(curve25519_public_key_t));
+
+  ed25519_secret_key_from_seed(&kp1.seckey,
+                          (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
+  ed25519_public_key_generate(&kp1.pubkey, &kp1.seckey);
+  ed25519_secret_key_from_seed(&kp2.seckey,
+                          (const uint8_t*)"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX");
+  ed25519_public_key_generate(&kp2.pubkey, &kp2.seckey);
+  ri->cache_info.signing_key_cert = tor_cert_create(&kp1,
+                                         CERT_TYPE_ID_SIGNING,
+                                         &kp2.pubkey,
+                                         now, 86400,
+                                         CERT_FLAG_INCLUDE_SIGNING_KEY);
+
+  crypto_pk_get_digest(ri->identity_pkey, d);
+  base16_encode(fp, HEX_DIGEST_LEN + 1, d, DIGEST_LEN);
+
+  /* Try on an empty fingerprint list */
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  /* Try an accepted router */
+  add_rsa_fingerprint_to_dir(fp, list, 0);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp1.pubkey, list, 0);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, 0);
+  RESET_FP_LIST(list);
+
+  /* Try a rejected router */
+  add_rsa_fingerprint_to_dir(fp, list, RTR_REJECT);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, RTR_REJECT);
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp1.pubkey, list, RTR_REJECT);
+  ret = dirserv_router_get_status(ri, &msg, LOG_INFO);
+  tt_int_op(ret, OP_EQ, RTR_REJECT);
+  RESET_FP_LIST(list);
+
+ done:
+  dirserv_free_fingerprint_list();
+  routerinfo_free(ri);
+  crypto_pk_free(pk);
+}
+
+static void
+test_dir_dirserv_would_reject_router(void *arg)
+{
+  authdir_config_t *list;
+  routerstatus_t rs;
+  vote_routerstatus_t vrs;
+  ed25519_keypair_t kp;
+  char fp[HEX_DIGEST_LEN+1];
+
+  (void)arg;
+
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+
+  /* Set up the routerstatus */
+  memset(&rs, 0, sizeof(rs));
+  rs.addr = 0xc0a80001u;
+  rs.or_port = 9001;
+  strlcpy(rs.nickname, "Nicole", sizeof(rs.nickname));
+  memcpy(rs.identity_digest, "Cloud nine is great ", DIGEST_LEN);
+
+  ed25519_secret_key_from_seed(&kp.seckey,
+                          (const uint8_t*)"YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY");
+  ed25519_public_key_generate(&kp.pubkey, &kp.seckey);
+
+  base16_encode(fp, HEX_DIGEST_LEN + 1, rs.identity_digest, DIGEST_LEN);
+
+  /* Setup the vote_routerstatus_t. */
+  memcpy(vrs.ed25519_id, &kp.pubkey, ED25519_PUBKEY_LEN);
+
+  /* Try an empty fingerprint list */
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  /* Try an accepted router */
+  add_rsa_fingerprint_to_dir(fp, list, 0);
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp.pubkey, list, 0);
+  tt_assert(!dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  /* Try a rejected router */
+  add_rsa_fingerprint_to_dir(fp, list, RTR_REJECT);
+  tt_assert(dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+  add_ed25519_to_dir(&kp.pubkey, list, RTR_REJECT);
+  tt_assert(dirserv_would_reject_router(&rs, &vrs));
+  RESET_FP_LIST(list);
+
+ done:
+  dirserv_free_fingerprint_list();
+}
+
+static void
+test_dir_dirserv_add_own_fingerprint(void *arg)
+{
+  authdir_config_t *list;
+  char digest[DIGEST_LEN];
+  crypto_pk_t *pk = pk_generate(0);
+
+  (void)arg;
+
+  init_mock_ed_keys(pk);
+  authdir_init_fingerprint_list();
+  list = authdir_return_fingerprint_list();
+  dirserv_add_own_fingerprint(pk, get_master_identity_key());
+
+  /* Check if we have a RSA key. */
+  crypto_pk_get_digest(pk, digest);
+  tt_assert(digestmap_get(list->status_by_digest, digest));
+
+  /* Check if we have a ed25519 key. */
+  tt_assert(digest256map_get(list->status_by_digest256,
+                             get_master_identity_key()->pubkey));
+
+  RESET_FP_LIST(list);
+
+ done:
+  dirserv_free_fingerprint_list();
+  crypto_pk_free(pk);
+}
+
+#ifndef COCCI
 #define DIR_LEGACY(name)                             \
   { #name, test_dir_ ## name , TT_FORK, NULL, NULL }
 
@@ -6353,10 +7438,26 @@ test_dir_format_versions_list(void *arg)
 /* where arg is a string constant */
 #define DIR_ARG(name,flags,arg)                      \
   { #name "_" arg, test_dir_##name, (flags), &passthrough_setup, (void*) arg }
+#endif /* !defined(COCCI) */
 
 struct testcase_t dir_tests[] = {
   DIR_LEGACY(nicknames),
-  DIR_LEGACY(formats),
+  /* extrainfo without any stats */
+  DIR_ARG(formats_rsa, TT_FORK, ""),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, ""),
+  /* on a bridge */
+  DIR_ARG(formats_rsa, TT_FORK, "b"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "b"),
+  /* extrainfo with basic stats */
+  DIR_ARG(formats_rsa, TT_FORK, "e"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "e"),
+  DIR_ARG(formats_rsa, TT_FORK, "be"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "be"),
+  /* extrainfo with all stats */
+  DIR_ARG(formats_rsa, TT_FORK, "es"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "es"),
+  DIR_ARG(formats_rsa, TT_FORK, "bes"),
+  DIR_ARG(formats_rsa_ed25519, TT_FORK, "bes"),
   DIR(routerinfo_parsing, 0),
   DIR(extrainfo_parsing, 0),
   DIR(parse_router_list, TT_FORK),
@@ -6370,6 +7471,7 @@ struct testcase_t dir_tests[] = {
   DIR_LEGACY(measured_bw_kb_line_is_after_headers),
   DIR_LEGACY(measured_bw_kb_cache),
   DIR_LEGACY(dirserv_read_measured_bandwidths),
+  DIR(bwauth_bw_file_digest256, 0),
   DIR_LEGACY(param_voting),
   DIR(param_voting_lookup, 0),
   DIR_LEGACY(v3_networkstatus),
@@ -6378,7 +7480,7 @@ struct testcase_t dir_tests[] = {
   DIR_LEGACY(clip_unmeasured_bw_kb),
   DIR_LEGACY(clip_unmeasured_bw_kb_alt),
   DIR(fmt_control_ns, 0),
-  DIR(dirserv_set_routerstatus_testing, 0),
+  DIR(dirserv_set_routerstatus_testing, TT_FORK),
   DIR(http_handling, 0),
   DIR(purpose_needs_anonymity_returns_true_for_bridges, 0),
   DIR(purpose_needs_anonymity_returns_false_for_own_bridge_desc, 0),
@@ -6410,9 +7512,14 @@ struct testcase_t dir_tests[] = {
   DIR_ARG(find_dl_min_delay, TT_FORK, "cfr"),
   DIR_ARG(find_dl_min_delay, TT_FORK, "car"),
   DIR(assumed_flags, 0),
+  DIR(matching_flags, 0),
   DIR(networkstatus_compute_bw_weights_v10, 0),
   DIR(platform_str, 0),
-  DIR(networkstatus_consensus_has_ipv6, TT_FORK),
   DIR(format_versions_list, TT_FORK),
+  DIR(add_fingerprint, TT_FORK),
+  DIR(dirserv_load_fingerprint_file, TT_FORK),
+  DIR(dirserv_router_get_status, TT_FORK),
+  DIR(dirserv_would_reject_router, TT_FORK),
+  DIR(dirserv_add_own_fingerprint, TT_FORK),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_dir_common.c b/src/test/test_dir_common.c
index 3723d6c31b..f2b4e8724b 100644
--- a/src/test/test_dir_common.c
+++ b/src/test/test_dir_common.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -13,7 +13,7 @@
 #include "feature/dirparse/authcert_parse.h"
 #include "feature/dirparse/ns_parse.h"
 #include "test/test_dir_common.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/nodelist/authority_cert_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -42,14 +42,20 @@ dir_common_authority_pk_init(authority_cert_t **cert1,
 {
   /* Parse certificates and keys. */
   authority_cert_t *cert;
-  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                          strlen(AUTHORITY_CERT_1),
+                                          NULL);
   tt_assert(cert);
   tt_assert(cert->identity_key);
   *cert1 = cert;
   tt_assert(*cert1);
-  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2, NULL);
+  *cert2 = authority_cert_parse_from_string(AUTHORITY_CERT_2,
+                                            strlen(AUTHORITY_CERT_2),
+                                            NULL);
   tt_assert(*cert2);
-  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3, NULL);
+  *cert3 = authority_cert_parse_from_string(AUTHORITY_CERT_3,
+                                            strlen(AUTHORITY_CERT_3),
+                                            NULL);
   tt_assert(*cert3);
   *sign_skey_1 = crypto_pk_new();
   *sign_skey_2 = crypto_pk_new();
@@ -266,7 +272,9 @@ dir_common_add_rs_and_parse(networkstatus_t *vote, networkstatus_t **vote_out,
   /* dump the vote and try to parse it. */
   v_text = format_networkstatus_vote(sign_skey, vote);
   tt_assert(v_text);
-  *vote_out = networkstatus_parse_vote_from_string(v_text, NULL, NS_TYPE_VOTE);
+  *vote_out = networkstatus_parse_vote_from_string(v_text,
+                                                   strlen(v_text),
+                                                   NULL, NS_TYPE_VOTE);
 
  done:
   if (v_text)
@@ -424,4 +432,3 @@ dir_common_construct_vote_3(networkstatus_t **vote, authority_cert_t *cert,
 
   return 0;
 }
-
diff --git a/src/test/test_dir_common.h b/src/test/test_dir_common.h
index d6c5241b14..d37496465c 100644
--- a/src/test/test_dir_common.h
+++ b/src/test/test_dir_common.h
@@ -1,8 +1,11 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#ifndef TOR_TEST_DIR_COMMON_H
+#define TOR_TEST_DIR_COMMON_H
+
 #include "core/or/or.h"
 #include "feature/nodelist/networkstatus.h"
 
@@ -49,3 +52,4 @@ int dir_common_construct_vote_3(networkstatus_t **vote,
                         networkstatus_t **vote_out, int *n_vrs, time_t now,
                         int clear_rl);
 
+#endif /* !defined(TOR_TEST_DIR_COMMON_H) */
diff --git a/src/test/test_dir_handle_get.c b/src/test/test_dir_handle_get.c
index 90691fff94..f446bbb5eb 100644
--- a/src/test/test_dir_handle_get.c
+++ b/src/test/test_dir_handle_get.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define RENDCOMMON_PRIVATE
@@ -20,6 +20,7 @@
 #include "lib/compress/compress.h"
 #include "feature/rend/rendcommon.h"
 #include "feature/rend/rendcache.h"
+#include "feature/relay/relay_config.h"
 #include "feature/relay/router.h"
 #include "feature/nodelist/authcert.h"
 #include "feature/nodelist/dirlist.h"
@@ -37,7 +38,7 @@
 #include "feature/dircache/dirserv.h"
 #include "feature/dirauth/dirvote.h"
 #include "test/log_test_helpers.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/dirclient/dir_server_st.h"
@@ -54,17 +55,15 @@
 #endif /* defined(_WIN32) */
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
 #include "vote_descriptors.inc"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-ENABLE_GCC_WARNING(overlength-strings)
+ENABLE_GCC_WARNING("-Woverlength-strings")
 #endif
 
-#define NS_MODULE dir_handle_get
-
 #define NOT_FOUND "HTTP/1.0 404 Not found\r\n\r\n"
 #define BAD_REQUEST "HTTP/1.0 400 Bad request\r\n\r\n"
 #define SERVER_BUSY "HTTP/1.0 503 Directory busy, try again later\r\n\r\n"
@@ -72,6 +71,8 @@ ENABLE_GCC_WARNING(overlength-strings)
 #define NOT_ENOUGH_CONSENSUS_SIGNATURES "HTTP/1.0 404 " \
   "Consensus not signed by sufficient number of requested authorities\r\n\r\n"
 
+#define consdiffmgr_add_consensus consdiffmgr_add_consensus_nulterm
+
 static dir_connection_t *
 new_dir_conn(void)
 {
@@ -116,7 +117,7 @@ test_dir_handle_get_v1_command_not_found(void *data)
   conn = new_dir_conn();
 
   // no frontpage configured
-  tt_ptr_op(get_dirportfrontpage(), OP_EQ, NULL);
+  tt_ptr_op(relay_get_dirportfrontpage(), OP_EQ, NULL);
 
   /* V1 path */
   tt_int_op(directory_handle_command_get(conn, GET("/tor/"), NULL, 0),
@@ -150,9 +151,9 @@ test_dir_handle_get_v1_command(void *data)
   (void) data;
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
-  MOCK(get_dirportfrontpage, mock_get_dirportfrontpage);
+  MOCK(relay_get_dirportfrontpage, mock_get_dirportfrontpage);
 
-  exp_body = get_dirportfrontpage();
+  exp_body = relay_get_dirportfrontpage();
   body_len = strlen(exp_body);
 
   conn = new_dir_conn();
@@ -175,7 +176,7 @@ test_dir_handle_get_v1_command(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    UNMOCK(get_dirportfrontpage);
+    UNMOCK(relay_get_dirportfrontpage);
     connection_free_minimal(TO_CONN(conn));
     tor_free(header);
     tor_free(body);
@@ -255,7 +256,7 @@ test_dir_handle_get_rendezvous2_not_found_if_not_encrypted(void *data)
   conn = new_dir_conn();
 
   // connection is not encrypted
-  tt_assert(!connection_dir_is_encrypted(conn))
+  tt_assert(!connection_dir_is_encrypted(conn));
 
   tt_int_op(directory_handle_command_get(conn, RENDEZVOUS2_GET(), NULL, 0),
             OP_EQ, 0);
@@ -361,12 +362,13 @@ test_dir_handle_get_rendezvous2_not_found(void *data)
     rend_cache_free_all();
 }
 
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t * dhg_tests_router_get_my_routerinfo(void);
+ATTR_UNUSED static int dhg_tests_router_get_my_routerinfo_called = 0;
 
 static routerinfo_t *mock_routerinfo;
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+dhg_tests_router_get_my_routerinfo(void)
 {
   if (!mock_routerinfo) {
     mock_routerinfo = tor_malloc_zero(sizeof(routerinfo_t));
@@ -391,7 +393,8 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
   (void) data;
 
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -434,7 +437,7 @@ test_dir_handle_get_rendezvous2_on_encrypted_conn_success(void *data)
 
   done:
     UNMOCK(connection_write_to_buf_impl_);
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
 
     connection_free_minimal(TO_CONN(conn));
     tor_free(header);
@@ -477,8 +480,7 @@ static or_options_t *mock_options = NULL;
 static void
 init_mock_options(void)
 {
-  mock_options = tor_malloc(sizeof(or_options_t));
-  memset(mock_options, 0, sizeof(or_options_t));
+  mock_options = options_new();
   mock_options->TestingTorNetwork = 1;
   mock_options->DataDirectory = tor_strdup(get_fname_rnd("datadir_tmp"));
   mock_options->CacheDirectory = tor_strdup(mock_options->DataDirectory);
@@ -767,7 +769,8 @@ test_dir_handle_get_server_descriptors_all(void* data)
   helper_setup_fake_routerlist();
 
   //TODO: change to router_get_my_extrainfo when testing "extra" path
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
   // We are one of the routers
@@ -809,7 +812,7 @@ test_dir_handle_get_server_descriptors_all(void* data)
   tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     connection_free_minimal(TO_CONN(conn));
     tor_free(header);
@@ -866,7 +869,8 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   crypto_pk_t *identity_pkey = pk_generate(0);
   (void) data;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
   /* init mock */
@@ -911,7 +915,7 @@ test_dir_handle_get_server_descriptors_authority(void* data)
   tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
@@ -931,7 +935,8 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   crypto_pk_t *identity_pkey = pk_generate(0);
   (void) data;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       dhg_tests_router_get_my_routerinfo);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
   /* init mock */
@@ -983,7 +988,7 @@ test_dir_handle_get_server_descriptors_fp(void* data)
   tt_ptr_op(conn->spool, OP_EQ, NULL);
 
   done:
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(router_get_my_routerinfo);
     UNMOCK(connection_write_to_buf_impl_);
     tor_free(mock_routerinfo->cache_info.signed_descriptor_body);
     tor_free(mock_routerinfo);
@@ -1275,7 +1280,9 @@ test_dir_handle_get_server_keys_authority(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
@@ -1425,7 +1432,9 @@ test_dir_handle_get_server_keys_sk(void* data)
   size_t body_used = 0;
   (void) data;
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
   MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
 
@@ -1783,13 +1792,14 @@ test_dir_handle_get_status_vote_current_consensus_too_old(void *data)
     or_options_free(mock_options); mock_options = NULL;
 }
 
-NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+static int dhg_tests_geoip_get_country_by_addr(const tor_addr_t *addr);
+ATTR_UNUSED static int dhg_tests_geoip_get_country_by_addr_called = 0;
 
 int
-NS(geoip_get_country_by_addr)(const tor_addr_t *addr)
+dhg_tests_geoip_get_country_by_addr(const tor_addr_t *addr)
 {
   (void)addr;
-  CALLED(geoip_get_country_by_addr)++;
+  dhg_tests_geoip_get_country_by_addr_called++;
   return 1;
 }
 
@@ -1853,7 +1863,8 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
   dirserv_free_all();
   clear_geoip_db();
 
-  NS_MOCK(geoip_get_country_by_addr);
+  MOCK(geoip_get_country_by_addr,
+       dhg_tests_geoip_get_country_by_addr);
   MOCK(get_options, mock_get_options);
 
   init_mock_options();
@@ -1890,7 +1901,7 @@ test_dir_handle_get_status_vote_current_consensus_ns(void* data)
   tt_str_op("ab=8", OP_EQ, hist);
 
   done:
-    NS_UNMOCK(geoip_get_country_by_addr);
+    UNMOCK(geoip_get_country_by_addr);
     UNMOCK(get_options);
     tor_free(header);
     tor_free(comp_body);
@@ -2069,12 +2080,12 @@ test_dir_handle_get_status_vote_d(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  voting_schedule_recalculate_timing(mock_options, now);
+  dirauth_sched_recalculate_timing(mock_options, now);
 
   const char *msg_out = NULL;
   int status_out = 0;
-  struct pending_vote_t *pv = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
-                                               &status_out);
+  struct pending_vote_t *pv = dirvote_add_vote(VOTE_BODY_V3, 0,
+                                               &msg_out, &status_out);
   tt_assert(pv);
 
   status_vote_current_d_test(&header, &body, &body_used);
@@ -2217,11 +2228,36 @@ test_dir_handle_get_status_vote_next_authority_not_found(void* data)
     tor_free(header);
 }
 
-NS_DECL(const char*,
-dirvote_get_pending_consensus, (consensus_flavor_t flav));
+static void
+test_dir_handle_get_status_vote_next_bandwidth_not_found(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL;
+  (void) data;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwdith"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      NULL, NULL, 1, 0);
+  tt_assert(header);
+  tt_str_op(NOT_FOUND, OP_EQ, header);
+
+  done:
+    UNMOCK(connection_write_to_buf_impl_);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+}
+
+static const char* dhg_tests_dirvote_get_pending_consensus(
+                                           consensus_flavor_t flav);
 
 const char*
-NS(dirvote_get_pending_consensus)(consensus_flavor_t flav)
+dhg_tests_dirvote_get_pending_consensus(consensus_flavor_t flav)
 {
   (void)flav;
   return "pending consensus";
@@ -2234,7 +2270,8 @@ test_dir_handle_get_status_vote_next_consensus(void* data)
   size_t body_used = 0;
   (void) data;
 
-  NS_MOCK(dirvote_get_pending_consensus);
+  MOCK(dirvote_get_pending_consensus,
+       dhg_tests_dirvote_get_pending_consensus);
 
   status_vote_next_consensus_test(&header, &body, &body_used);
   tt_assert(header);
@@ -2247,7 +2284,7 @@ test_dir_handle_get_status_vote_next_consensus(void* data)
   tt_str_op("pending consensus", OP_EQ, body);
 
   done:
-    NS_UNMOCK(dirvote_get_pending_consensus);
+    UNMOCK(dirvote_get_pending_consensus);
     tor_free(header);
     tor_free(body);
 }
@@ -2260,7 +2297,8 @@ test_dir_handle_get_status_vote_next_consensus_busy(void* data)
   (void) data;
 
   MOCK(get_options, mock_get_options);
-  NS_MOCK(dirvote_get_pending_consensus);
+  MOCK(dirvote_get_pending_consensus,
+       dhg_tests_dirvote_get_pending_consensus);
 
   //Make it busy
   init_mock_options();
@@ -2272,7 +2310,7 @@ test_dir_handle_get_status_vote_next_consensus_busy(void* data)
   tt_str_op(SERVER_BUSY, OP_EQ, header);
 
   done:
-    NS_UNMOCK(dirvote_get_pending_consensus);
+    UNMOCK(dirvote_get_pending_consensus);
     UNMOCK(get_options);
     tor_free(header);
     tor_free(body);
@@ -2316,11 +2354,10 @@ test_dir_handle_get_status_vote_next_consensus_signatures_not_found(void* data)
     tor_free(body);
 }
 
-NS_DECL(const char*,
-dirvote_get_pending_detached_signatures, (void));
+static const char* dhg_tests_dirvote_get_pending_detached_signatures(void);
 
 const char*
-NS(dirvote_get_pending_detached_signatures)(void)
+dhg_tests_dirvote_get_pending_detached_signatures(void)
 {
   return "pending detached sigs";
 }
@@ -2332,7 +2369,8 @@ test_dir_handle_get_status_vote_next_consensus_signatures(void* data)
   size_t body_used = 0;
   (void) data;
 
-  NS_MOCK(dirvote_get_pending_detached_signatures);
+  MOCK(dirvote_get_pending_detached_signatures,
+       dhg_tests_dirvote_get_pending_detached_signatures);
 
   status_vote_next_consensus_signatures_test(&header, &body, &body_used);
   tt_assert(header);
@@ -2345,7 +2383,7 @@ test_dir_handle_get_status_vote_next_consensus_signatures(void* data)
   tt_str_op("pending detached sigs", OP_EQ, body);
 
   done:
-    NS_UNMOCK(dirvote_get_pending_detached_signatures);
+    UNMOCK(dirvote_get_pending_detached_signatures);
     tor_free(header);
     tor_free(body);
 }
@@ -2357,7 +2395,8 @@ test_dir_handle_get_status_vote_next_consensus_signatures_busy(void* data)
   size_t body_used;
   (void) data;
 
-  NS_MOCK(dirvote_get_pending_detached_signatures);
+  MOCK(dirvote_get_pending_detached_signatures,
+       dhg_tests_dirvote_get_pending_detached_signatures);
   MOCK(get_options, mock_get_options);
 
   //Make it busy
@@ -2371,7 +2410,7 @@ test_dir_handle_get_status_vote_next_consensus_signatures_busy(void* data)
 
   done:
     UNMOCK(get_options);
-    NS_UNMOCK(dirvote_get_pending_detached_signatures);
+    UNMOCK(dirvote_get_pending_detached_signatures);
     tor_free(header);
     tor_free(body);
     or_options_free(mock_options); mock_options = NULL;
@@ -2393,7 +2432,9 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2416,10 +2457,10 @@ test_dir_handle_get_status_vote_next_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455 -1;
-  voting_schedule_recalculate_timing(mock_options, now);
+  dirauth_sched_recalculate_timing(mock_options, now);
 
-  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
-                                                 &status_out);
+  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, 0,
+                                                 &msg_out, &status_out);
   tt_assert(vote);
 
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
@@ -2455,6 +2496,85 @@ test_dir_handle_get_status_vote_next_authority(void* data)
 }
 
 static void
+test_dir_handle_get_status_vote_next_bandwidth(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL, *body = NULL;
+  size_t body_used = 0;
+  (void) data;
+
+  const char *content =
+    "1541171221\n"
+    "node_id=$68A483E05A2ABDCA6DA5A3EF8DB5177638A27F80 "
+    "master_key_ed25519=YaqV4vbvPYKucElk297eVdNArDz9HtIwUoIeo0+cVIpQ "
+    "bw=760 nick=Test time=2018-05-08T16:13:26\n";
+
+  init_mock_options();
+  MOCK(get_options, mock_get_options);
+  mock_options->V3BandwidthsFile = tor_strdup(
+    get_fname_rnd("V3BandwidthsFile")
+  );
+
+  write_str_to_file(mock_options->V3BandwidthsFile, content, 0);
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
+  tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
+  tt_assert(strstr(header, "Content-Length: 167\r\n"));
+
+  /* Check cache lifetime */
+  char expbuf[RFC1123_TIME_LEN+1];
+  time_t now = approx_time();
+  /* BANDWIDTH_CACHE_LIFETIME is defined in dircache.c. */
+  format_rfc1123_time(expbuf, (time_t)(now + 30*60));
+  char *expires = NULL;
+  /* Change to 'Cache-control: max-age=%d' if using http/1.1. */
+  tor_asprintf(&expires, "Expires: %s\r\n", expbuf);
+  tt_assert(strstr(header, expires));
+
+  tt_int_op(body_used, OP_EQ, strlen(body));
+  tt_str_op(content, OP_EQ, body);
+
+  tor_free(header);
+  tor_free(body);
+
+  /* Request the file using compression, the result should be the same. */
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/next/bandwidth.z"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(content)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Encoding: deflate\r\n"));
+
+  /* Since using connection_write_to_buf_mock instead of mocking
+   * connection_buf_add_compress, the content is not actually compressed.
+   * If it would, the size and content would be different than the original.
+  */
+
+ done:
+  UNMOCK(get_options);
+  UNMOCK(connection_write_to_buf_impl_);
+  connection_free_minimal(TO_CONN(conn));
+  tor_free(header);
+  tor_free(body);
+  tor_free(expires);
+  or_options_free(mock_options);
+}
+
+static void
 test_dir_handle_get_status_vote_current_authority(void* data)
 {
   dir_connection_t *conn = NULL;
@@ -2471,7 +2591,9 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   routerlist_free_all();
   dirvote_free_all();
 
-  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE, NULL);
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
 
   /* create a trusted ds */
   ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
@@ -2495,10 +2617,10 @@ test_dir_handle_get_status_vote_current_authority(void* data)
   mock_options->TestingV3AuthInitialDistDelay = 1;
 
   time_t now = 1441223455;
-  voting_schedule_recalculate_timing(mock_options, now-1);
+  dirauth_sched_recalculate_timing(mock_options, now-1);
 
-  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, &msg_out,
-                                                 &status_out);
+  struct pending_vote_t *vote = dirvote_add_vote(VOTE_BODY_V3, 0,
+                                                 &msg_out, &status_out);
   tt_assert(vote);
 
   // move the pending vote to previous vote
@@ -2536,6 +2658,183 @@ test_dir_handle_get_status_vote_current_authority(void* data)
     dirvote_free_all();
 }
 
+/* Test that a late vote is rejected, but an on-time vote is accepted. */
+static void
+test_dir_handle_get_status_vote_too_late(void* data)
+{
+  dir_connection_t *conn = NULL;
+  char *header = NULL, *body = NULL;
+  const char *msg_out = NULL;
+  int status_out = 0;
+  size_t body_used = 0;
+  const char digest[DIGEST_LEN] = "";
+
+  dir_server_t *ds = NULL;
+  const char* mode = (const char *)data;
+
+  clear_dir_servers();
+  routerlist_free_all();
+  dirvote_free_all();
+
+  mock_cert = authority_cert_parse_from_string(TEST_CERTIFICATE,
+                                               strlen(TEST_CERTIFICATE),
+                                               NULL);
+
+  /* create a trusted ds */
+  ds = trusted_dir_server_new("ds", "127.0.0.1", 9059, 9060, NULL, digest,
+                              NULL, V3_DIRINFO, 1.0);
+  tt_assert(ds);
+  dir_server_add(ds);
+
+  /* ds v3_identity_digest is the certificate's identity_key */
+  base16_decode(ds->v3_identity_digest, DIGEST_LEN,
+                TEST_CERT_IDENT_KEY, HEX_DIGEST_LEN);
+
+  tt_int_op(0, OP_EQ, trusted_dirs_load_certs_from_string(TEST_CERTIFICATE,
+    TRUSTED_DIRS_CERTS_SRC_DL_BY_ID_DIGEST, 1, NULL));
+
+  init_mock_options();
+  mock_options->AuthoritativeDir = 1;
+  mock_options->V3AuthoritativeDir = 1;
+
+  int base_delay = 0;
+  int vote_interval = 0;
+  int start_offset = 0;
+
+  tt_assert(mode);
+  /* Set the required timings, see below for details */
+  if (strcmp(mode, "min") == 0) {
+    /* The minimum valid test network timing */
+    base_delay = 2;
+    vote_interval = 10;
+    start_offset = vote_interval - 5;
+  } else if (strcmp(mode, "chutney") == 0) {
+    /* The test network timing used by chutney */
+    base_delay = 4;
+    vote_interval = 20;
+    start_offset = vote_interval - 5;
+  } else if (strcmp(mode, "half-public") == 0) {
+    /* The short consensus failure timing used in the public network */
+    base_delay = 5*60;
+    vote_interval = 30*60;
+    start_offset = vote_interval - 9*60 - 5;
+  } else if (strcmp(mode, "public") == 0) {
+    /* The standard timing used in the public network */
+    base_delay = 5*60;
+    vote_interval = 60*60;
+    start_offset = vote_interval - 9*60 - 5;
+  }
+
+  tt_assert(base_delay > 0);
+  tt_assert(vote_interval > 0);
+  tt_assert(start_offset > 0);
+
+  /* Skew the time to fit the fixed time in the vote */
+  mock_options->TestingV3AuthVotingStartOffset = start_offset;
+  /* Calculate the rest of the timings */
+  mock_options->TestingV3AuthInitialVotingInterval = vote_interval;
+  mock_options->TestingV3AuthInitialVoteDelay = base_delay;
+  mock_options->TestingV3AuthInitialDistDelay = base_delay;
+
+  time_t now = 1441223455;
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+  const time_t voting_starts = voting_schedule.voting_starts;
+  const time_t fetch_missing = voting_schedule.fetch_missing_votes;
+
+  struct pending_vote_t *vote = NULL;
+
+  /* Next voting interval */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          fetch_missing + vote_interval,
+                          &msg_out, &status_out);
+  tt_assert(!vote);
+  tt_int_op(status_out, OP_EQ, 400);
+  tt_str_op(msg_out, OP_EQ,
+            "Posted vote received too late, would be dangerous to count it");
+
+  /* Just after fetch missing */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          fetch_missing + 1,
+                          &msg_out, &status_out);
+  tt_assert(!vote);
+  tt_int_op(status_out, OP_EQ, 400);
+  tt_str_op(msg_out, OP_EQ,
+            "Posted vote received too late, would be dangerous to count it");
+
+  /* On fetch missing */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          fetch_missing,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+  /* And reset the timing */
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+
+  /* Between voting starts and fetch missing */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          voting_starts + 1,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+  /* And reset the timing */
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+
+  /* On voting starts */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          voting_starts,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+  /* And reset the timing */
+  dirauth_sched_recalculate_timing(mock_options, now-1);
+
+  /* Just before voting starts */
+  vote = dirvote_add_vote(VOTE_BODY_V3,
+                          voting_starts - 1,
+                          &msg_out, &status_out);
+  tt_assert(vote);
+
+  /* Move the pending vote to previous vote */
+  dirvote_act(mock_options, now+1);
+
+  MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  conn = new_dir_conn();
+  tt_int_op(0, OP_EQ, directory_handle_command_get(conn,
+    GET("/tor/status-vote/current/authority"), NULL, 0));
+
+  fetch_from_buf_http(TO_CONN(conn)->outbuf, &header, MAX_HEADERS_SIZE,
+                      &body, &body_used, strlen(VOTE_BODY_V3)+1, 0);
+
+  tt_assert(header);
+  tt_ptr_op(strstr(header, "HTTP/1.0 200 OK\r\n"), OP_EQ, header);
+  tt_assert(strstr(header, "Content-Type: text/plain\r\n"));
+  tt_assert(strstr(header, "Content-Encoding: identity\r\n"));
+  tt_assert(strstr(header, "Content-Length: 4135\r\n"));
+
+  tt_str_op(VOTE_BODY_V3, OP_EQ, body);
+
+  done:
+    UNMOCK(connection_write_to_buf_impl_);
+    UNMOCK(get_my_v3_authority_cert);
+    connection_free_minimal(TO_CONN(conn));
+    tor_free(header);
+    tor_free(body);
+    authority_cert_free(mock_cert); mock_cert = NULL;
+    or_options_free(mock_options); mock_options = NULL;
+
+    clear_dir_servers();
+    routerlist_free_all();
+    dirvote_free_all();
+}
+
 static void
 test_dir_handle_get_parse_accept_encoding(void *arg)
 {
@@ -2586,6 +2885,16 @@ test_dir_handle_get_parse_accept_encoding(void *arg)
 #define DIR_HANDLE_CMD(name,flags) \
   { #name, test_dir_handle_get_##name, (flags), NULL, NULL }
 
+#ifdef COCCI
+/* Coccinelle doesn't like the stringification in this macro */
+#define DIR_HANDLE_CMD_ARG(name,flags,arg) \
+  DIR_HANDLE_CMD(name,flags)
+#else
+#define DIR_HANDLE_CMD_ARG(name,flags,arg) \
+  { #name "/" arg, test_dir_handle_get_##name, (flags), \
+    &passthrough_setup, (void *)(arg) }
+#endif /* defined(COCCI) */
+
 struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(not_found, 0),
   DIR_HANDLE_CMD(bad_request, 0),
@@ -2625,8 +2934,14 @@ struct testcase_t dir_handle_get_tests[] = {
   DIR_HANDLE_CMD(status_vote_next_not_found, 0),
   DIR_HANDLE_CMD(status_vote_current_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_current_authority, 0),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "min"),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "chutney"),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "half-public"),
+  DIR_HANDLE_CMD_ARG(status_vote_too_late, 0, "public"),
   DIR_HANDLE_CMD(status_vote_next_authority_not_found, 0),
   DIR_HANDLE_CMD(status_vote_next_authority, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth_not_found, 0),
+  DIR_HANDLE_CMD(status_vote_next_bandwidth, 0),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_enough_sigs, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_ns_not_found, TT_FORK),
   DIR_HANDLE_CMD(status_vote_current_consensus_too_old, TT_FORK),
diff --git a/src/test/test_dispatch.c b/src/test/test_dispatch.c
new file mode 100644
index 0000000000..77f33e4b15
--- /dev/null
+++ b/src/test/test_dispatch.c
@@ -0,0 +1,278 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_NEW_PRIVATE
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_cfg.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static dispatch_t *dispatcher_in_use=NULL;
+
+static void
+test_dispatch_max_in_u16_sl(void *arg)
+{
+  (void)arg;
+  smartlist_t *sl = smartlist_new();
+  uint16_t nums[] = { 10, 20, 30 };
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(-1, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[1]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[0]);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, NULL);
+  tt_int_op(20, OP_EQ, max_in_u16_sl(sl, -1));
+
+  smartlist_add(sl, &nums[2]);
+  tt_int_op(30, OP_EQ, max_in_u16_sl(sl, -1));
+
+ done:
+  smartlist_free(sl);
+}
+
+/* Construct an empty dispatch_t. */
+static void
+test_dispatch_empty(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+
+  cfg = dcfg_new();
+  d = dispatch_new(cfg);
+  tt_assert(d);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+static int total_recv1_simple = 0;
+static int total_recv2_simple = 0;
+
+static void
+simple_recv1(const msg_t *m)
+{
+  total_recv1_simple += m->aux_data__.u64;
+}
+
+static char *recv2_received = NULL;
+
+static void
+simple_recv2(const msg_t *m)
+{
+  tor_free(recv2_received);
+  recv2_received = dispatch_fmt_msg_data(dispatcher_in_use, m);
+
+  total_recv2_simple += m->aux_data__.u64*10;
+}
+
+/* Construct a dispatch_t with two messages, make sure that they both get
+ * delivered. */
+static void
+test_dispatch_simple(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  r += dcfg_add_recv(cfg,0,1,simple_recv1);
+  r += dcfg_msg_set_type(cfg,1,0);
+  r += dcfg_msg_set_chan(cfg,1,0);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2);
+  r += dcfg_add_recv(cfg,1,1,simple_recv2); /* second copy */
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = {.u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+  tt_int_op(total_recv1_simple, OP_EQ, 7);
+  tt_int_op(total_recv2_simple, OP_EQ, 0);
+
+  total_recv1_simple = 0;
+  r = dispatch_send(d, 99, 0, 1, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(total_recv1_simple, OP_EQ, 0);
+  tt_int_op(total_recv2_simple, OP_EQ, 140);
+
+  tt_str_op(recv2_received, OP_EQ, "<>"); // no format function was set.
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(recv2_received);
+}
+
+/* Construct a dispatch_t with a message and no reciever; make sure that it
+ * gets dropped properly. */
+static void
+test_dispatch_no_recipient(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,0,0);
+  r += dcfg_msg_set_chan(cfg,0,0);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  msg_aux_data_t data = { .u64 = 7};
+  r = dispatch_send(d, 99, 0, 0, 0, data);
+  tt_int_op(r, OP_EQ, 0);
+
+  r = dispatch_flush(d, 0, INT_MAX);
+  tt_int_op(r, OP_EQ, 0);
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+}
+
+struct coord_t { int x; int y; };
+static void
+free_coord(msg_aux_data_t d)
+{
+  tor_free(d.ptr);
+}
+static char *
+fmt_coord(msg_aux_data_t d)
+{
+  char *v;
+  struct coord_t *c = d.ptr;
+  tor_asprintf(&v, "[%d, %d]", c->x, c->y);
+  return v;
+}
+static dispatch_typefns_t coord_fns = {
+  .fmt_fn = fmt_coord,
+  .free_fn = free_coord,
+};
+static void
+alert_run_immediate(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  (void)arg;
+  dispatch_flush(d, ch, INT_MAX);
+}
+
+static char *received_data=NULL;
+
+static void
+recv_typed_data(const msg_t *m)
+{
+  tor_free(received_data);
+  received_data = dispatch_fmt_msg_data(dispatcher_in_use, m);
+}
+
+static void
+test_dispatch_with_types(void *arg)
+{
+  (void)arg;
+
+  dispatch_t *d=NULL;
+  dispatch_cfg_t *cfg=NULL;
+  int r;
+
+  cfg = dcfg_new();
+  r = dcfg_msg_set_type(cfg,5,3);
+  r += dcfg_msg_set_chan(cfg,5,2);
+  r += dcfg_add_recv(cfg,5,0,recv_typed_data);
+  r += dcfg_type_set_fns(cfg,3,&coord_fns);
+  tt_int_op(r, OP_EQ, 0);
+
+  d = dispatch_new(cfg);
+  tt_assert(d);
+  dispatcher_in_use = d;
+
+  /* Make this message get run immediately. */
+  r = dispatch_set_alert_fn(d, 2, alert_run_immediate, NULL);
+  tt_int_op(r, OP_EQ, 0);
+
+  struct coord_t *xy = tor_malloc(sizeof(*xy));
+  xy->x = 13;
+  xy->y = 37;
+  msg_aux_data_t data = {.ptr = xy};
+  r = dispatch_send(d, 99/*sender*/, 2/*channel*/, 5/*msg*/, 3/*type*/, data);
+  tt_int_op(r, OP_EQ, 0);
+  tt_str_op(received_data, OP_EQ, "[13, 37]");
+
+ done:
+  dispatch_free(d);
+  dcfg_free(cfg);
+  tor_free(received_data);
+  dispatcher_in_use = NULL;
+}
+
+static void
+test_dispatch_bad_type_setup(void *arg)
+{
+  (void)arg;
+  static dispatch_typefns_t fns;
+  dispatch_cfg_t *cfg = dcfg_new();
+
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &coord_fns));
+
+  fns = coord_fns;
+  fns.fmt_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  fns.free_fn = NULL;
+  tt_int_op(-1, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+  fns = coord_fns;
+  tt_int_op(0, OP_EQ, dcfg_type_set_fns(cfg, 7, &fns));
+
+ done:
+  dcfg_free(cfg);
+}
+
+#define T(name)                                                 \
+  { #name, test_dispatch_ ## name, TT_FORK, NULL, NULL }
+
+struct testcase_t dispatch_tests[] = {
+  T(max_in_u16_sl),
+  T(empty),
+  T(simple),
+  T(no_recipient),
+  T(with_types),
+  T(bad_type_setup),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_dns.c b/src/test/test_dns.c
index 41a56f65d8..299321ab64 100644
--- a/src/test/test_dns.c
+++ b/src/test/test_dns.c
@@ -1,6 +1,7 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#include "orconfig.h"
 #include "core/or/or.h"
 #include "test/test.h"
 
@@ -13,29 +14,81 @@
 
 #include "core/or/edge_connection_st.h"
 #include "core/or/or_circuit_st.h"
+#include "app/config/or_options_st.h"
+#include "app/config/config.h"
 
-#define NS_MODULE dns
+#include <event2/event.h>
+#include <event2/dns.h>
 
-#define NS_SUBMODULE clip_ttl
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+
+static or_options_t options = {
+  .ORPort_set = 1,
+};
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &options;
+}
 
 static void
-NS(test_main)(void *arg)
+test_dns_configure_ns_fallback(void *arg)
 {
   (void)arg;
+  tor_addr_t *nameserver_addr = NULL;
 
-  uint32_t ttl_mid = MIN_DNS_TTL_AT_EXIT / 2 + MAX_DNS_TTL_AT_EXIT / 2;
+  MOCK(get_options, mock_get_options);
 
-  tt_int_op(dns_clip_ttl(MIN_DNS_TTL_AT_EXIT - 1),OP_EQ,MIN_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(ttl_mid),OP_EQ,MAX_DNS_TTL_AT_EXIT);
-  tt_int_op(dns_clip_ttl(MAX_DNS_TTL_AT_EXIT + 1),OP_EQ,MAX_DNS_TTL_AT_EXIT);
+  options.ServerDNSResolvConfFile = (char *)"no_such_file!!!";
 
-  done:
+  dns_init(); // calls configure_nameservers()
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+
+#ifndef _WIN32
+  tor_free(nameserver_addr);
+
+  options.ServerDNSResolvConfFile = (char *)"/dev/null";
+
+  dns_init();
+
+  tt_int_op(number_of_configured_nameservers(), OP_EQ, 1);
+
+  nameserver_addr = configured_nameserver_address(0);
+
+  tt_assert(tor_addr_family(nameserver_addr) == AF_INET);
+  tt_assert(tor_addr_eq_ipv4h(nameserver_addr, 0x7f000001));
+#endif /* !defined(_WIN32) */
+
+  UNMOCK(get_options);
+
+ done:
+  tor_free(nameserver_addr);
   return;
 }
 
-#undef NS_SUBMODULE
+#endif /* defined(HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR) */
+
+static void
+test_dns_clip_ttl(void *arg)
+{
+  (void)arg;
 
-#define NS_SUBMODULE resolve
+  uint32_t ttl_mid = MIN_DNS_TTL / 2 + MAX_DNS_TTL / 2;
+
+  tt_int_op(clip_dns_ttl(MIN_DNS_TTL - 1),OP_EQ,MIN_DNS_TTL);
+  tt_int_op(clip_dns_ttl(ttl_mid),OP_EQ,MAX_DNS_TTL);
+  tt_int_op(clip_dns_ttl(MAX_DNS_TTL + 1),OP_EQ,MAX_DNS_TTL);
+
+  done:
+  return;
+}
 
 static int resolve_retval = 0;
 static int resolve_made_conn_pending = 0;
@@ -44,10 +97,11 @@ static cached_resolve_t *cache_entry_mock = NULL;
 
 static int n_fake_impl = 0;
 
-NS_DECL(int, dns_resolve_impl, (edge_connection_t *exitconn, int is_resolve,
-                                or_circuit_t *oncirc, char **hostname_out,
-                                int *made_connection_pending_out,
-                                cached_resolve_t **resolve_out));
+static int dns_resolve_dns_resolve_impl(edge_connection_t *exitconn,
+                     int is_resolve, or_circuit_t *oncirc,
+                     char **hostname_out, int *made_connection_pending_out,
+                     cached_resolve_t **resolve_out);
+ATTR_UNUSED static int dns_resolve_dns_resolve_impl_called = 0;
 
 /** This will be our configurable substitute for <b>dns_resolve_impl</b> in
  * dns.c. It will return <b>resolve_retval</b>,
@@ -58,7 +112,7 @@ NS_DECL(int, dns_resolve_impl, (edge_connection_t *exitconn, int is_resolve,
  * 1.
  */
 static int
-NS(dns_resolve_impl)(edge_connection_t *exitconn, int is_resolve,
+dns_resolve_dns_resolve_impl(edge_connection_t *exitconn, int is_resolve,
                      or_circuit_t *oncirc, char **hostname_out,
                      int *made_connection_pending_out,
                      cached_resolve_t **resolve_out)
@@ -88,7 +142,7 @@ static uint8_t last_answer_type = 0;
 static cached_resolve_t *last_resolved;
 
 static void
-NS(send_resolved_cell)(edge_connection_t *conn, uint8_t answer_type,
+dns_resolve_send_resolved_cell(edge_connection_t *conn, uint8_t answer_type,
                        const cached_resolve_t *resolved)
 {
   conn_for_resolved_cell = conn;
@@ -104,7 +158,7 @@ static int n_send_resolved_hostname_cell_replacement = 0;
 static char *last_resolved_hostname = NULL;
 
 static void
-NS(send_resolved_hostname_cell)(edge_connection_t *conn,
+dns_resolve_send_resolved_hostname_cell(edge_connection_t *conn,
                                 const char *hostname)
 {
   conn_for_resolved_cell = conn;
@@ -118,7 +172,7 @@ NS(send_resolved_hostname_cell)(edge_connection_t *conn,
 static int n_dns_cancel_pending_resolve_replacement = 0;
 
 static void
-NS(dns_cancel_pending_resolve)(const char *address)
+dns_resolve_dns_cancel_pending_resolve(const char *address)
 {
   (void) address;
   n_dns_cancel_pending_resolve_replacement++;
@@ -128,7 +182,7 @@ static int n_connection_free = 0;
 static connection_t *last_freed_conn = NULL;
 
 static void
-NS(connection_free_)(connection_t *conn)
+dns_resolve_connection_free_(connection_t *conn)
 {
    n_connection_free++;
 
@@ -136,7 +190,7 @@ NS(connection_free_)(connection_t *conn)
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_resolve(void *arg)
 {
   (void) arg;
   int retval;
@@ -155,9 +209,12 @@ NS(test_main)(void *arg)
   memset(exitconn,0,sizeof(edge_connection_t));
   memset(nextconn,0,sizeof(edge_connection_t));
 
-  NS_MOCK(dns_resolve_impl);
-  NS_MOCK(send_resolved_cell);
-  NS_MOCK(send_resolved_hostname_cell);
+  MOCK(dns_resolve_impl,
+       dns_resolve_dns_resolve_impl);
+  MOCK(send_resolved_cell,
+       dns_resolve_send_resolved_cell);
+  MOCK(send_resolved_hostname_cell,
+       dns_resolve_send_resolved_hostname_cell);
 
   /*
    * CASE 1: dns_resolve_impl returns 1 and sets a hostname. purpose is
@@ -270,8 +327,10 @@ NS(test_main)(void *arg)
    * on exitconn with type being RESOLVED_TYPE_ERROR.
    */
 
-  NS_MOCK(dns_cancel_pending_resolve);
-  NS_MOCK(connection_free_);
+  MOCK(dns_cancel_pending_resolve,
+       dns_resolve_dns_cancel_pending_resolve);
+  MOCK(connection_free_,
+       dns_resolve_connection_free_);
 
   exitconn->on_circuit = &(on_circuit->base_);
   exitconn->base_.purpose = EXIT_PURPOSE_RESOLVE;
@@ -294,11 +353,11 @@ NS(test_main)(void *arg)
   tt_assert(last_freed_conn == TO_CONN(exitconn));
 
   done:
-  NS_UNMOCK(dns_resolve_impl);
-  NS_UNMOCK(send_resolved_cell);
-  NS_UNMOCK(send_resolved_hostname_cell);
-  NS_UNMOCK(dns_cancel_pending_resolve);
-  NS_UNMOCK(connection_free_);
+  UNMOCK(dns_resolve_impl);
+  UNMOCK(send_resolved_cell);
+  UNMOCK(send_resolved_hostname_cell);
+  UNMOCK(dns_cancel_pending_resolve);
+  UNMOCK(connection_free_);
   tor_free(on_circuit);
   tor_free(exitconn);
   tor_free(nextconn);
@@ -308,8 +367,6 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
 /** Create an <b>edge_connection_t</b> instance that is considered a
  * valid exit connection by asserts in dns_resolve_impl.
  */
@@ -326,8 +383,6 @@ create_valid_exitconn(void)
   return exitconn;
 }
 
-#define NS_SUBMODULE ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve)
-
 /*
  * Given that <b>exitconn->base_.address</b> is IP address string, we
  * want dns_resolve_impl() to parse it and store in
@@ -336,7 +391,7 @@ create_valid_exitconn(void)
  */
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_addr_is_ip(void *arg)
 {
   int retval;
   int made_pending;
@@ -369,21 +424,17 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, non_exit)
-
 /** Given that Tor instance is not configured as an exit node, we want
  * dns_resolve_impl() to fail with return value -1.
  */
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_non_exit_router_my_exit_policy_is_reject_star(void)
 {
   return 1;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_non_exit(void *arg)
 {
   int retval;
   int made_pending;
@@ -395,7 +446,8 @@ NS(test_main)(void *arg)
 
   TO_CONN(exitconn)->address = tor_strdup("torproject.org");
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_non_exit_router_my_exit_policy_is_reject_star);
 
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
@@ -406,27 +458,23 @@ NS(test_main)(void *arg)
   tor_free(TO_CONN(exitconn)->address);
   tor_free(exitconn);
   tor_free(on_circ);
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, addr_is_invalid_dest)
-
 /** Given that address is not a valid destination (as judged by
  * address_is_invalid_destination() function), we want dns_resolve_impl()
  * function to fail with return value -1.
  */
 
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_addr_is_invalid_dest_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_addr_is_invalid_dest(void *arg)
 {
   int retval;
   int made_pending;
@@ -436,7 +484,8 @@ NS(test_main)(void *arg)
 
   (void)arg;
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_addr_is_invalid_dest_router_my_exit_policy_is_reject_star);
 
   TO_CONN(exitconn)->address = tor_strdup("invalid#@!.org");
 
@@ -446,29 +495,25 @@ NS(test_main)(void *arg)
   tt_int_op(retval,OP_EQ,-1);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(exitconn);
   tor_free(on_circ);
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, malformed_ptr)
-
 /** Given that address is a malformed PTR name, we want dns_resolve_impl to
  * fail.
  */
 
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_malformed_ptr_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_malformed_ptr(void *arg)
 {
   int retval;
   int made_pending;
@@ -480,7 +525,8 @@ NS(test_main)(void *arg)
 
   TO_CONN(exitconn)->address = tor_strdup("1.0.0.127.in-addr.arpa");
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_malformed_ptr_router_my_exit_policy_is_reject_star);
 
   retval = dns_resolve_impl(exitconn, 1, on_circ, NULL, &made_pending,
                             NULL);
@@ -498,30 +544,26 @@ NS(test_main)(void *arg)
   tt_int_op(retval,OP_EQ,-1);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(exitconn);
   tor_free(on_circ);
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, cache_hit_pending)
-
 /* Given that there is already a pending resolve for the given address,
  * we want dns_resolve_impl to append our exit connection to list
  * of pending connections for the pending DNS request and return 0.
  */
 
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_cache_hit_pending_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_cache_hit_pending(void *arg)
 {
   int retval;
   int made_pending = 0;
@@ -544,7 +586,8 @@ NS(test_main)(void *arg)
   strlcpy(cache_entry->address, TO_CONN(exitconn)->address,
           sizeof(cache_entry->address));
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_cache_hit_pending_router_my_exit_policy_is_reject_star);
 
   dns_init();
 
@@ -562,7 +605,7 @@ NS(test_main)(void *arg)
   tt_assert(pending_conn->conn == exitconn);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(router_my_exit_policy_is_reject_star);
   tor_free(on_circ);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(cache_entry->pending_connections);
@@ -571,16 +614,12 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, cache_hit_cached)
-
 /* Given that a finished DNS resolve is available in our cache, we want
  * dns_resolve_impl() return it to called via resolve_out and pass the
  * handling to set_exitconn_info_from_resolve function.
  */
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_cache_hit_cached_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
@@ -589,7 +628,8 @@ static edge_connection_t *last_exitconn = NULL;
 static cached_resolve_t *last_resolve = NULL;
 
 static int
-NS(set_exitconn_info_from_resolve)(edge_connection_t *exitconn,
+dns_impl_cache_hit_cached_set_exitconn_info_from_resolve(
+                                   edge_connection_t *exitconn,
                                    const cached_resolve_t *resolve,
                                    char **hostname_out)
 {
@@ -602,7 +642,7 @@ NS(set_exitconn_info_from_resolve)(edge_connection_t *exitconn,
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_cache_hit_cached(void *arg)
 {
   int retval;
   int made_pending = 0;
@@ -625,8 +665,10 @@ NS(test_main)(void *arg)
   strlcpy(cache_entry->address, TO_CONN(exitconn)->address,
           sizeof(cache_entry->address));
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
-  NS_MOCK(set_exitconn_info_from_resolve);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_cache_hit_cached_router_my_exit_policy_is_reject_star);
+  MOCK(set_exitconn_info_from_resolve,
+       dns_impl_cache_hit_cached_set_exitconn_info_from_resolve);
 
   dns_init();
 
@@ -643,8 +685,8 @@ NS(test_main)(void *arg)
   tt_assert(last_resolve == cache_entry);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
-  NS_UNMOCK(set_exitconn_info_from_resolve);
+  UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(set_exitconn_info_from_resolve);
   tor_free(on_circ);
   tor_free(TO_CONN(exitconn)->address);
   tor_free(cache_entry->pending_connections);
@@ -652,10 +694,6 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE ASPECT(resolve_impl, cache_miss)
-
 /* Given that there are neither pending nor pre-cached resolve for a given
  * address, we want dns_resolve_impl() to create a new cached_resolve_t
  * object, mark it as pending, insert it into the cache, attach the exit
@@ -663,7 +701,7 @@ NS(test_main)(void *arg)
  * with the cached_resolve_t object it created.
  */
 static int
-NS(router_my_exit_policy_is_reject_star)(void)
+dns_impl_cache_miss_router_my_exit_policy_is_reject_star(void)
 {
   return 0;
 }
@@ -671,7 +709,7 @@ NS(router_my_exit_policy_is_reject_star)(void)
 static cached_resolve_t *last_launched_resolve = NULL;
 
 static int
-NS(launch_resolve)(cached_resolve_t *resolve)
+dns_impl_cache_miss_launch_resolve(cached_resolve_t *resolve)
 {
   last_launched_resolve = resolve;
 
@@ -679,7 +717,7 @@ NS(launch_resolve)(cached_resolve_t *resolve)
 }
 
 static void
-NS(test_main)(void *arg)
+test_dns_impl_cache_miss(void *arg)
 {
   int retval;
   int made_pending = 0;
@@ -698,8 +736,10 @@ NS(test_main)(void *arg)
 
   strlcpy(query.address, TO_CONN(exitconn)->address, sizeof(query.address));
 
-  NS_MOCK(router_my_exit_policy_is_reject_star);
-  NS_MOCK(launch_resolve);
+  MOCK(router_my_exit_policy_is_reject_star,
+       dns_impl_cache_miss_router_my_exit_policy_is_reject_star);
+  MOCK(launch_resolve,
+       dns_impl_cache_miss_launch_resolve);
 
   dns_init();
 
@@ -722,8 +762,8 @@ NS(test_main)(void *arg)
   tt_str_op(cache_entry->address,OP_EQ,TO_CONN(exitconn)->address);
 
   done:
-  NS_UNMOCK(router_my_exit_policy_is_reject_star);
-  NS_UNMOCK(launch_resolve);
+  UNMOCK(router_my_exit_policy_is_reject_star);
+  UNMOCK(launch_resolve);
   tor_free(on_circ);
   tor_free(TO_CONN(exitconn)->address);
   if (cache_entry)
@@ -733,19 +773,22 @@ NS(test_main)(void *arg)
   return;
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t dns_tests[] = {
-   TEST_CASE(clip_ttl),
-   TEST_CASE(resolve),
-   TEST_CASE_ASPECT(resolve_impl, addr_is_ip_no_need_to_resolve),
-   TEST_CASE_ASPECT(resolve_impl, non_exit),
-   TEST_CASE_ASPECT(resolve_impl, addr_is_invalid_dest),
-   TEST_CASE_ASPECT(resolve_impl, malformed_ptr),
-   TEST_CASE_ASPECT(resolve_impl, cache_hit_pending),
-   TEST_CASE_ASPECT(resolve_impl, cache_hit_cached),
-   TEST_CASE_ASPECT(resolve_impl, cache_miss),
+#ifdef HAVE_EVDNS_BASE_GET_NAMESERVER_ADDR
+   { "configure_ns_fallback", test_dns_configure_ns_fallback,
+     TT_FORK, NULL, NULL },
+#endif
+   { "clip_ttl", test_dns_clip_ttl, TT_FORK, NULL, NULL },
+   { "resolve", test_dns_resolve, TT_FORK, NULL, NULL },
+   { "impl_addr_is_ip", test_dns_impl_addr_is_ip, TT_FORK, NULL, NULL },
+   { "impl_non_exit", test_dns_impl_non_exit, TT_FORK, NULL, NULL },
+   { "impl_addr_is_invalid_dest", test_dns_impl_addr_is_invalid_dest,
+     TT_FORK, NULL, NULL },
+   { "impl_malformed_ptr", test_dns_impl_malformed_ptr, TT_FORK, NULL, NULL },
+   { "impl_cache_hit_pending", test_dns_impl_cache_hit_pending,
+     TT_FORK, NULL, NULL },
+   { "impl_cache_hit_cached", test_dns_impl_cache_hit_cached,
+     TT_FORK, NULL, NULL },
+   { "impl_cache_miss", test_dns_impl_cache_miss, TT_FORK, NULL, NULL },
    END_OF_TESTCASES
 };
-
-#undef NS_MODULE
diff --git a/src/test/test_dos.c b/src/test/test_dos.c
index 01d7cd006e..527e5bbe7f 100644
--- a/src/test/test_dos.c
+++ b/src/test/test_dos.c
@@ -1,8 +1,8 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define DOS_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITLIST_PRIVATE
 
 #include "core/or/or.h"
@@ -411,7 +411,7 @@ test_dos_bucket_refill(void *arg)
   }
   tt_uint_op(current_circ_count, OP_EQ, 0);
   tt_uint_op(dos_stats->cc_stats.circuit_bucket, OP_EQ, current_circ_count);
-#endif
+#endif /* SIZEOF_TIME_T == 8 */
 
  done:
   tor_free(chan);
diff --git a/src/test/test_entryconn.c b/src/test/test_entryconn.c
index fc7c5d5800..9cdd7f6d0e 100644
--- a/src/test/test_entryconn.c
+++ b/src/test/test_entryconn.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,7 +11,7 @@
 
 #include "feature/client/addressmap.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"
 #include "feature/nodelist/nodelist.h"
diff --git a/src/test/test_entrynodes.c b/src/test/test_entrynodes.c
index a486b13ae1..5ddd1a3db0 100644
--- a/src/test/test_entrynodes.c
+++ b/src/test/test_entrynodes.c
@@ -1,10 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CIRCUITLIST_PRIVATE
 #define CIRCUITBUILD_PRIVATE
+#define CONFIG_PRIVATE
 #define STATEFILE_PRIVATE
 #define ENTRYNODES_PRIVATE
 #define ROUTERLIST_PRIVATE
@@ -17,7 +18,7 @@
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/dircommon/directory.h"
 #include "feature/dirclient/dirclient.h"
@@ -67,7 +68,7 @@ static networkstatus_t *dummy_consensus = NULL;
 
 static smartlist_t *big_fake_net_nodes = NULL;
 
-static smartlist_t *
+static const smartlist_t *
 bfn_mock_nodelist_get_list(void)
 {
   return big_fake_net_nodes;
@@ -127,6 +128,9 @@ big_fake_network_cleanup(const struct testcase_t *testcase, void *ptr)
   return 1; /* NOP */
 }
 
+#define REASONABLY_FUTURE " reasonably-future"
+#define REASONABLY_PAST " reasonably-past"
+
 /* Unittest setup function: Setup a fake network. */
 static void *
 big_fake_network_setup(const struct testcase_t *testcase)
@@ -138,9 +142,10 @@ big_fake_network_setup(const struct testcase_t *testcase)
   const int N_NODES = 271;
 
   const char *argument = testcase->setup_data;
-  int reasonably_live_consensus = 0;
+  int reasonably_future_consensus = 0, reasonably_past_consensus = 0;
   if (argument) {
-    reasonably_live_consensus = strstr(argument, "reasonably-live") != NULL;
+    reasonably_future_consensus = strstr(argument, REASONABLY_FUTURE) != NULL;
+    reasonably_past_consensus = strstr(argument, REASONABLY_PAST) != NULL;
   }
 
   big_fake_net_nodes = smartlist_new();
@@ -193,16 +198,21 @@ big_fake_network_setup(const struct testcase_t *testcase)
       n->md->exit_policy = parse_short_policy("accept 443");
     }
 
+    n->nodelist_idx = smartlist_len(big_fake_net_nodes);
     smartlist_add(big_fake_net_nodes, n);
   }
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
   dummy_consensus = tor_malloc_zero(sizeof(networkstatus_t));
-  if (reasonably_live_consensus) {
-    /* Make the dummy consensus valid from 4 hours ago, but expired an hour
+  if (reasonably_future_consensus) {
+    /* Make the dummy consensus valid in 6 hours, and expiring in 7 hours. */
+    dummy_consensus->valid_after = approx_time() + 6*3600;
+    dummy_consensus->valid_until = approx_time() + 7*3600;
+  } else if (reasonably_past_consensus) {
+    /* Make the dummy consensus valid from 16 hours ago, but expired 12 hours
      * ago. */
-    dummy_consensus->valid_after = approx_time() - 4*3600;
-    dummy_consensus->valid_until = approx_time() - 3600;
+    dummy_consensus->valid_after = approx_time() - 16*3600;
+    dummy_consensus->valid_until = approx_time() - 12*3600;
   } else {
     /* Make the dummy consensus valid for an hour either side of now. */
     dummy_consensus->valid_after = approx_time() - 3600;
@@ -226,12 +236,12 @@ mock_randomize_time_no_randomization(time_t a, time_t b)
   return a;
 }
 
-static or_options_t mocked_options;
+static or_options_t *mocked_options;
 
 static const or_options_t *
 mock_get_options(void)
 {
-  return &mocked_options;
+  return mocked_options;
 }
 
 #define TEST_IPV4_ADDR "123.45.67.89"
@@ -250,7 +260,7 @@ test_node_preferred_orport(void *arg)
   tor_addr_port_t ap;
 
   /* Setup options */
-  memset(&mocked_options, 0, sizeof(mocked_options));
+  mocked_options = options_new();
   /* We don't test ClientPreferIPv6ORPort here, because it's used in
    * nodelist_set_consensus to setup node.ipv6_preferred, which we set
    * directly. */
@@ -273,8 +283,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're only using IPv4, regardless
    * of whether we prefer it or not */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 0;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 0;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -287,8 +297,8 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv4 if we're using IPv4 and IPv6, but
    * don't prefer the IPv6 address */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv4_addr));
@@ -296,28 +306,29 @@ test_node_preferred_orport(void *arg)
 
   /* Check the preferred address is IPv6 if we prefer it and
    * ClientUseIPv6 is 1, regardless of ClientUseIPv4 */
-  mocked_options.ClientUseIPv4 = 1;
-  mocked_options.ClientUseIPv6 = 1;
+  mocked_options->ClientUseIPv4 = 1;
+  mocked_options->ClientUseIPv6 = 1;
   node.ipv6_preferred = 1;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
-  mocked_options.ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv4 = 0;
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
   /* Check the preferred address is IPv6 if we don't prefer it, but
    * ClientUseIPv4 is 0 */
-  mocked_options.ClientUseIPv4 = 0;
-  mocked_options.ClientUseIPv6 = 1;
-  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(&mocked_options);
+  mocked_options->ClientUseIPv4 = 0;
+  mocked_options->ClientUseIPv6 = 1;
+  node.ipv6_preferred = fascist_firewall_prefer_ipv6_orport(mocked_options);
   node_get_pref_orport(&node, &ap);
   tt_assert(tor_addr_eq(&ap.addr, &ipv6_addr));
   tt_assert(ap.port == ipv6_port);
 
  done:
+  or_options_free(mocked_options);
   UNMOCK(get_options);
 }
 
@@ -379,12 +390,13 @@ test_entry_guard_encode_for_state_minimal(void *arg)
   eg->confirmed_idx = -1;
 
   char *s = NULL;
-  s = entry_guard_encode_for_state(eg);
+  s = entry_guard_encode_for_state(eg, 0);
 
   tt_str_op(s, OP_EQ,
             "in=wubwub "
             "rsa_id=706C75727079666C75727079736C75727079646F "
             "sampled_on=2016-11-14T00:00:00 "
+            "sampled_idx=0 "
             "listed=0");
 
  done:
@@ -410,10 +422,11 @@ test_entry_guard_encode_for_state_maximal(void *arg)
   eg->currently_listed = 1;
   eg->confirmed_on_date = 1479081690;
   eg->confirmed_idx = 333;
+  eg->sampled_idx = 42;
   eg->extra_state_fields = tor_strdup("and the green grass grew all around");
 
   char *s = NULL;
-  s = entry_guard_encode_for_state(eg);
+  s = entry_guard_encode_for_state(eg, 0);
 
   tt_str_op(s, OP_EQ,
             "in=default "
@@ -421,6 +434,7 @@ test_entry_guard_encode_for_state_maximal(void *arg)
             "bridge_addr=8.8.4.4:9999 "
             "nickname=Fred "
             "sampled_on=2016-11-14T00:00:00 "
+            "sampled_idx=0 "
             "sampled_by=1.2.3 "
             "unlisted_since=2016-11-14T00:00:45 "
             "listed=1 "
@@ -610,39 +624,47 @@ test_entry_guard_parse_from_state_full(void *arg)
   const char STATE[] =
   "Guard in=default rsa_id=214F44BD5B638E8C817D47FF7C97397790BF0345 "
     "nickname=TotallyNinja sampled_on=2016-11-12T19:32:49 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1\n"
   "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
     "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_idx=1 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
     "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
     "pb_successful_circuits_closed=2.000000\n"
   "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=2 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=4 "
     "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
     "pb_successful_circuits_closed=5.000000\n"
   "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1\n"
   "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
     "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_idx=3 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
     "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_idx=10 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
     "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev listed=1 "
     "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
     "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
     "pb_successful_circuits_closed=13.000000\n"
   "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
     "bridge_addr=37.218.246.143:28366 "
-    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n";
+    "sampled_on=2016-11-18T15:07:34 sampled_idx=1 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n";
 
   config_line_t *lines = NULL;
   or_state_t *state = tor_malloc_zero(sizeof(or_state_t));
@@ -718,35 +740,42 @@ test_entry_guard_parse_from_state_full(void *arg)
   tt_str_op(joined, OP_EQ,
   "Guard in=default rsa_id=052900AB0EA3ED54BAB84AE8A99E74E8693CE2B2 "
     "nickname=5OfNovember sampled_on=2016-11-20T04:32:05 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-22T08:13:28 confirmed_idx=0 "
     "pb_circ_attempts=4.000000 pb_circ_successes=2.000000 "
     "pb_successful_circuits_closed=2.000000\n"
   "Guard in=default rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=1 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1 confirmed_on=2016-11-24T08:45:30 confirmed_idx=1 "
     "pb_circ_attempts=5.000000 pb_circ_successes=5.000000 "
     "pb_successful_circuits_closed=5.000000\n"
   "Guard in=default rsa_id=E9025AD60D86875D5F11548D536CC6AF60F0EF5E "
     "nickname=maibrunn sampled_on=2016-11-25T22:36:38 "
+    "sampled_idx=2 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=default rsa_id=DCD30B90BA3A792DA75DC54A327EF353FB84C38E "
     "nickname=Unnamed sampled_on=2016-11-25T14:34:00 "
+    "sampled_idx=3 "
     "sampled_by=0.3.0.0-alpha-dev listed=1\n"
   "Guard in=wobblesome rsa_id=7B700C0C207EBD0002E00F499BE265519AC3C25A "
     "nickname=dc6jgk11 sampled_on=2016-11-28T11:50:13 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev "
     "listed=1\n"
   "Guard in=bridges rsa_id=8FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF2E "
     "bridge_addr=24.1.1.1:443 sampled_on=2016-11-25T06:44:14 "
+    "sampled_idx=0 "
     "sampled_by=0.3.0.0-alpha-dev listed=1 "
     "confirmed_on=2016-11-29T10:36:06 confirmed_idx=0 "
     "pb_circ_attempts=8.000000 pb_circ_successes=8.000000 "
     "pb_successful_circuits_closed=13.000000\n"
   "Guard in=bridges rsa_id=5800000000000000000000000000000000000000 "
     "bridge_addr=37.218.246.143:28366 "
-    "sampled_on=2016-11-18T15:07:34 sampled_by=0.3.0.0-alpha-dev listed=1\n");
+    "sampled_on=2016-11-18T15:07:34 sampled_idx=1 "
+    "sampled_by=0.3.0.0-alpha-dev listed=1\n");
 
  done:
   config_free_lines(lines);
@@ -1450,8 +1479,8 @@ test_entry_guard_confirming_guards(void *arg)
   tt_i64_op(g1->confirmed_on_date, OP_EQ, start+10);
   tt_i64_op(g2->confirmed_on_date, OP_EQ, start);
   tt_i64_op(g3->confirmed_on_date, OP_EQ, start+10);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g2);
   tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
 
   /* Now make sure we can regenerate the confirmed_entry_guards list. */
@@ -1463,8 +1492,8 @@ test_entry_guard_confirming_guards(void *arg)
   tt_int_op(g1->confirmed_idx, OP_EQ, 1);
   tt_int_op(g2->confirmed_idx, OP_EQ, 0);
   tt_int_op(g3->confirmed_idx, OP_EQ, 2);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g2);
-  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 0), OP_EQ, g1);
+  tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 1), OP_EQ, g2);
   tt_ptr_op(smartlist_get(gs->confirmed_entry_guards, 2), OP_EQ, g3);
 
   /* Now make sure we can regenerate the confirmed_entry_guards list if
@@ -1481,9 +1510,9 @@ test_entry_guard_confirming_guards(void *arg)
   g1 = smartlist_get(gs->confirmed_entry_guards, 0);
   g2 = smartlist_get(gs->confirmed_entry_guards, 1);
   g3 = smartlist_get(gs->confirmed_entry_guards, 2);
-  tt_int_op(g1->confirmed_idx, OP_EQ, 0);
-  tt_int_op(g2->confirmed_idx, OP_EQ, 1);
-  tt_int_op(g3->confirmed_idx, OP_EQ, 2);
+  tt_int_op(g1->sampled_idx, OP_EQ, 0);
+  tt_int_op(g2->sampled_idx, OP_EQ, 1);
+  tt_int_op(g3->sampled_idx, OP_EQ, 8);
   tt_assert(g1 != g2);
   tt_assert(g1 != g3);
   tt_assert(g2 != g3);
@@ -1499,9 +1528,6 @@ test_entry_guard_sample_reachable_filtered(void *arg)
   (void)arg;
   guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
   entry_guards_expand_sample(gs);
-  const int N = 10000;
-  bitarray_t *selected = NULL;
-  int i, j;
 
   /* We've got a sampled list now; let's make one non-usable-filtered; some
    * confirmed, some primary, some pending.
@@ -1536,32 +1562,21 @@ test_entry_guard_sample_reachable_filtered(void *arg)
     { SAMPLE_EXCLUDE_PENDING, 0 },
     { -1, -1},
   };
-
+  int j;
   for (j = 0; tests[j].flag >= 0; ++j) {
-    selected = bitarray_init_zero(n_guards);
     const int excluded_flags = tests[j].flag;
     const int excluded_idx = tests[j].idx;
-    for (i = 0; i < N; ++i) {
-      g = sample_reachable_filtered_entry_guards(gs, NULL, excluded_flags);
-      tor_assert(g);
-      int pos = smartlist_pos(gs->sampled_entry_guards, g);
-      tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
-      tt_int_op(pos, OP_GE, 0);
-      tt_int_op(pos, OP_LT, n_guards);
-      bitarray_set(selected, pos);
-    }
-    for (i = 0; i < n_guards; ++i) {
-      const int should_be_set = (i != excluded_idx &&
-                                 i != 3); // filtered out.
-      tt_int_op(!!bitarray_is_set(selected, i), OP_EQ, should_be_set);
-    }
-    bitarray_free(selected);
-    selected = NULL;
+    g = first_reachable_filtered_entry_guard(gs, NULL, excluded_flags);
+    tor_assert(g);
+    int pos = smartlist_pos(gs->sampled_entry_guards, g);
+    tt_int_op(smartlist_len(gs->sampled_entry_guards), OP_EQ, n_guards);
+    const int should_be_set = (pos != excluded_idx &&
+                                 pos != 3); // filtered out.
+    tt_int_op(1, OP_EQ, should_be_set);
   }
 
  done:
   guard_selection_free(gs);
-  bitarray_free(selected);
 }
 
 static void
@@ -1573,7 +1588,7 @@ test_entry_guard_sample_reachable_filtered_empty(void *arg)
   SMARTLIST_FOREACH(big_fake_net_nodes, node_t *, n,
                     n->is_possible_guard = 0);
 
-  entry_guard_t *g = sample_reachable_filtered_entry_guards(gs, NULL, 0);
+  entry_guard_t *g = first_reachable_filtered_entry_guard(gs, NULL, 0);
   tt_ptr_op(g, OP_EQ, NULL);
 
  done:
@@ -1664,10 +1679,13 @@ test_entry_guard_manage_primary(void *arg)
     tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx));
   });
 
-  /* If we have one confirmed guard, that guards becomes the first primary
-   * guard, and the other primary guards get kept. */
+  /**
+   * If we have one confirmed guard, that guards becomes the first primary
+   * only if its sampled_idx is smaller
+   * */
 
-  /* find a non-primary guard... */
+  /* find a non-primary guard... it should have a sampled_idx higher than
+   * existing primary guards */
   entry_guard_t *confirmed = NULL;
   SMARTLIST_FOREACH(gs->sampled_entry_guards, entry_guard_t *, g, {
     if (! g->is_primary) {
@@ -1683,15 +1701,13 @@ test_entry_guard_manage_primary(void *arg)
   smartlist_add_all(prev_guards, gs->primary_entry_guards);
   entry_guards_update_primary(gs);
 
-  /*  and see what's primary now! */
+  /* the confirmed guard should be at the end of the primary list! Hopefully,
+   * one of the primary guards with a lower sampled_idx will confirm soon :)
+   * Doing this won't make the client switches between primaries depending on
+   * the order of confirming events */
   tt_int_op(smartlist_len(gs->primary_entry_guards), OP_EQ, n_primary);
-  tt_ptr_op(smartlist_get(gs->primary_entry_guards, 0), OP_EQ, confirmed);
-  SMARTLIST_FOREACH(gs->primary_entry_guards, entry_guard_t *, g, {
-    tt_assert(g->is_primary);
-    if (g_sl_idx == 0)
-      continue;
-    tt_ptr_op(g, OP_EQ, smartlist_get(prev_guards, g_sl_idx - 1));
-  });
+  tt_ptr_op(smartlist_get(gs->primary_entry_guards,
+        smartlist_len(gs->primary_entry_guards)-1), OP_EQ, confirmed);
   {
     entry_guard_t *prev_last_guard = smartlist_get(prev_guards, n_primary-1);
     tt_assert(! prev_last_guard->is_primary);
@@ -1782,6 +1798,57 @@ test_entry_guard_guard_preferred(void *arg)
 }
 
 static void
+test_entry_guard_correct_cascading_order(void *arg)
+{
+  (void)arg;
+  smartlist_t *old_primary_guards = smartlist_new();
+  guard_selection_t *gs = guard_selection_new("default", GS_TYPE_NORMAL);
+  entry_guards_expand_sample(gs);
+  /** First, a test in which the primary guards need be pulled from different
+   * lists to fill up the primary list -- this may happen, if for example, not
+   * enough guards have confirmed yet */
+  entry_guard_t *g;
+  /** just one confirmed */
+  g = smartlist_get(gs->sampled_entry_guards, 2);
+  make_guard_confirmed(gs, g);
+  entry_guards_update_primary(gs);
+  g = smartlist_get(gs->primary_entry_guards, 0);
+  tt_int_op(g->sampled_idx, OP_EQ, 0);
+  g = smartlist_get(gs->primary_entry_guards, 1);
+  tt_int_op(g->sampled_idx, OP_EQ, 1);
+  g = smartlist_get(gs->primary_entry_guards, 2);
+  tt_int_op(g->sampled_idx, OP_EQ, 2);
+
+  /** Now the primaries get all confirmed, and the primary list should not
+   * change */
+  make_guard_confirmed(gs, smartlist_get(gs->primary_entry_guards, 0));
+  make_guard_confirmed(gs, smartlist_get(gs->primary_entry_guards, 1));
+  smartlist_add_all(old_primary_guards, gs->primary_entry_guards);
+  entry_guards_update_primary(gs);
+  smartlist_ptrs_eq(gs->primary_entry_guards, old_primary_guards);
+  /** the confirmed guards should also have the same set of guards, in the same
+   * order :-) */
+  smartlist_ptrs_eq(gs->confirmed_entry_guards, gs->primary_entry_guards);
+  /** Now select a guard for a circuit, and make sure it is the first primary
+   * guard */
+  unsigned state = 9999;
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  tt_ptr_op(g, OP_EQ, smartlist_get(gs->primary_entry_guards, 0));
+  /** Now, let's mark this guard as unreachable and let's update the lists */
+  g->is_reachable = GUARD_REACHABLE_NO;
+  g->failing_since = approx_time() - 10;
+  g->last_tried_to_connect = approx_time() - 10;
+  state = 9999;
+  entry_guards_update_primary(gs);
+  g = select_entry_guard_for_circuit(gs, GUARD_USAGE_TRAFFIC, NULL, &state);
+  /** we should have switched to the next one is sampled order */
+  tt_int_op(g->sampled_idx, OP_EQ, 1);
+ done:
+  smartlist_free(old_primary_guards);
+  guard_selection_free(gs);
+}
+
+static void
 test_entry_guard_select_for_circuit_no_confirmed(void *arg)
 {
   /* Simpler cases: no gaurds are confirmed yet. */
@@ -3028,6 +3095,7 @@ static const struct testcase_setup_t upgrade_circuits = {
   upgrade_circuits_setup, upgrade_circuits_cleanup
 };
 
+#ifndef COCCI
 #define NO_PREFIX_TEST(name) \
   { #name, test_ ## name, 0, NULL, NULL }
 
@@ -3039,13 +3107,18 @@ static const struct testcase_setup_t upgrade_circuits = {
 
 #define BFN_TEST(name) \
   EN_TEST_BASE(name, TT_FORK, &big_fake_network, NULL), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &big_fake_network, (void*)("reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &big_fake_network, (void*)(REASONABLY_PAST) }
 
 #define UPGRADE_TEST(name, arg) \
   EN_TEST_BASE(name, TT_FORK, &upgrade_circuits, arg), \
-  { #name "_reasonably_live", test_entry_guard_ ## name, TT_FORK, \
-    &upgrade_circuits, (void*)(arg " reasonably-live") }
+  { #name "_reasonably_future", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_FUTURE) }, \
+  { #name "_reasonably_past", test_entry_guard_ ## name, TT_FORK, \
+    &upgrade_circuits, (void*)(arg REASONABLY_PAST) }
+#endif /* !defined(COCCI) */
 
 struct testcase_t entrynodes_tests[] = {
   NO_PREFIX_TEST(node_preferred_orport),
@@ -3077,6 +3150,7 @@ struct testcase_t entrynodes_tests[] = {
   BFN_TEST(sample_reachable_filtered_empty),
   BFN_TEST(retry_unreachable),
   BFN_TEST(manage_primary),
+  BFN_TEST(correct_cascading_order),
 
   EN_TEST_FORK(guard_preferred),
 
diff --git a/src/test/test_extorport.c b/src/test/test_extorport.c
index 0c34d37a71..7935530653 100644
--- a/src/test/test_extorport.c
+++ b/src/test/test_extorport.c
@@ -1,15 +1,15 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONNECTION_PRIVATE
 #define EXT_ORPORT_PRIVATE
 #define MAINLOOP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"
-#include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "feature/relay/ext_orport.h"
 #include "core/mainloop/mainloop.h"
@@ -18,6 +18,7 @@
 
 #include "test/test.h"
 #include "test/test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -176,7 +177,7 @@ test_ext_or_init_auth(void *arg)
   /* Shouldn't be initialized already, or our tests will be a bit
    * meaningless */
   ext_or_auth_cookie = tor_malloc_zero(32);
-  tt_assert(tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Now make sure we use a temporary file */
   fn = get_fname("ext_cookie_file");
@@ -201,7 +202,7 @@ test_ext_or_init_auth(void *arg)
   tt_mem_op(cp,OP_EQ, "! Extended ORPort Auth Cookie !\x0a", 32);
   tt_mem_op(cp+32,OP_EQ, ext_or_auth_cookie, 32);
   memcpy(cookie0, ext_or_auth_cookie, 32);
-  tt_assert(!tor_mem_is_zero((char*)ext_or_auth_cookie, 32));
+  tt_assert(!fast_mem_is_zero((char*)ext_or_auth_cookie, 32));
 
   /* Operation should be idempotent. */
   tt_int_op(0, OP_EQ, init_ext_or_cookie_authentication(1));
@@ -303,16 +304,6 @@ test_ext_or_cookie_auth(void *arg)
 }
 
 static void
-crypto_rand_return_tse_str(char *to, size_t n)
-{
-  if (n != 32) {
-    TT_FAIL(("Asked for %d bytes, not 32", (int)n));
-    return;
-  }
-  memcpy(to, "te road There is always another ", 32);
-}
-
-static void
 test_ext_or_cookie_auth_testvec(void *arg)
 {
   char *reply=NULL, *client_hash=NULL;
@@ -326,7 +317,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
   memcpy(ext_or_auth_cookie, "Gliding wrapt in a brown mantle," , 32);
   ext_or_auth_cookie_is_set = 1;
 
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
 
   tt_int_op(0, OP_EQ,
             handle_client_auth_nonce(client_nonce, 32, &client_hash, &reply,
@@ -351,7 +342,7 @@ test_ext_or_cookie_auth_testvec(void *arg)
                  "33b3cd77ff79bd80c2074bbf438119a2");
 
  done:
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tor_free(reply);
   tor_free(client_hash);
   tor_free(mem_op_hex_tmp);
@@ -414,9 +405,9 @@ do_ext_or_handshake(or_connection_t *conn)
   CONTAINS("\x01\x00", 2);
   WRITE("\x01", 1);
   WRITE("But when I look ahead up the whi", 32);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   tt_int_op(TO_CONN(conn)->state, OP_EQ,
             EXT_OR_CONN_STATE_AUTH_WAIT_CLIENT_HASH);
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
@@ -481,9 +472,9 @@ test_ext_or_handshake(void *arg)
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
   /* send the rest of the nonce. */
   WRITE("ahead up the whi", 16);
-  MOCK(crypto_rand, crypto_rand_return_tse_str);
+  testing_enable_prefilled_rng("te road There is always another ", 32);
   tt_int_op(0, OP_EQ, connection_ext_or_process_inbuf(conn));
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   /* We should get the right reply from the server. */
   CONTAINS("\xec\x80\xed\x6e\x54\x6d\x3b\x36\xfd\xfc\x22\xfe\x13\x15\x41\x6b"
            "\x02\x9f\x1a\xde\x76\x10\xd9\x10\x87\x8b\x62\xee\xb7\x40\x38\x21"
@@ -582,7 +573,7 @@ test_ext_or_handshake(void *arg)
 
  done:
   UNMOCK(connection_write_to_buf_impl_);
-  UNMOCK(crypto_rand);
+  testing_disable_prefilled_rng();
   if (conn)
     connection_free_minimal(TO_CONN(conn));
 #undef CONTAINS
@@ -596,6 +587,6 @@ struct testcase_t extorport_tests[] = {
   { "cookie_auth", test_ext_or_cookie_auth, TT_FORK, NULL, NULL },
   { "cookie_auth_testvec", test_ext_or_cookie_auth_testvec, TT_FORK,
     NULL, NULL },
-  { "handshake", test_ext_or_handshake, TT_FORK, NULL, NULL },
+  { "handshake", test_ext_or_handshake, TT_FORK, &helper_pubsub_setup, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_geoip.c b/src/test/test_geoip.c
index 16c566bdbc..bf9932c169 100644
--- a/src/test/test_geoip.c
+++ b/src/test/test_geoip.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_guardfraction.c b/src/test/test_guardfraction.c
index ac8bfbfded..00c200e0fd 100644
--- a/src/test/test_guardfraction.c
+++ b/src/test/test_guardfraction.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define GUARDFRACTION_PRIVATE
diff --git a/src/test/test_handles.c b/src/test/test_handles.c
index 7f1d6e1898..dbb5b1a18e 100644
--- a/src/test/test_handles.c
+++ b/src/test/test_handles.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_helpers.c b/src/test/test_helpers.c
index 802d0a9ebe..f31c28b24d 100644
--- a/src/test/test_helpers.c
+++ b/src/test/test_helpers.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -9,25 +9,33 @@
 #define ROUTERLIST_PRIVATE
 #define CONFIG_PRIVATE
 #define CONNECTION_PRIVATE
+#define CONNECTION_OR_PRIVATE
 #define MAINLOOP_PRIVATE
 
 #include "orconfig.h"
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
+#include "app/main/subsysmgr.h"
 #include "core/mainloop/connection.h"
+#include "core/or/connection_or.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/mainloop/mainloop.h"
 #include "feature/nodelist/nodelist.h"
 #include "core/or/relay.h"
 #include "feature/nodelist/routerlist.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_connect.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
 
 #include "core/or/cell_st.h"
 #include "core/or/connection_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
 #include "feature/nodelist/routerlist_st.h"
@@ -37,14 +45,14 @@
 #include "test/test_connection.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
 #include "test_descriptors.inc"
 #include "core/or/circuitlist.h"
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-ENABLE_GCC_WARNING(overlength-strings)
+ENABLE_GCC_WARNING("-Woverlength-strings")
 #endif
 
 /* Return a statically allocated string representing yesterday's date
@@ -78,7 +86,7 @@ helper_setup_fake_routerlist(void)
 {
   int retval;
   routerlist_t *our_routerlist = NULL;
-  smartlist_t *our_nodelist = NULL;
+  const smartlist_t *our_nodelist = NULL;
 
   /* Read the file that contains our test descriptors. */
 
@@ -189,6 +197,14 @@ fake_close_socket(tor_socket_t sock)
   return 0;
 }
 
+/* Helper for test_conn_get_proxy_or_connection() */
+void
+mock_connection_or_change_state(or_connection_t *conn, uint8_t state)
+{
+  tor_assert(conn);
+  conn->base_.state = state;
+}
+
 static int mock_connection_connect_sockaddr_called = 0;
 static int fake_socket_number = TEST_CONN_FD_INIT;
 
@@ -223,6 +239,77 @@ mock_connection_connect_sockaddr(connection_t *conn,
   return 1;
 }
 
+or_connection_t *
+test_conn_get_proxy_or_connection(unsigned int proxy_type)
+{
+  or_connection_t *conn = NULL;
+  tor_addr_t dst_addr;
+  tor_addr_t proxy_addr;
+  int socket_err = 0;
+  int in_progress = 0;
+
+  MOCK(connection_connect_sockaddr,
+       mock_connection_connect_sockaddr);
+  MOCK(connection_write_to_buf_impl_,
+       connection_write_to_buf_mock);
+  MOCK(connection_or_change_state,
+       mock_connection_or_change_state);
+  MOCK(tor_close_socket, fake_close_socket);
+
+  tor_init_connection_lists();
+
+  conn = or_connection_new(CONN_TYPE_OR, TEST_CONN_FAMILY);
+  tt_assert(conn);
+
+  /* Set up a destination address. */
+  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS, TEST_CONN_FAMILY,
+                               &dst_addr);
+  tt_assert(!tor_addr_is_null(&dst_addr));
+
+  conn->proxy_type = proxy_type;
+  conn->base_.proxy_state = PROXY_INFANT;
+
+  tor_addr_copy_tight(&conn->base_.addr, &dst_addr);
+  conn->base_.address = tor_addr_to_str_dup(&dst_addr);
+  conn->base_.port = TEST_CONN_PORT;
+
+  /* Set up a proxy address. */
+  test_conn_lookup_addr_helper(TEST_CONN_ADDRESS_2, TEST_CONN_FAMILY,
+                               &proxy_addr);
+  tt_assert(!tor_addr_is_null(&proxy_addr));
+
+  conn->base_.state = OR_CONN_STATE_CONNECTING;
+
+  mock_connection_connect_sockaddr_called = 0;
+  in_progress = connection_connect(TO_CONN(conn), TEST_CONN_ADDRESS_PORT,
+                                   &proxy_addr, TEST_CONN_PORT, &socket_err);
+  tt_int_op(mock_connection_connect_sockaddr_called, OP_EQ, 1);
+  tt_assert(!socket_err);
+  tt_assert(in_progress == 0 || in_progress == 1);
+
+  assert_connection_ok(TO_CONN(conn), time(NULL));
+
+  in_progress = connection_or_finished_connecting(conn);
+  tt_int_op(in_progress, OP_EQ, 0);
+
+  assert_connection_ok(TO_CONN(conn), time(NULL));
+
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(connection_or_change_state);
+  UNMOCK(tor_close_socket);
+  return conn;
+
+  /* On failure */
+ done:
+  UNMOCK(connection_connect_sockaddr);
+  UNMOCK(connection_write_to_buf_impl_);
+  UNMOCK(connection_or_change_state);
+  UNMOCK(tor_close_socket);
+  connection_free_(TO_CONN(conn));
+  return NULL;
+}
+
 /** Create and return a new connection/stream */
 connection_t *
 test_conn_get_connection(uint8_t state, uint8_t type, uint8_t purpose)
@@ -290,7 +377,7 @@ helper_parse_options(const char *conf)
   if (ret != 0) {
     goto done;
   }
-  ret = config_assign(&options_format, opt, line, 0, &msg);
+  ret = config_assign(get_options_mgr(), opt, line, 0, &msg);
   if (ret != 0) {
     goto done;
   }
@@ -303,3 +390,54 @@ helper_parse_options(const char *conf)
   }
   return opt;
 }
+
+/**
+ * Dispatch alertfn callback: flush all messages right now. Implements
+ * DELIV_IMMEDIATE.
+ **/
+static void
+alertfn_immediate(dispatch_t *d, channel_id_t chan, void *arg)
+{
+  (void) arg;
+  dispatch_flush(d, chan, INT_MAX);
+}
+
+/**
+ * Setup helper for tests that need pubsub active
+ *
+ * Does not hook up mainloop events.  Does set immediate delivery for
+ * all channels.
+ */
+void *
+helper_setup_pubsub(const struct testcase_t *testcase)
+{
+  dispatch_t *dispatcher = NULL;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  channel_id_t chan = get_channel_id("orconn");
+
+  (void)testcase;
+  (void)subsystems_add_pubsub(builder);
+  dispatcher = pubsub_builder_finalize(builder, NULL);
+  tor_assert(dispatcher);
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  chan = get_channel_id("ocirc");
+  dispatch_set_alert_fn(dispatcher, chan, alertfn_immediate, NULL);
+  return dispatcher;
+}
+
+/**
+ * Cleanup helper for tests that need pubsub active
+ */
+int
+helper_cleanup_pubsub(const struct testcase_t *testcase, void *dispatcher_)
+{
+  dispatch_t *dispatcher = dispatcher_;
+
+  (void)testcase;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+const struct testcase_setup_t helper_pubsub_setup = {
+  helper_setup_pubsub, helper_cleanup_pubsub
+};
diff --git a/src/test/test_helpers.h b/src/test/test_helpers.h
index 9e376a563d..eaf18e19e2 100644
--- a/src/test/test_helpers.h
+++ b/src/test/test_helpers.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TOR_TEST_HELPERS_H
@@ -7,6 +7,7 @@
 #define BUFFERS_PRIVATE
 
 #include "core/or/or.h"
+#include "tinytest.h"
 
 const char *get_yesterday_date_str(void);
 
@@ -25,11 +26,19 @@ char *buf_get_contents(buf_t *buf, size_t *sz_out);
 int mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
                                             uint16_t family, tor_addr_t *out);
 
+void mock_connection_or_change_state(or_connection_t *conn, uint8_t state);
+
+or_connection_t *test_conn_get_proxy_or_connection(unsigned int proxy_type);
 connection_t *test_conn_get_connection(uint8_t state,
                                        uint8_t type, uint8_t purpose);
 or_options_t *helper_parse_options(const char *conf);
 
 extern const char TEST_DESCRIPTORS[];
 
+void *helper_setup_pubsub(const struct testcase_t *);
+int helper_cleanup_pubsub(const struct testcase_t *, void *);
+
+extern const struct testcase_setup_t helper_pubsub_setup;
+
 #endif /* !defined(TOR_TEST_HELPERS_H) */
 
diff --git a/src/test/test_hs.c b/src/test/test_hs.c
index a611b46ca6..46b4493a3d 100644
--- a/src/test/test_hs.c
+++ b/src/test/test_hs.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,7 +6,7 @@
  * \brief Unit tests for hidden service.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define RENDCOMMON_PRIVATE
 #define RENDSERVICE_PRIVATE
@@ -15,6 +15,8 @@
 #include "core/or/or.h"
 #include "test/test.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_fmt.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
 #include "feature/rend/rendcommon.h"
@@ -321,6 +323,16 @@ test_hs_desc_event(void *arg)
   tt_str_op(received_msg,OP_EQ, expected_msg);
   tor_free(received_msg);
 
+  /* test HSDir rate limited */
+  rend_query.auth_type = REND_NO_AUTH;
+  control_event_hsv2_descriptor_failed(&rend_query.base_, NULL,
+                                     "QUERY_RATE_LIMITED");
+  expected_msg = "650 HS_DESC FAILED "STR_HS_ADDR" NO_AUTH " \
+                 "UNKNOWN REASON=QUERY_RATE_LIMITED\r\n";
+  tt_assert(received_msg);
+  tt_str_op(received_msg,OP_EQ, expected_msg);
+  tor_free(received_msg);
+
   /* Test invalid content with no HSDir fingerprint. */
   char *exp_msg;
   control_event_hs_descriptor_content(rend_query.onion_address,
@@ -436,7 +448,7 @@ test_hs_rend_data(void *arg)
   tt_int_op(client_v2->auth_type, OP_EQ, REND_BASIC_AUTH);
   tt_int_op(strlen(client_v2->onion_address), OP_EQ, 0);
   tt_mem_op(client_v2->desc_id_fetch, OP_EQ, desc_id, sizeof(desc_id));
-  tt_int_op(tor_mem_is_zero(client_v2->descriptor_cookie,
+  tt_int_op(fast_mem_is_zero(client_v2->descriptor_cookie,
                             sizeof(client_v2->descriptor_cookie)), OP_EQ, 1);
   tt_assert(client->hsdirs_fp);
   tt_int_op(smartlist_len(client->hsdirs_fp), OP_EQ, 0);
diff --git a/src/test/test_hs_cache.c b/src/test/test_hs_cache.c
index 742fa349d9..df96b2c791 100644
--- a/src/test/test_hs_cache.c
+++ b/src/test/test_hs_cache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,6 +10,7 @@
 #define DIRCACHE_PRIVATE
 #define DIRCLIENT_PRIVATE
 #define HS_CACHE_PRIVATE
+#define CHANNEL_OBJECT_PRIVATE
 
 #include "trunnel/ed25519_cert.h"
 #include "feature/hs/hs_cache.h"
@@ -19,8 +20,14 @@
 #include "feature/nodelist/networkstatus.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_http.h"
+#include "core/or/circuitlist.h"
+#include "core/or/channel.h"
 #include "lib/crypt_ops/crypto_format.h"
+#include "lib/crypt_ops/crypto_rand.h"
 
+#include "core/or/edge_connection_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/or_connection_st.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 
@@ -232,22 +239,31 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
   /* The dir conn we are going to simulate */
   dir_connection_t *conn = NULL;
+  edge_connection_t *edge_conn = NULL;
+  or_circuit_t *or_circ = NULL;
 
   /* First extract the blinded public key that we are going to use in our
      query, and then build the actual query string. */
   {
     char hsdir_cache_key[ED25519_BASE64_LEN+1];
 
-    retval = ed25519_public_to_base64(hsdir_cache_key,
-                                      blinded_key);
-    tt_int_op(retval, OP_EQ, 0);
+    ed25519_public_to_base64(hsdir_cache_key, blinded_key);
     tor_asprintf(&hsdir_query_str, GET("/tor/hs/3/%s"), hsdir_cache_key);
   }
 
   /* Simulate an HTTP GET request to the HSDir */
   conn = dir_connection_new(AF_INET);
+  tt_assert(conn);
+  TO_CONN(conn)->linked = 1; /* Signal that it is encrypted. */
   tor_addr_from_ipv4h(&conn->base_.addr, 0x7f000001);
-  TO_CONN(conn)->linked = 1;/* Pretend the conn is encrypted :) */
+
+  /* Pretend this conn is anonymous. */
+  edge_conn = edge_connection_new(CONN_TYPE_EXIT, AF_INET);
+  TO_CONN(conn)->linked_conn = TO_CONN(edge_conn);
+  or_circ = or_circuit_new(0, NULL);
+  or_circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+  edge_conn->on_circuit = TO_CIRCUIT(or_circ);
+
   retval = directory_handle_command_get(conn, hsdir_query_str,
                                         NULL, 0);
   tt_int_op(retval, OP_EQ, 0);
@@ -264,8 +280,11 @@ helper_fetch_desc_from_hsdir(const ed25519_public_key_t *blinded_key)
 
  done:
   tor_free(hsdir_query_str);
-  if (conn)
+  if (conn) {
+    tor_free(or_circ->p_chan);
+    connection_free_minimal(TO_CONN(conn)->linked_conn);
     connection_free_minimal(TO_CONN(conn));
+  }
 
   return received_desc;
 }
@@ -351,7 +370,7 @@ test_hsdir_revision_counter_check(void *arg)
   hs_descriptor_t *published_desc = NULL;
   char *published_desc_str = NULL;
 
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
   char *received_desc_str = NULL;
   hs_descriptor_t *received_desc = NULL;
 
@@ -388,12 +407,12 @@ test_hsdir_revision_counter_check(void *arg)
     const ed25519_public_key_t *blinded_key;
 
     blinded_key = &published_desc->plaintext_data.blinded_pubkey;
-    hs_get_subcredential(&signing_kp.pubkey, blinded_key, subcredential);
+    hs_get_subcredential(&signing_kp.pubkey, blinded_key, &subcredential);
     received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
 
     retval = hs_desc_decode_descriptor(received_desc_str,
-                                       subcredential, NULL, &received_desc);
-    tt_int_op(retval, OP_EQ, 0);
+                                       &subcredential, NULL, &received_desc);
+    tt_int_op(retval, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(received_desc);
 
     /* Check that the revision counter is correct */
@@ -425,8 +444,8 @@ test_hsdir_revision_counter_check(void *arg)
     received_desc_str = helper_fetch_desc_from_hsdir(blinded_key);
 
     retval = hs_desc_decode_descriptor(received_desc_str,
-                                       subcredential, NULL, &received_desc);
-    tt_int_op(retval, OP_EQ, 0);
+                                       &subcredential, NULL, &received_desc);
+    tt_int_op(retval, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(received_desc);
 
     /* Check that the revision counter is the latest */
@@ -458,7 +477,7 @@ test_client_cache(void *arg)
   ed25519_keypair_t signing_kp;
   hs_descriptor_t *published_desc = NULL;
   char *published_desc_str = NULL;
-  uint8_t wanted_subcredential[DIGEST256_LEN];
+  hs_subcredential_t wanted_subcredential;
   response_handler_args_t *args = NULL;
   dir_connection_t *conn = NULL;
 
@@ -487,8 +506,10 @@ test_client_cache(void *arg)
     retval = hs_desc_encode_descriptor(published_desc, &signing_kp,
                                        NULL, &published_desc_str);
     tt_int_op(retval, OP_EQ, 0);
-    memcpy(wanted_subcredential, published_desc->subcredential, DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)wanted_subcredential, DIGEST256_LEN));
+    memcpy(&wanted_subcredential, &published_desc->subcredential,
+           sizeof(hs_subcredential_t));
+    tt_assert(!fast_mem_is_zero((char*)wanted_subcredential.subcred,
+                                DIGEST256_LEN));
   }
 
   /* Test handle_response_fetch_hsdesc_v3() */
@@ -522,8 +543,9 @@ test_client_cache(void *arg)
     const hs_descriptor_t *cached_desc = NULL;
     cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
     tt_assert(cached_desc);
-    tt_mem_op(cached_desc->subcredential, OP_EQ, wanted_subcredential,
-              DIGEST256_LEN);
+    tt_mem_op(cached_desc->subcredential.subcred,
+              OP_EQ, wanted_subcredential.subcred,
+              SUBCRED_LEN);
   }
 
   /* Progress time to next TP and check that desc was cleaned */
@@ -550,6 +572,136 @@ test_client_cache(void *arg)
   }
 }
 
+/** Test that we can store HS descriptors in the client HS cache. */
+static void
+test_client_cache_decrypt(void *arg)
+{
+  int ret;
+  char *desc_encoded = NULL;
+  uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+  curve25519_keypair_t client_kp;
+  ed25519_keypair_t service_kp;
+  hs_descriptor_t *desc = NULL;
+  const hs_descriptor_t *search_desc;
+  const char *search_desc_encoded;
+
+  (void) arg;
+
+  /* Initialize HSDir cache subsystem */
+  hs_init();
+
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate a valid descriptor with normal values. */
+  {
+    ret = ed25519_keypair_generate(&service_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    ret = curve25519_keypair_generate(&client_kp, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    crypto_rand((char *) descriptor_cookie, sizeof(descriptor_cookie));
+
+    desc = hs_helper_build_hs_desc_with_client_auth(descriptor_cookie,
+                                                    &client_kp.pubkey,
+                                                    &service_kp);
+    tt_assert(desc);
+    ret = hs_desc_encode_descriptor(desc, &service_kp, descriptor_cookie,
+                                    &desc_encoded);
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  /* Put it in the cache. Should not be decrypted since the client
+   * authorization creds were not added to the global map. */
+  ret = hs_cache_store_as_client(desc_encoded, &service_kp.pubkey);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
+
+  /* We should not be able to decrypt anything. */
+  ret = hs_cache_client_new_auth_parse(&service_kp.pubkey);
+  tt_int_op(ret, OP_EQ, false);
+
+  /* Add client auth to global map. */
+  hs_helper_add_client_auth(&service_kp.pubkey, &client_kp.seckey);
+
+  /* We should not be able to decrypt anything. */
+  ret = hs_cache_client_new_auth_parse(&service_kp.pubkey);
+  tt_int_op(ret, OP_EQ, true);
+
+  /* Lookup the cache to make sure it is usable and there. */
+  search_desc = hs_cache_lookup_as_client(&service_kp.pubkey);
+  tt_assert(search_desc);
+  search_desc_encoded = hs_cache_lookup_encoded_as_client(&service_kp.pubkey);
+  tt_mem_op(search_desc_encoded, OP_EQ, desc_encoded, strlen(desc_encoded));
+
+ done:
+  hs_descriptor_free(desc);
+  tor_free(desc_encoded);
+
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+}
+
+static void
+test_client_cache_remove(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  /* Set consensus time. Lookup will not return the entry if it has expired
+   * and it is checked against the consensus valid_after time. */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Build a descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* Remove the cached entry. */
+  hs_cache_remove_as_client(&service_kp.pubkey);
+  tt_assert(!hs_cache_lookup_as_client(&service_kp.pubkey));
+
+ done:
+  hs_descriptor_free(desc1);
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+}
+
 struct testcase_t hs_cache[] = {
   /* Encoding tests. */
   { "directory", test_directory, TT_FORK,
@@ -562,6 +714,10 @@ struct testcase_t hs_cache[] = {
     NULL, NULL },
   { "client_cache", test_client_cache, TT_FORK,
     NULL, NULL },
+  { "client_cache_decrypt", test_client_cache_decrypt, TT_FORK,
+    NULL, NULL },
+  { "client_cache_remove", test_client_cache_remove, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_cell.c b/src/test/test_hs_cell.c
index f8af631c8b..5406339276 100644
--- a/src/test/test_hs_cell.c
+++ b/src/test/test_hs_cell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,6 +20,7 @@
 #include "feature/hs/hs_service.h"
 
 /* Trunnel. */
+#include "trunnel/hs/cell_common.h"
 #include "trunnel/hs/cell_establish_intro.h"
 
 /** We simulate the creation of an outgoing ESTABLISH_INTRO cell, and then we
@@ -38,11 +39,13 @@ test_gen_establish_intro_cell(void *arg)
   /* Create outgoing ESTABLISH_INTRO cell and extract its payload so that we
      attempt to parse it. */
   {
+    hs_service_config_t config;
+    memset(&config, 0, sizeof(config));
     /* We only need the auth key pair here. */
-    hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+    hs_service_intro_point_t *ip = service_intro_point_new(NULL);
     /* Auth key pair is generated in the constructor so we are all set for
      * using this IP object. */
-    ret = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+    ret = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
     service_intro_point_free(ip);
     tt_u64_op(ret, OP_GT, 0);
   }
@@ -97,6 +100,9 @@ test_gen_establish_intro_cell_bad(void *arg)
   trn_cell_establish_intro_t *cell = NULL;
   char circ_nonce[DIGEST_LEN] = {0};
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   MOCK(ed25519_sign_prefixed, mock_ed25519_sign_prefixed);
 
@@ -107,8 +113,8 @@ test_gen_establish_intro_cell_bad(void *arg)
      ed25519_sign_prefixed() function and make it fail. */
   cell = trn_cell_establish_intro_new();
   tt_assert(cell);
-  ip = service_intro_point_new(NULL, 0, 0);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, NULL);
+  ip = service_intro_point_new(NULL);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, NULL);
   service_intro_point_free(ip);
   expect_log_msg_containing("Unable to make signature for "
                             "ESTABLISH_INTRO cell.");
@@ -120,11 +126,97 @@ test_gen_establish_intro_cell_bad(void *arg)
   UNMOCK(ed25519_sign_prefixed);
 }
 
+static void
+test_gen_establish_intro_dos_ext(void *arg)
+{
+  ssize_t ret;
+  hs_service_config_t config;
+  hs_service_intro_point_t *ip = NULL;
+  trn_cell_extension_t *extensions = NULL;
+  trn_cell_extension_dos_t *dos = NULL;
+
+  (void) arg;
+
+  memset(&config, 0, sizeof(config));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+
+  /* Case 1: No DoS parameters so no extension to be built. */
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 0);
+  trn_cell_extension_free(extensions);
+  extensions = NULL;
+
+  /* Case 2: Enable the DoS extension. Parameter set to 0 should indicate to
+   * disable the defense on the intro point but there should be an extension
+   * nonetheless in the cell. */
+  config.has_dos_defense_enabled = 1;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  const trn_cell_extension_field_t *field =
+    trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  const trn_cell_extension_dos_param_t *param =
+    trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 0);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+  /* Case 3: Enable the DoS extension. Parameter set to some normal values. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 42;
+  config.intro_dos_burst_per_sec = 250;
+  extensions = build_establish_intro_extensions(&config, ip);
+  tt_int_op(trn_cell_extension_get_num(extensions), OP_EQ, 1);
+  /* Validate the extension. */
+  field = trn_cell_extension_getconst_fields(extensions, 0);
+  tt_int_op(trn_cell_extension_field_get_field_type(field), OP_EQ,
+            TRUNNEL_CELL_EXTENSION_TYPE_DOS);
+  ret = trn_cell_extension_dos_parse(&dos,
+                 trn_cell_extension_field_getconstarray_field(field),
+                 trn_cell_extension_field_getlen_field(field));
+  tt_int_op(ret, OP_EQ, 19);
+  /* Rate per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 0);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 42);
+  /* Burst per sec param. */
+  param = trn_cell_extension_dos_getconst_params(dos, 1);
+  tt_int_op(trn_cell_extension_dos_param_get_type(param), OP_EQ,
+            TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC);
+  tt_u64_op(trn_cell_extension_dos_param_get_value(param), OP_EQ, 250);
+  trn_cell_extension_dos_free(dos); dos = NULL;
+  trn_cell_extension_free(extensions); extensions = NULL;
+
+ done:
+  service_intro_point_free(ip);
+  trn_cell_extension_dos_free(dos);
+  trn_cell_extension_free(extensions);
+}
+
 struct testcase_t hs_cell_tests[] = {
   { "gen_establish_intro_cell", test_gen_establish_intro_cell, TT_FORK,
     NULL, NULL },
   { "gen_establish_intro_cell_bad", test_gen_establish_intro_cell_bad, TT_FORK,
     NULL, NULL },
+  { "gen_establish_intro_dos_ext", test_gen_establish_intro_dos_ext, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_client.c b/src/test/test_hs_client.c
index 53ee3c53d2..0cd7d81eea 100644
--- a/src/test/test_hs_client.c
+++ b/src/test/test_hs_client.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -10,10 +10,11 @@
 #define CRYPTO_PRIVATE
 #define MAINLOOP_PRIVATE
 #define HS_CLIENT_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define CIRCUITBUILD_PRIVATE
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -24,6 +25,7 @@
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_dh.h"
+#include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/channeltls.h"
 #include "feature/dircommon/directory.h"
 #include "core/mainloop/mainloop.h"
@@ -36,6 +38,7 @@
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_ident.h"
 #include "feature/hs/hs_cache.h"
+#include "feature/rend/rendcache.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitbuild.h"
 #include "core/mainloop/connection.h"
@@ -44,6 +47,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/dircommon/dir_connection_st.h"
 #include "core/or/entry_connection_st.h"
 #include "core/or/extend_info_st.h"
@@ -78,6 +82,23 @@ mock_networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
 }
 
 static int
+mock_write_str_to_file(const char *path, const char *str, int bin)
+{
+  (void) bin;
+  (void) path;
+  (void) str;
+  return 0;
+}
+
+static or_options_t mocked_options;
+
+static const or_options_t *
+mock_get_options(void)
+{
+  return &mocked_options;
+}
+
+static int
 helper_config_client(const char *conf, int validate_only)
 {
   int ret = 0;
@@ -91,6 +112,24 @@ helper_config_client(const char *conf, int validate_only)
   return ret;
 }
 
+static void
+helper_add_random_client_auth(const ed25519_public_key_t *service_pk)
+{
+  char *conf = NULL;
+#define conf_fmt "ClientOnionAuthDir %s\n"
+  tor_asprintf(&conf, conf_fmt, get_fname("auth_keys"));
+#undef conf_fmt
+  helper_config_client(conf, 0);
+  tor_free(conf);
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  hs_client_service_authorization_t *auth =
+    tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  curve25519_secret_key_generate(&auth->enc_seckey, 0);
+  hs_build_address(service_pk, HS_VERSION_THREE, auth->onion_address);
+  digest256map_set(client_auths, service_pk->pubkey, auth);
+}
+
 /* Test helper function: Setup a circuit and a stream with the same hidden
  * service destination, and put them in <b>circ_out</b> and
  * <b>conn_out</b>. Make the stream wait for circuits to be established to the
@@ -159,8 +198,7 @@ helper_get_circ_and_stream_for_test(origin_circuit_t **circ_out,
     or_circ->rend_data = rend_data_dup(conn_rend_data);
   } else {
     /* prop224: Setup hs ident on the circuit */
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
   }
 
   TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
@@ -243,12 +281,14 @@ test_e2e_rend_circuit_setup_legacy(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(
+         crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA1);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -285,7 +325,7 @@ test_e2e_rend_circuit_setup(void *arg)
        mock_connection_ap_handshake_send_begin);
 
   /* Setup */
-  retval = helper_get_circ_and_stream_for_test( &or_circ, &conn, 0);
+  retval = helper_get_circ_and_stream_for_test(&or_circ, &conn, 0);
   tt_int_op(retval, OP_EQ, 0);
   tt_assert(or_circ);
   tt_assert(conn);
@@ -301,9 +341,8 @@ test_e2e_rend_circuit_setup(void *arg)
   /**********************************************/
 
   /* Setup the circuit */
-  retval = hs_circuit_setup_e2e_rend_circ(or_circ,
-                                          ntor_key_seed, sizeof(ntor_key_seed),
-                                          0);
+  retval = hs_circuit_setup_e2e_rend_circ(or_circ, ntor_key_seed,
+                                          sizeof(ntor_key_seed), 0);
   tt_int_op(retval, OP_EQ, 0);
 
   /**********************************************/
@@ -313,12 +352,12 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check that the crypt path has prop224 algorithm parameters */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_C_REND_JOINED);
@@ -389,15 +428,17 @@ test_client_pick_intro(void *arg)
     tt_assert(encoded);
 
     /* store it */
-    hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
 
     /* fetch it to make sure it works */
     const hs_descriptor_t *fetched_desc =
       hs_cache_lookup_as_client(&service_kp.pubkey);
     tt_assert(fetched_desc);
-    tt_mem_op(fetched_desc->subcredential, OP_EQ, desc->subcredential,
-              DIGEST256_LEN);
-    tt_assert(!tor_mem_is_zero((char*)fetched_desc->subcredential,
+    tt_mem_op(fetched_desc->subcredential.subcred,
+              OP_EQ, desc->subcredential.subcred,
+              SUBCRED_LEN);
+    tt_assert(!fast_mem_is_zero((char*)fetched_desc->subcredential.subcred,
                                DIGEST256_LEN));
     tor_free(encoded);
   }
@@ -405,6 +446,9 @@ test_client_pick_intro(void *arg)
   /* 2) Mark all intro points except _the chosen one_ as failed. Then query the
    *   desc and get a random intro: check that we got _the chosen one_. */
   {
+    /* Tell hs_get_extend_info_from_lspecs() to skip the private address check.
+     */
+    get_options_mutable()->ExtendAllowPrivateAddresses = 1;
     /* Pick the chosen intro point and get its ei */
     hs_desc_intro_point_t *chosen_intro_point =
       smartlist_get(desc->encrypted_data.intro_points, 0);
@@ -432,7 +476,7 @@ test_client_pick_intro(void *arg)
     for (int i = 0; i < 64; ++i) {
       extend_info_t *ip = client_get_random_intro(&service_kp.pubkey);
       tor_assert(ip);
-      tt_assert(!tor_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
+      tt_assert(!fast_mem_is_zero((char*)ip->identity_digest, DIGEST_LEN));
       tt_mem_op(ip->identity_digest, OP_EQ, chosen_intro_ei->identity_digest,
                 DIGEST_LEN);
       extend_info_free(ip);
@@ -478,6 +522,18 @@ test_client_pick_intro(void *arg)
     SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
                             hs_desc_intro_point_t *, ip) {
       extend_info_t *intro_ei = desc_intro_point_to_extend_info(ip);
+      /* desc_intro_point_to_extend_info() doesn't return IPv6 intro points
+       * yet, because we can't extend to them. See #24404, #24451, and #24181.
+       */
+      if (intro_ei == NULL) {
+        /* Pretend we're making a direct connection, and that we can use IPv6
+         */
+        get_options_mutable()->ClientUseIPv6 = 1;
+        intro_ei = hs_get_extend_info_from_lspecs(ip->link_specifiers,
+                                                  &ip->onion_key, 1);
+        tt_assert(tor_addr_family(&intro_ei->addr) == AF_INET6);
+      }
+      tt_assert(intro_ei);
       if (intro_ei) {
         const char *ptr;
         char ip_addr[TOR_ADDR_BUF_LEN];
@@ -531,6 +587,17 @@ mock_connection_mark_unattached_ap_(entry_connection_t *conn, int endreason,
 }
 
 static void
+mock_connection_mark_unattached_ap_no_close(entry_connection_t *conn,
+                                            int endreason, int line,
+                                            const char *file)
+{
+  (void) conn;
+  (void) endreason;
+  (void) line;
+  (void) file;
+}
+
+static void
 test_descriptor_fetch(void *arg)
 {
   int ret;
@@ -668,6 +735,10 @@ test_parse_auth_file_content(void *arg)
   /* Bigger key than it should be */
   tt_assert(!parse_auth_file_content("xx:descriptor:x25519:"
                      "vjqea4jbhwwc4hto7ekyvqfbeodghbaq6nxi45hz4wr3qvhqv3yqa"));
+  /* All-zeroes key */
+  tt_assert(!parse_auth_file_content("xx:descriptor:x25519:"
+            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+
  done:
   tor_free(auth);
 }
@@ -805,6 +876,7 @@ test_desc_has_arrived_cleanup(void *arg)
   ed25519_keypair_t signing_kp;
   entry_connection_t *socks1 = NULL, *socks2 = NULL;
   hs_ident_dir_conn_t hs_dir_ident;
+  dir_connection_t *dir_conn = NULL;
 
   (void) arg;
 
@@ -833,7 +905,7 @@ test_desc_has_arrived_cleanup(void *arg)
 
   /* Store in the client cache. */
   ret = hs_cache_store_as_client(desc_str, &signing_kp.pubkey);
-  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
   cached_desc = hs_cache_lookup_as_client(&signing_kp.pubkey);
   tt_assert(cached_desc);
   hs_helper_desc_equal(desc, cached_desc);
@@ -861,9 +933,11 @@ test_desc_has_arrived_cleanup(void *arg)
    * SOCKS connection to be ended with a resolved failed. */
   hs_ident_dir_conn_init(&signing_kp.pubkey,
                          &desc->plaintext_data.blinded_pubkey, &hs_dir_ident);
-  hs_client_desc_has_arrived(&hs_dir_ident);
+  dir_conn = dir_connection_new(AF_INET);
+  dir_conn->hs_ident = hs_ident_dir_conn_dup(&hs_dir_ident);
+  hs_client_dir_fetch_done(dir_conn, "A reason", desc_str, 200);
+  connection_free_minimal(TO_CONN(dir_conn));
   tt_int_op(socks1->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
-  /* XXX: MUST work with OP_EQ. */
   tt_int_op(socks2->edge_.end_reason, OP_EQ, END_STREAM_REASON_RESOLVEFAILED);
 
   /* Now let say tor cleans up the intro state cache which resets all intro
@@ -872,7 +946,6 @@ test_desc_has_arrived_cleanup(void *arg)
 
   /* Retrying all SOCKS which should basically do nothing since we don't have
    * any pending SOCKS connection in AP_CONN_STATE_RENDDESC_WAIT state. */
-  /* XXX: BUG() is triggered here, shouldn't if socks2 wasn't alive. */
   retry_all_socks_conn_waiting_for_desc();
 
  done:
@@ -899,6 +972,7 @@ test_close_intro_circuits_new_desc(void *arg)
   (void) arg;
 
   hs_init();
+  rend_cache_init();
 
   /* This is needed because of the client cache expiration timestamp is based
    * on having a consensus. See cached_client_descriptor_has_expired(). */
@@ -923,6 +997,51 @@ test_close_intro_circuits_new_desc(void *arg)
   circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
   ocirc = TO_ORIGIN_CIRCUIT(circ);
 
+  /* Build a descriptor _without_ client authorization and thus not
+   * decryptable. Make sure the close circuit code path is not triggered. */
+  {
+    char *desc_encoded = NULL;
+    uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+    curve25519_keypair_t client_kp;
+    hs_descriptor_t *desc = NULL;
+
+    tt_int_op(0, OP_EQ, curve25519_keypair_generate(&client_kp, 0));
+    crypto_rand((char *) descriptor_cookie, sizeof(descriptor_cookie));
+
+    desc = hs_helper_build_hs_desc_with_client_auth(descriptor_cookie,
+                                                    &client_kp.pubkey,
+                                                    &service_kp);
+    tt_assert(desc);
+    ret = hs_desc_encode_descriptor(desc, &service_kp, descriptor_cookie,
+                                    &desc_encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Associate descriptor intro key with the dummy circuit. */
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+    hs_descriptor_free(desc);
+    tt_assert(desc_encoded);
+    /* Put it in the cache. Should not be decrypted since the client
+     * authorization creds were not added to the global map. */
+    ret = hs_cache_store_as_client(desc_encoded, &service_kp.pubkey);
+    tor_free(desc_encoded);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
+
+    /* Clean cache with a future timestamp. It will trigger the clean up and
+     * attempt to close the circuit but only if the descriptor is decryptable.
+     * Cache object should be removed and circuit untouched. */
+    hs_cache_clean_as_client(mock_ns.valid_after + (60 * 60 * 24));
+    tt_assert(!hs_cache_lookup_as_client(&service_kp.pubkey));
+
+    /* Make sure the circuit still there. */
+    tt_assert(circuit_get_next_intro_circ(NULL, true));
+    /* Get rid of the ident, it will be replaced in the next tests. */
+    hs_ident_circuit_free(ocirc->hs_ident);
+  }
+
   /* Build the first descriptor and cache it. */
   {
     char *encoded;
@@ -934,7 +1053,7 @@ test_close_intro_circuits_new_desc(void *arg)
 
     /* Store it */
     ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tor_free(encoded);
     tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
   }
@@ -944,8 +1063,7 @@ test_close_intro_circuits_new_desc(void *arg)
     const hs_desc_intro_point_t *ip =
       smartlist_get(desc1->encrypted_data.intro_points, 0);
     tt_assert(ip);
-    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey,
-                                           HS_IDENT_CIRCUIT_INTRO);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
     ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
                         &ip->auth_key_cert->signed_key);
   }
@@ -971,7 +1089,7 @@ test_close_intro_circuits_new_desc(void *arg)
     tt_assert(encoded);
 
     ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tor_free(encoded);
     tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
   }
@@ -988,6 +1106,451 @@ test_close_intro_circuits_new_desc(void *arg)
   UNMOCK(networkstatus_get_reasonably_live_consensus);
 }
 
+static void
+test_close_intro_circuits_cache_clean(void *arg)
+{
+  int ret;
+  ed25519_keypair_t service_kp;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  hs_descriptor_t *desc1 = NULL;
+
+  (void) arg;
+
+  hs_init();
+  rend_cache_init();
+
+  /* This is needed because of the client cache expiration timestamp is based
+   * on having a consensus. See cached_client_descriptor_has_expired(). */
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                     &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                     &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                     &mock_ns.valid_until);
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuits.
+   * We'll then make sure the hs_ident is attached to a dummy descriptor. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+
+  /* Build the first descriptor and cache it. */
+  {
+    char *encoded;
+    desc1 = hs_helper_build_hs_desc_with_ip(&service_kp);
+    tt_assert(desc1);
+    ret = hs_desc_encode_descriptor(desc1, &service_kp, NULL, &encoded);
+    tt_int_op(ret, OP_EQ, 0);
+    tt_assert(encoded);
+
+    /* Store it */
+    ret = hs_cache_store_as_client(encoded, &service_kp.pubkey);
+    tt_int_op(ret, OP_EQ, 0);
+    tor_free(encoded);
+    tt_assert(hs_cache_lookup_as_client(&service_kp.pubkey));
+  }
+
+  /* We'll pick one introduction point and associate it with the circuit. */
+  {
+    const hs_desc_intro_point_t *ip =
+      smartlist_get(desc1->encrypted_data.intro_points, 0);
+    tt_assert(ip);
+    ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+    ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk,
+                        &ip->auth_key_cert->signed_key);
+  }
+
+  /* Before we are about to clean up the intro circuits, make sure it is
+   * actually there. */
+  tt_assert(circuit_get_next_intro_circ(NULL, true));
+
+  /* Cleanup the client cache. The ns valid after time is what decides if the
+   * descriptor has expired so put it in the future enough (72h) so we are
+   * sure to always expire. */
+  mock_ns.valid_after = approx_time() + (72 * 24 * 60 * 60);
+  hs_cache_clean_as_client(0);
+
+  /* Once stored, our intro circuit should be closed because it is related to
+   * an old introduction point that doesn't exists anymore. */
+  tt_assert(!circuit_get_next_intro_circ(NULL, true));
+
+ done:
+  circuit_free(circ);
+  hs_descriptor_free(desc1);
+  hs_free_all();
+  rend_cache_free_all();
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+}
+
+static void
+test_socks_hs_errors(void *arg)
+{
+  int ret;
+  char digest[DIGEST_LEN];
+  char *desc_encoded = NULL;
+  circuit_t *circ = NULL;
+  origin_circuit_t *ocirc = NULL;
+  tor_addr_t addr;
+  ed25519_keypair_t service_kp;
+  ed25519_keypair_t signing_kp;
+  entry_connection_t *socks_conn = NULL;
+  dir_connection_t *dir_conn = NULL;
+  hs_descriptor_t *desc = NULL;
+  uint8_t descriptor_cookie[HS_DESC_DESCRIPTOR_COOKIE_LEN];
+
+  (void) arg;
+
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+  MOCK(connection_mark_unattached_ap_,
+       mock_connection_mark_unattached_ap_no_close);
+  MOCK(read_file_to_str, mock_read_file_to_str);
+  MOCK(tor_listdir, mock_tor_listdir);
+  MOCK(check_private_dir, mock_check_private_dir);
+
+    /* Set consensus time */
+  parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  parse_rfc1123_time("Sat, 26 Oct 1985 16:00:00 UTC",
+                           &mock_ns.valid_until);
+
+  hs_init();
+
+  ret = ed25519_keypair_generate(&service_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = ed25519_keypair_generate(&signing_kp, 0);
+  tt_int_op(ret, OP_EQ, 0);
+
+  socks_conn = helper_build_socks_connection(&service_kp.pubkey,
+                                             AP_CONN_STATE_RENDDESC_WAIT);
+  tt_assert(socks_conn);
+
+  /* Create directory connection. */
+  dir_conn = dir_connection_new(AF_INET);
+  dir_conn->hs_ident = tor_malloc_zero(sizeof(hs_ident_dir_conn_t));
+  TO_CONN(dir_conn)->purpose = DIR_PURPOSE_FETCH_HSDESC;
+  ed25519_pubkey_copy(&dir_conn->hs_ident->identity_pk, &service_kp.pubkey);
+
+  /* Encode descriptor so we can decode it. */
+  desc = hs_helper_build_hs_desc_with_ip(&service_kp);
+  tt_assert(desc);
+
+  /* Before testing the client authentication error code, encode the
+   * descriptor with no client auth. */
+  ret = hs_desc_encode_descriptor(desc, &service_kp, NULL, &desc_encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(desc_encoded);
+
+  /*
+   * Test the introduction failure codes (X'F2' and X'F7')
+   */
+
+  /* First, we have to put all the IPs in the failure cache. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                     &ip->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_GENERIC);
+  } SMARTLIST_FOREACH_END(ip);
+
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_INTRO_FAILED);
+
+  /* Purge client cache of the descriptor so we can go again. */
+  hs_cache_purge_as_client();
+
+  /* Second, set all failures to be time outs. */
+  SMARTLIST_FOREACH_BEGIN(desc->encrypted_data.intro_points,
+                          hs_desc_intro_point_t *, ip) {
+    hs_cache_client_intro_state_note(&service_kp.pubkey,
+                                     &ip->auth_key_cert->signed_key,
+                                     INTRO_POINT_FAILURE_TIMEOUT);
+  } SMARTLIST_FOREACH_END(ip);
+
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_INTRO_TIMEDOUT);
+
+  /* Purge client cache of the descriptor so we can go again. */
+  hs_cache_purge_as_client();
+
+  /*
+   * Test the rendezvous failure codes (X'F3')
+   */
+
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_REND_READY;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  /* Attach socks connection to this rendezvous circuit. */
+  ocirc->p_streams = ENTRY_TO_EDGE_CONN(socks_conn);
+  /* Trigger the rendezvous failure. Timeout the circuit and free. */
+  circuit_mark_for_close(circ, END_CIRC_REASON_TIMEOUT);
+
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_REND_FAILED);
+
+  /*
+   * Test client authorization codes.
+   */
+
+  tor_free(desc_encoded);
+  crypto_rand((char *) descriptor_cookie, sizeof(descriptor_cookie));
+  ret = hs_desc_encode_descriptor(desc, &service_kp, descriptor_cookie,
+                                  &desc_encoded);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_assert(desc_encoded);
+
+  /* Try decoding. Point this to an existing descriptor. The following should
+   * fail thus the desc_out should be set to NULL. */
+  hs_descriptor_t *desc_out = desc;
+  ret = hs_client_decode_descriptor(desc_encoded, &service_kp.pubkey,
+                                    &desc_out);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
+  tt_assert(desc_out == NULL);
+
+  /* The caching will fail to decrypt because the descriptor_cookie used above
+   * is not known to the HS subsystem. This will lead to a missing client
+   * auth. */
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_MISSING_CLIENT_AUTH);
+
+  /* Add in the global client auth list bad creds for this service. */
+  helper_add_random_client_auth(&service_kp.pubkey);
+
+  ret = hs_client_decode_descriptor(desc_encoded, &service_kp.pubkey,
+                                    &desc_out);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_BAD_CLIENT_AUTH);
+  tt_assert(desc_out == NULL);
+
+  /* Simmulate a fetch done again. This should replace the cached descriptor
+   * and signal a bad client authorization. */
+  hs_client_dir_fetch_done(dir_conn, "Reason", desc_encoded, 200);
+  tt_int_op(socks_conn->socks_request->socks_extended_error_code, OP_EQ,
+            SOCKS5_HS_BAD_CLIENT_AUTH);
+
+ done:
+  connection_free_minimal(ENTRY_TO_CONN(socks_conn));
+  connection_free_minimal(TO_CONN(dir_conn));
+  hs_descriptor_free(desc);
+  tor_free(desc_encoded);
+  circuit_free(circ);
+
+  hs_free_all();
+
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+  UNMOCK(connection_mark_unattached_ap_);
+  UNMOCK(read_file_to_str);
+  UNMOCK(tor_listdir);
+  UNMOCK(check_private_dir);
+}
+
+static void
+test_close_intro_circuit_failure(void *arg)
+{
+  char digest[DIGEST_LEN];
+  circuit_t *circ = NULL;
+  ed25519_keypair_t service_kp, intro_kp;
+  origin_circuit_t *ocirc = NULL;
+  tor_addr_t addr;
+  const hs_cache_intro_state_t *entry;
+
+  (void) arg;
+
+  hs_init();
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&intro_kp, 0));
+
+  /* Create and add to the global list a dummy client introduction circuit at
+   * the ACK WAIT state. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCE_ACK_WAIT;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  /* We'll make for close the circuit for a timeout failure. It should _NOT_
+   * end up in the failure cache just yet. We do that on free() only. */
+  circuit_mark_for_close(circ, END_CIRC_REASON_TIMEOUT);
+  tt_assert(!hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                              &intro_kp.pubkey));
+  /* Time to free. It should get removed. */
+  circuit_free(circ);
+  entry = hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                           &intro_kp.pubkey);
+  tt_assert(entry);
+  tt_uint_op(entry->timed_out, OP_EQ, 1);
+  hs_cache_client_intro_state_purge();
+
+  /* Again, create and add to the global list a dummy client introduction
+   * circuit at the INTRODUCING state. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  /* On free, we should get an unreachable failure. */
+  circuit_free(circ);
+  entry = hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                           &intro_kp.pubkey);
+  tt_assert(entry);
+  tt_uint_op(entry->unreachable_count, OP_EQ, 1);
+  hs_cache_client_intro_state_purge();
+
+  /* Again, create and add to the global list a dummy client introduction
+   * circuit at the INTRODUCING state but we'll close it for timeout. It
+   * should not be noted as a timeout failure. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ocirc->build_state = tor_malloc_zero(sizeof(cpath_build_state_t));
+  /* Code path will log this exit so build it. */
+  ocirc->build_state->chosen_exit = extend_info_new("TestNickname", digest,
+                                                    NULL, NULL, NULL, &addr,
+                                                    4242);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  circuit_mark_for_close(circ, END_CIRC_REASON_TIMEOUT);
+  circuit_free(circ);
+  tt_assert(!hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                              &intro_kp.pubkey));
+
+  /* Again, create and add to the global list a dummy client introduction
+   * circuit at the INTRODUCING state but without a chosen_exit. In theory, it
+   * can not happen but we'll make sure it doesn't end up in the failure cache
+   * anyway. */
+  circ = dummy_origin_circuit_new(0);
+  tt_assert(circ);
+  circ->purpose = CIRCUIT_PURPOSE_C_INTRODUCING;
+  ocirc = TO_ORIGIN_CIRCUIT(circ);
+  ocirc->hs_ident = hs_ident_circuit_new(&service_kp.pubkey);
+  ed25519_pubkey_copy(&ocirc->hs_ident->intro_auth_pk, &intro_kp.pubkey);
+
+  circuit_free(circ);
+  tt_assert(!hs_cache_client_intro_state_find(&service_kp.pubkey,
+                                              &intro_kp.pubkey));
+
+ done:
+  circuit_free(circ);
+  hs_free_all();
+}
+
+static void
+test_purge_ephemeral_client_auth(void *arg)
+{
+  ed25519_keypair_t service_kp;
+  hs_client_service_authorization_t *auth = NULL;
+  hs_client_register_auth_status_t status;
+
+  (void) arg;
+
+  /* We will try to write on disk client credentials. */
+  MOCK(check_private_dir, mock_check_private_dir);
+  MOCK(get_options, mock_get_options);
+  MOCK(write_str_to_file, mock_write_str_to_file);
+
+  /* Boggus directory so when we try to write the permanent client
+   * authorization data to disk, we don't fail. See
+   * store_permanent_client_auth_credentials() for more details. */
+  mocked_options.ClientOnionAuthDir = tor_strdup("auth_dir");
+
+  hs_init();
+
+  /* Generate service keypair */
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&service_kp, 0));
+
+  /* Generate a client authorization object. */
+  auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+
+  /* Set it up. No flags meaning it is ephemeral. */
+  curve25519_secret_key_generate(&auth->enc_seckey, 0);
+  hs_build_address(&service_kp.pubkey, HS_VERSION_THREE, auth->onion_address);
+  auth->flags = 0;
+
+  /* Confirm that there is nothing in the client auth map. It is unallocated
+   * until we add the first entry. */
+  tt_assert(!get_hs_client_auths_map());
+
+  /* Add an entry to the client auth list. We loose ownership of the auth
+   * object so nullify it. */
+  status = hs_client_register_auth_credentials(auth);
+  auth = NULL;
+  tt_int_op(status, OP_EQ, REGISTER_SUCCESS);
+
+  /* We should have the entry now. */
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(client_auths);
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 1);
+
+  /* Purge the cache that should remove all ephemeral values. */
+  purge_ephemeral_client_auth();
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 0);
+
+  /* Now add a new authorization object but permanent. */
+  /* Generate a client authorization object. */
+  auth = tor_malloc_zero(sizeof(hs_client_service_authorization_t));
+  curve25519_secret_key_generate(&auth->enc_seckey, 0);
+  hs_build_address(&service_kp.pubkey, HS_VERSION_THREE, auth->onion_address);
+  auth->flags = CLIENT_AUTH_FLAG_IS_PERMANENT;
+
+  /* Add an entry to the client auth list. We loose ownership of the auth
+   * object so nullify it. */
+  status = hs_client_register_auth_credentials(auth);
+  auth = NULL;
+  tt_int_op(status, OP_EQ, REGISTER_SUCCESS);
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 1);
+
+  /* Purge again, the entry should still be there. */
+  purge_ephemeral_client_auth();
+  tt_int_op(digest256map_size(client_auths), OP_EQ, 1);
+
+ done:
+  client_service_authorization_free(auth);
+  hs_free_all();
+  tor_free(mocked_options.ClientOnionAuthDir);
+
+  UNMOCK(check_private_dir);
+  UNMOCK(get_options);
+  UNMOCK(write_str_to_file);
+}
+
 struct testcase_t hs_client_tests[] = {
   { "e2e_rend_circuit_setup_legacy", test_e2e_rend_circuit_setup_legacy,
     TT_FORK, NULL, NULL },
@@ -1005,8 +1568,19 @@ struct testcase_t hs_client_tests[] = {
     TT_FORK, NULL, NULL },
   { "desc_has_arrived_cleanup", test_desc_has_arrived_cleanup,
     TT_FORK, NULL, NULL },
+  { "close_intro_circuit_failure", test_close_intro_circuit_failure,
+    TT_FORK, NULL, NULL },
   { "close_intro_circuits_new_desc", test_close_intro_circuits_new_desc,
     TT_FORK, NULL, NULL },
+  { "close_intro_circuits_cache_clean", test_close_intro_circuits_cache_clean,
+    TT_FORK, NULL, NULL },
+
+  /* SOCKS5 Extended Error Code. */
+  { "socks_hs_errors", test_socks_hs_errors, TT_FORK, NULL, NULL },
+
+  /* Client authorization. */
+  { "purge_ephemeral_client_auth", test_purge_ephemeral_client_auth, TT_FORK,
+    NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_common.c b/src/test/test_hs_common.c
index c6104301e1..e3d130fb32 100644
--- a/src/test/test_hs_common.c
+++ b/src/test/test_hs_common.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,6 +6,7 @@
  * \brief Test hidden service common functionalities.
  */
 
+#define CONNECTION_EDGE_PRIVATE
 #define HS_COMMON_PRIVATE
 #define HS_CLIENT_PRIVATE
 #define HS_SERVICE_PRIVATE
@@ -31,7 +32,7 @@
 #include "app/config/statefile.h"
 #include "core/or/circuitlist.h"
 #include "feature/dirauth/shared_random.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -52,14 +53,14 @@ test_validate_address(void *arg)
   setup_full_capture_of_logs(LOG_WARN);
   ret = hs_address_is_valid("blah");
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg_containing("has an invalid length");
+  expect_log_msg_containing("Invalid length");
   teardown_capture_of_logs();
 
   setup_full_capture_of_logs(LOG_WARN);
   ret = hs_address_is_valid(
            "p3xnclpu4mu22dwaurjtsybyqk4xfjmcfz6z62yl24uwmhjatiwnlnadb");
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg_containing("has an invalid length");
+  expect_log_msg_containing("Invalid length");
   teardown_capture_of_logs();
 
   /* Invalid checksum (taken from prop224) */
@@ -82,7 +83,7 @@ test_validate_address(void *arg)
   ret = hs_address_is_valid(
            "????????????????????????????????????????????????????????");
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg_containing("can't be decoded");
+  expect_log_msg_containing("Unable to base32 decode");
   teardown_capture_of_logs();
 
   /* Valid address. */
@@ -275,7 +276,7 @@ test_start_time_of_next_time_period(void *arg)
 static void
 cleanup_nodelist(void)
 {
-  smartlist_t *nodelist = nodelist_get_list();
+  const smartlist_t *nodelist = nodelist_get_list();
   SMARTLIST_FOREACH_BEGIN(nodelist, node_t *, node) {
     tor_free(node->md);
     node->md = NULL;
@@ -509,6 +510,7 @@ test_desc_reupload_logic(void *arg)
                 pubkey_hex, strlen(pubkey_hex));
   hs_build_address(&pubkey, HS_VERSION_THREE, onion_addr);
   service = tor_malloc_zero(sizeof(hs_service_t));
+  tt_assert(service);
   memcpy(service->onion_address, onion_addr, sizeof(service->onion_address));
   ed25519_secret_key_generate(&service->keys.identity_sk, 0);
   ed25519_public_key_generate(&service->keys.identity_pk,
@@ -610,6 +612,10 @@ test_desc_reupload_logic(void *arg)
   SMARTLIST_FOREACH(ns->routerstatus_list,
                     routerstatus_t *, rs, routerstatus_free(rs));
   smartlist_clear(ns->routerstatus_list);
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   networkstatus_vote_free(ns);
   cleanup_nodelist();
   hs_free_all();
@@ -637,7 +643,7 @@ test_disaster_srv(void *arg)
   get_disaster_srv(1, srv_one);
   get_disaster_srv(2, srv_two);
 
-  /* Check that the cached ones where updated */
+  /* Check that the cached ones were updated */
   tt_mem_op(cached_disaster_srv_one, OP_EQ, srv_one, DIGEST256_LEN);
   tt_mem_op(cached_disaster_srv_two, OP_EQ, srv_two, DIGEST256_LEN);
 
@@ -780,6 +786,7 @@ static void
 test_parse_extended_hostname(void *arg)
 {
   (void) arg;
+  hostname_type_t type;
 
   char address1[] = "fooaddress.onion";
   char address2[] = "aaaaaaaaaaaaaaaa.onion";
@@ -790,21 +797,48 @@ test_parse_extended_hostname(void *arg)
   char address7[] = ".abcdefghijklmnop.onion";
   char address8[] =
     "www.25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion";
+  char address9[] =
+    "www.15njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid.onion";
+  char address10[] =
+    "15njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid7jdl.onion";
+
+  tt_assert(!parse_extended_hostname(address1, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
+  tt_assert(parse_extended_hostname(address2, &type));
+  tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME);
+  tt_str_op(address2, OP_EQ, "aaaaaaaaaaaaaaaa");
+
+  tt_assert(parse_extended_hostname(address3, &type));
+  tt_int_op(type, OP_EQ, EXIT_HOSTNAME);
+
+  tt_assert(parse_extended_hostname(address4, &type));
+  tt_int_op(type, OP_EQ, NORMAL_HOSTNAME);
 
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address1));
-  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address2));
-  tt_str_op(address2,OP_EQ, "aaaaaaaaaaaaaaaa");
-  tt_assert(EXIT_HOSTNAME == parse_extended_hostname(address3));
-  tt_assert(NORMAL_HOSTNAME == parse_extended_hostname(address4));
-  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address5));
-  tt_str_op(address5,OP_EQ, "abcdefghijklmnop");
-  tt_assert(ONION_V2_HOSTNAME == parse_extended_hostname(address6));
-  tt_str_op(address6,OP_EQ, "abcdefghijklmnop");
-  tt_assert(BAD_HOSTNAME == parse_extended_hostname(address7));
-  tt_assert(ONION_V3_HOSTNAME == parse_extended_hostname(address8));
+  tt_assert(parse_extended_hostname(address5, &type));
+  tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME);
+  tt_str_op(address5, OP_EQ, "abcdefghijklmnop");
+
+  tt_assert(parse_extended_hostname(address6, &type));
+  tt_int_op(type, OP_EQ, ONION_V2_HOSTNAME);
+  tt_str_op(address6, OP_EQ, "abcdefghijklmnop");
+
+  tt_assert(!parse_extended_hostname(address7, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
+  tt_assert(parse_extended_hostname(address8, &type));
+  tt_int_op(type, OP_EQ, ONION_V3_HOSTNAME);
   tt_str_op(address8, OP_EQ,
             "25njqamcweflpvkl73j4szahhihoc4xt3ktcgjnpaingr5yhkenl5sid");
 
+  /* Invalid v3 address. */
+  tt_assert(!parse_extended_hostname(address9, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
+  /* Invalid v3 address: too long */
+  tt_assert(!parse_extended_hostname(address10, &type));
+  tt_int_op(type, OP_EQ, BAD_HOSTNAME);
+
  done: ;
 }
 
@@ -832,7 +866,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 01:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -840,7 +874,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 12:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -848,7 +882,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 1);
 
@@ -856,7 +890,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 27 Oct 1985 00:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 1);
 
@@ -864,7 +898,7 @@ test_time_between_tp_and_srv(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   ret = parse_rfc1123_time("Sat, 27 Oct 1985 01:00:00 UTC", &ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns.valid_after);
   ret = hs_in_period_between_tp_and_srv(&ns, 0);
   tt_int_op(ret, OP_EQ, 0);
 
@@ -1354,7 +1388,7 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario)
                       &mock_service_ns->valid_until);
   set_consensus_times(cfg->service_valid_until,
                       &mock_service_ns->fresh_until);
-  voting_schedule_recalculate_timing(get_options(),
+  dirauth_sched_recalculate_timing(get_options(),
                                      mock_service_ns->valid_after);
   /* Check that service is in the right time period point */
   tt_int_op(hs_in_period_between_tp_and_srv(mock_service_ns, 0), OP_EQ,
@@ -1367,7 +1401,7 @@ run_reachability_scenario(const reachability_cfg_t *cfg, int num_scenario)
                       &mock_client_ns->valid_until);
   set_consensus_times(cfg->client_valid_until,
                       &mock_client_ns->fresh_until);
-  voting_schedule_recalculate_timing(get_options(),
+  dirauth_sched_recalculate_timing(get_options(),
                                      mock_client_ns->valid_after);
   /* Check that client is in the right time period point */
   tt_int_op(hs_in_period_between_tp_and_srv(mock_client_ns, 0), OP_EQ,
@@ -1590,7 +1624,7 @@ helper_set_consensus_and_system_time(networkstatus_t *ns, int position)
   } else {
     tt_assert(0);
   }
-  voting_schedule_recalculate_timing(get_options(), ns->valid_after);
+  dirauth_sched_recalculate_timing(get_options(), ns->valid_after);
 
   /* Set system time: pretend to be just 2 minutes before consensus expiry */
   real_time = ns->valid_until - 120;
diff --git a/src/test/test_hs_config.c b/src/test/test_hs_config.c
index c2c556307d..dc3b598c34 100644
--- a/src/test/test_hs_config.c
+++ b/src/test/test_hs_config.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
 
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
@@ -61,8 +62,9 @@ test_invalid_service(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, 1);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceAllowUnknownPorts must be "
-                              "between 0 and 1, not 2");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceAllowUnknownPorts: Unrecognized "
+                              "value 2. Allowed values are 0 and 1.");
     teardown_capture_of_logs();
   }
 
@@ -75,8 +77,9 @@ test_invalid_service(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, 1);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceDirGroupReadable must be "
-                              "between 0 and 1, not 2");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceDirGroupReadable: "
+                              "Unrecognized value 2.");
     teardown_capture_of_logs();
   }
 
@@ -89,8 +92,9 @@ test_invalid_service(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, 1);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceMaxStreamsCloseCircuit must "
-                              "be between 0 and 1, not 2");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceMaxStreamsCloseCircuit: "
+                              "Unrecognized value 2");
     teardown_capture_of_logs();
   }
 
@@ -227,8 +231,8 @@ test_invalid_service_v2(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, validate_only);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
-                              "be between 0 and 10, not 11");
+    expect_log_msg_containing("HiddenServiceNumIntroductionPoints must "
+                              "be between 0 and 10, not 11.");
     teardown_capture_of_logs();
   }
 
@@ -242,8 +246,9 @@ test_invalid_service_v2(void *arg)
     setup_full_capture_of_logs(LOG_WARN);
     ret = helper_config_service(conf, validate_only);
     tt_int_op(ret, OP_EQ, -1);
-    expect_log_msg_containing("HiddenServiceNumIntroductionPoints should "
-                              "be between 0 and 10, not -1");
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceNumIntroductionPoints: "
+                              "Integer -1 is malformed or out of bounds.");
     teardown_capture_of_logs();
   }
 
@@ -272,6 +277,7 @@ test_valid_service_v2(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. Basic client authorization. */
   {
@@ -314,7 +320,7 @@ test_valid_service_v2(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -392,6 +398,7 @@ test_valid_service_v3(void *arg)
   int ret;
 
   (void) arg;
+  mock_hostname_resolver();
 
   /* Valid complex configuration. */
   {
@@ -448,7 +455,7 @@ test_valid_service_v3(void *arg)
   }
 
  done:
-  ;
+  unmock_hostname_resolver();
 }
 
 static void
@@ -489,6 +496,114 @@ test_staging_service_v3(void *arg)
   hs_free_all();
 }
 
+static void
+test_dos_parameters(void *arg)
+{
+  int ret;
+
+  (void) arg;
+
+  hs_init();
+
+  /* Valid configuration. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_INFO);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, 0);
+    expect_log_msg_containing("Service INTRO2 DoS defenses rate set to: 42");
+    expect_log_msg_containing("Service INTRO2 DoS defenses burst set to: 87");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid rate. Value of 2^37. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 137438953472\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 87\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceEnableIntroDoSRatePerSec: "
+                              "Integer 137438953472 is malformed or out of "
+                              "bounds.");
+    teardown_capture_of_logs();
+  }
+
+  /* Invalid burst. Value of 2^38. Max allowed is 2^31. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 274877906944\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceEnableIntroDoSBurstPerSec: "
+                              "Integer 274877906944 is malformed or out "
+                              "of bounds.");
+    teardown_capture_of_logs();
+  }
+
+  /* Burst is smaller than rate. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec 42\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 27\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Hidden service DoS defenses burst (27) can "
+                              "not be smaller than the rate value (42).");
+    teardown_capture_of_logs();
+  }
+
+  /* Negative value. */
+  {
+    const char *conf =
+      "HiddenServiceDir /tmp/tor-test-hs-RANDOM/hs3\n"
+      "HiddenServiceVersion 3\n"
+      "HiddenServicePort 22 1.1.1.1:22\n"
+      "HiddenServiceEnableIntroDoSDefense 1\n"
+      "HiddenServiceEnableIntroDoSRatePerSec -1\n"
+      "HiddenServiceEnableIntroDoSBurstPerSec 42\n";
+
+    setup_full_capture_of_logs(LOG_WARN);
+    ret = helper_config_service(conf, 0);
+    tt_int_op(ret, OP_EQ, -1);
+    expect_log_msg_containing("Could not parse "
+                              "HiddenServiceEnableIntroDoSRatePerSec: "
+                              "Integer -1 is malformed or out of bounds.");
+    teardown_capture_of_logs();
+  }
+
+ done:
+  hs_free_all();
+}
+
 struct testcase_t hs_config_tests[] = {
   /* Invalid service not specific to any version. */
   { "invalid_service", test_invalid_service, TT_FORK,
@@ -512,6 +627,9 @@ struct testcase_t hs_config_tests[] = {
   { "staging_service_v3", test_staging_service_v3, TT_FORK,
     NULL, NULL },
 
+  /* Test HS DoS parameters. */
+  { "dos_parameters", test_dos_parameters, TT_FORK,
+    NULL, NULL },
+
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_control.c b/src/test/test_hs_control.c
index ba67712f1b..80bbf547dc 100644
--- a/src/test/test_hs_control.c
+++ b/src/test/test_hs_control.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,13 +6,21 @@
  * \brief Unit tests for hidden service control port event and command.
  **/
 
-#define CONTROL_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define HS_CLIENT_PRIVATE
 
 #include "core/or/or.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
+#include "core/mainloop/connection.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
+#include "feature/control/control_cmd.h"
+#include "feature/control/control_fmt.h"
+#include "feature/control/control_connection_st.h"
 #include "app/config/config.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_client.h"
 #include "feature/hs/hs_control.h"
 #include "feature/nodelist/nodelist.h"
 
@@ -20,7 +28,16 @@
 #include "feature/nodelist/routerstatus_st.h"
 #include "lib/crypt_ops/crypto_format.h"
 
-#include "test/test_helpers.h"
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifdef _WIN32
+/* For mkdir() */
+#include <direct.h>
+#else
+#include <dirent.h>
+#endif /* defined(_WIN32) */
 
 /* mock ID digest and longname for node that's in nodelist */
 #define HSDIR_EXIST_ID \
@@ -105,8 +122,7 @@ test_hs_desc_event(void *arg)
   memset(&blinded_pk, 'B', sizeof(blinded_pk));
   memset(&hsdir_rs, 0, sizeof(hsdir_rs));
   memcpy(hsdir_rs.identity_digest, HSDIR_EXIST_ID, DIGEST_LEN);
-  ret = ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_public_to_base64(base64_blinded_pk, &blinded_pk);
   memcpy(&ident.identity_pk, &identity_kp.pubkey,
          sizeof(ed25519_public_key_t));
   memcpy(&ident.blinded_pk, &blinded_pk, sizeof(blinded_pk));
@@ -186,9 +202,552 @@ test_hs_desc_event(void *arg)
   tor_free(expected_msg);
 }
 
+/** Test that we can correctly add, remove and view client auth credentials
+ *  using the control port. */
+static void
+test_hs_control_good_onion_client_auth_add(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  ed25519_public_key_t service_identity_pk_2fv, service_identity_pk_jt4,
+                       service_identity_pk_jam;
+  control_connection_t conn;
+  char *args = NULL;
+  char *cp1 = NULL;
+  size_t sz;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_ADD");
+  }
+
+  { /* Setup the services */
+    retval = hs_parse_address(
+                 "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd",
+                 &service_identity_pk_2fv,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = hs_parse_address(
+                 "jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd",
+                 &service_identity_pk_jt4,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+
+    retval = hs_parse_address(
+                 "jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd",
+                 &service_identity_pk_jam,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+  /* Register first service */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ= ");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+           "x25519:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA= DropSound=No");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd "
+           "x25519:FCV0c0ELDKKDpSFgVIB8Yow8Evj5iD+GoiTtK878NkQ= "
+           "ClientName=MeganNicole ");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  client_auths = get_hs_client_auths_map();
+  tt_assert(client_auths);
+  tt_uint_op(digest256map_size(client_auths), OP_EQ, 3);
+
+  hs_client_service_authorization_t *client_2fv =
+    digest256map_get(client_auths, service_identity_pk_2fv.pubkey);
+  tt_assert(client_2fv);
+  tt_int_op(client_2fv->flags, OP_EQ, 0);
+
+  hs_client_service_authorization_t *client_jt4 =
+    digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(client_jt4);
+  tt_int_op(client_jt4->flags, OP_EQ, 0);
+
+  hs_client_service_authorization_t *client_jam =
+    digest256map_get(client_auths, service_identity_pk_jam.pubkey);
+  tt_assert(client_jam);
+  tt_int_op(client_jam->flags, OP_EQ, 0);
+
+  /* Now let's VIEW the auth credentials */
+  tor_free(conn.current_cmd);
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_VIEW");
+
+  /* First go with no arguments, so that we view all the credentials */
+  tor_free(args);
+  args = tor_strdup("");
+
+#define VIEW_CORRECT_REPLY_NO_ADDR "250-ONION_CLIENT_AUTH_VIEW\r\n"   \
+  "250-CLIENT 2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "  \
+    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ=\r\n"   \
+  "250-CLIENT jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd " \
+    "x25519:FCV0c0ELDKKDpSFgVIB8Yow8Evj5iD+GoiTtK878NkQ= " \
+    "ClientName=MeganNicole\r\n" \
+  "250-CLIENT jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd " \
+    "x25519:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA=\r\n"             \
+  "250 OK\r\n"
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, VIEW_CORRECT_REPLY_NO_ADDR);
+  tor_free(cp1);
+
+  /* Now specify an HS addr, and see that we only view those creds */
+  tor_free(args);
+  args =
+    tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd");
+
+#define VIEW_CORRECT_REPLY_JT4 "250-ONION_CLIENT_AUTH_VIEW " \
+    "jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd\r\n"   \
+  "250-CLIENT jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd " \
+    "x25519:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA=\r\n" \
+  "250 OK\r\n"
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, VIEW_CORRECT_REPLY_JT4);
+  tor_free(cp1);
+
+  /* Now try to REMOVE the auth credentials */
+  tor_free(conn.current_cmd);
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_REMOVE");
+
+  /* First try with a wrong addr */
+  tor_free(args);
+  args = tor_strdup("thatsok");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Invalid v3 address \"thatsok\"\r\n");
+  tor_free(cp1);
+
+  client_jt4 = digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(client_jt4);
+
+  /* Now actually remove them. */
+  tor_free(args);
+  args =tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  client_jt4 = digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(!client_jt4);
+
+  /* Now try another time (we should get 'already removed' msg) */
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "251 No credentials for "
+           "\"jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd\"\r\n");
+  tor_free(cp1);
+
+  client_jt4 = digest256map_get(client_auths, service_identity_pk_jt4.pubkey);
+  tt_assert(!client_jt4);
+
+  /* Now also remove the other one */
+  tor_free(args);
+  args =
+    tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  /* Now also remove the other one */
+  tor_free(args);
+  args =
+    tor_strdup("jamie3vkiwibfiwucd6vxijskbhpjdyajmzeor4mc4i7yopvpo4p7cyd");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+  tor_free(cp1);
+
+  /* Finally, do another VIEW and see that we get nothing. */
+  tor_free(conn.current_cmd);
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_VIEW");
+  tor_free(args);
+  args = tor_strdup("");
+
+#define VIEW_CORRECT_REPLY_NOTHING "250-ONION_CLIENT_AUTH_VIEW\r\n250 OK\r\n"
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, VIEW_CORRECT_REPLY_NOTHING);
+  tor_free(cp1);
+
+  /* And a final VIEW with a wrong HS addr */
+  tor_free(args);
+  args = tor_strdup("house");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Invalid v3 address \"house\"\r\n");
+
+ done:
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+  hs_client_free_all();
+}
+
+/** Test some error cases of ONION_CLIENT_AUTH_ADD */
+static void
+test_hs_control_bad_onion_client_auth_add(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  control_connection_t conn;
+  char *cp1 = NULL;
+  size_t sz;
+  char *args = NULL;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_ADD");
+  }
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+  /* Register first service */
+  args = tor_strdup(
+                "badaddr x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ=");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Invalid v3 address \"badaddr\"\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+                    "love:eIIdIGoSZwI2Q/lSzpf92akGki5I+PZIDz37MA5BhlA=");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "552 Unrecognized key type \"love\"\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register second service (even with an unrecognized argument) */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+                    "x25519:QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUEK");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "512 Failed to decode x25519 private key\r\n");
+
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Register with an all zero client key */
+  args = tor_strdup("jt4grrjwzyz3pjkylwfau5xnjaj23vxmhskqaeyfhrfylelw4hvxcuyd "
+                    "x25519:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check contents */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "553 Invalid private key \"AAAAAAAAAAAAAAAAAAAA"
+                        "AAAAAAAAAAAAAAAAAAAAAAA=\"\r\n");
+
+  client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+ done:
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+  hs_client_free_all();
+}
+
+/** Test that we can correctly add permanent client auth credentials using the
+ *  control port. */
+static void
+test_hs_control_store_permanent_creds(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  ed25519_public_key_t service_identity_pk_2fv;
+  control_connection_t conn;
+  char *args = NULL;
+  char *cp1 = NULL;
+  char *creds_file_str = NULL;
+  char *creds_fname = NULL;
+
+  size_t sz;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_ADD");
+  }
+
+  { /* Setup the services */
+    retval = hs_parse_address(
+                 "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd",
+                 &service_identity_pk_2fv,
+                 NULL, NULL);
+    tt_int_op(retval, OP_EQ, 0);
+  }
+
+  digest256map_t *client_auths = get_hs_client_auths_map();
+  tt_assert(!client_auths);
+
+  /* Try registering first service with no ClientOnionAuthDir set */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ= "
+                    "Flags=Permanent");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response. This one should fail. */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "553 Unable to store creds for "
+        "\"2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd\"\r\n");
+
+  { /* Setup ClientOnionAuthDir */
+    int ret;
+    char *perm_creds_dir = tor_strdup(get_fname("permanent_credentials"));
+    get_options_mutable()->ClientOnionAuthDir = perm_creds_dir;
+
+    #ifdef _WIN32
+      ret = mkdir(perm_creds_dir);
+    #else
+      ret = mkdir(perm_creds_dir, 0700);
+    #endif
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
+  tor_free(args);
+  tor_free(cp1);
+
+  /* Try the control port command again. This time it should work! */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:iJ1tjKCrMAbiFT2bVrCjhbfMDnE1fpaRbIS5ZHKUvEQ= "
+                    "Flags=Permanent");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+
+  /* Check file contents! */
+  creds_fname = tor_strdup(get_fname("permanent_credentials/"
+     "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd.auth_private"));
+  creds_file_str = read_file_to_str(creds_fname, RFTS_BIN, NULL);
+
+  tt_assert(creds_file_str);
+  tt_str_op(creds_file_str, OP_EQ,
+         "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd:descriptor:"
+         /* This is the base32 represenation of the base64 iJ1t... key above */
+         "x25519:rcow3dfavmyanyqvhwnvnmfdqw34ydtrgv7jnelmqs4wi4uuxrca");
+
+  tor_free(args);
+  tor_free(cp1);
+
+  /* Overwrite the credentials and check that they got overwrited. */
+  args = tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd "
+                    "x25519:UDRvZLvcJo0QRLvDfkpgbtsqbkhIUQZyeo2FNBrgS18= "
+                    "Flags=Permanent");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response: we replaced! */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "251 Client for onion existed and replaced\r\n");
+
+  tor_free(creds_file_str);
+
+  /* Check creds file contents again. See that the key got updated */
+  creds_file_str = read_file_to_str(creds_fname, RFTS_BIN, NULL);
+  tt_assert(creds_file_str);
+  tt_str_op(creds_file_str, OP_EQ,
+         "2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd:descriptor:"
+         /* This is the base32 represenation of the base64 UDRv... key above */
+         "x25519:ka2g6zf33qti2ecexpbx4stan3nsu3sijbiqm4t2rwctigxajnpq");
+
+  /* Now for our next act!!! Actually get the HS client subsystem to parse the
+   * whole directory and make sure that it extracted the right credential! */
+  hs_config_client_authorization(get_options(), 0);
+
+  client_auths = get_hs_client_auths_map();
+  tt_assert(client_auths);
+  tt_uint_op(digest256map_size(client_auths), OP_EQ, 1);
+
+  hs_client_service_authorization_t *client_2fv =
+    digest256map_get(client_auths, service_identity_pk_2fv.pubkey);
+  tt_assert(client_2fv);
+  tt_int_op(client_2fv->flags, OP_EQ, CLIENT_AUTH_FLAG_IS_PERMANENT);
+  tt_str_op(hex_str((char*)client_2fv->enc_seckey.secret_key, 32), OP_EQ,
+           "50346F64BBDC268D1044BBC37E4A606EDB2A6E48485106727A8D85341AE04B5F");
+
+  /* And now for the final act! Use the REMOVE control port command to remove
+     the credential, and ensure that the file has also been removed! */
+  tor_free(conn.current_cmd);
+  tor_free(cp1);
+  tor_free(args);
+
+  /* Ensure that the creds file exists */
+  tt_int_op(file_status(creds_fname), OP_EQ, FN_FILE);
+
+  /* Do the REMOVE */
+  conn.current_cmd = tor_strdup("ONION_CLIENT_AUTH_REMOVE");
+  args =tor_strdup("2fvhjskjet3n5syd6yfg5lhvwcs62bojmthr35ko5bllr3iqdb4ctdyd");
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "250 OK\r\n");
+
+  /* Ensure that the file has been removed and the map is empty */
+  tt_int_op(file_status(creds_fname), OP_EQ, FN_NOENT);
+  tt_uint_op(digest256map_size(client_auths), OP_EQ, 0);
+
+ done:
+  tor_free(get_options_mutable()->ClientOnionAuthDir);
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+  tor_free(creds_fname);
+  tor_free(creds_file_str);
+  hs_client_free_all();
+}
+
+/** Test that ADD_ONION properly handles an attacker passing it a bad private
+ *  key. */
+static void
+test_hs_control_add_onion_with_bad_pubkey(void *arg)
+{
+  (void) arg;
+
+  MOCK(connection_write_to_buf_impl_, connection_write_to_buf_mock);
+
+  int retval;
+  control_connection_t conn;
+  char *args = NULL;
+  char *cp1 = NULL;
+  size_t sz;
+
+  hs_init();
+
+  { /* Setup the control conn */
+    memset(&conn, 0, sizeof(control_connection_t));
+    TO_CONN(&conn)->outbuf = buf_new();
+    conn.current_cmd = tor_strdup("ADD_ONION");
+  }
+
+  args = tor_strdup("ED25519-V3:AAAAAAAAAAAAAAAAAAAAAAAAAAAAA"
+                    "AAAAAAAAAAAAAAAAAAAAAAA"
+                    "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+                    "Port=9735,127.0.0.1 Flags=DiscardPK");
+
+  retval = handle_control_command(&conn, (uint32_t) strlen(args), args);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check control port response */
+  cp1 = buf_get_contents(TO_CONN(&conn)->outbuf, &sz);
+  tt_str_op(cp1, OP_EQ, "551 Failed to generate onion address\r\n");
+
+ done:
+  tor_free(args);
+  tor_free(cp1);
+  buf_free(TO_CONN(&conn)->outbuf);
+  tor_free(conn.current_cmd);
+}
+
 struct testcase_t hs_control_tests[] = {
   { "hs_desc_event", test_hs_desc_event, TT_FORK,
     NULL, NULL },
+  { "hs_control_good_onion_client_auth_add",
+    test_hs_control_good_onion_client_auth_add, TT_FORK,
+    NULL, NULL },
+  { "hs_control_bad_onion_client_auth_add",
+    test_hs_control_bad_onion_client_auth_add, TT_FORK,
+    NULL, NULL },
+  { "hs_control_store_permanent_creds",
+    test_hs_control_store_permanent_creds, TT_FORK, NULL, NULL },
+  { "hs_control_add_onion_with_bad_pubkey",
+    test_hs_control_add_onion_with_bad_pubkey, TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_hs_descriptor.c b/src/test/test_hs_descriptor.c
index de584ed47a..782b78306c 100644
--- a/src/test/test_hs_descriptor.c
+++ b/src/test/test_hs_descriptor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -14,6 +14,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "trunnel/ed25519_cert.h"
 #include "core/or/or.h"
+#include "app/config/config.h"
 #include "feature/hs/hs_descriptor.h"
 #include "test/test.h"
 #include "feature/nodelist/torcert.h"
@@ -21,21 +22,15 @@
 #include "test/hs_test_helpers.h"
 #include "test/test_helpers.h"
 #include "test/log_test_helpers.h"
+#include "test/rng_test_helpers.h"
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
 #include "test_hs_descriptor.inc"
-ENABLE_GCC_WARNING(overlength-strings)
-
-/* Mock function to fill all bytes with 1 */
-static void
-mock_crypto_strongest_rand(uint8_t *out, size_t out_len)
-{
-  memset(out, 1, out_len);
-}
+ENABLE_GCC_WARNING("-Woverlength-strings")
 
 /* Test certificate encoding put in a descriptor. */
 static void
@@ -43,7 +38,6 @@ test_cert_encoding(void *arg)
 {
   int ret;
   char *encoded = NULL;
-  time_t now = time(NULL);
   ed25519_keypair_t kp;
   ed25519_public_key_t signed_key;
   ed25519_secret_key_t secret_key;
@@ -51,6 +45,10 @@ test_cert_encoding(void *arg)
 
   (void) arg;
 
+  /* Change time to 03-01-2002 23:36 UTC */
+  update_approx_time(1010101010);
+  time_t now = approx_time();
+
   ret = ed25519_keypair_generate(&kp, 0);
   tt_int_op(ret, == , 0);
   ret = ed25519_secret_key_generate(&secret_key, 0);
@@ -94,13 +92,31 @@ test_cert_encoding(void *arg)
     /* The cert did have the signing key? */
     ret= ed25519_pubkey_eq(&parsed_cert->signing_key, &kp.pubkey);
     tt_int_op(ret, OP_EQ, 1);
-    tor_cert_free(parsed_cert);
 
     /* Get to the end part of the certificate. */
     pos += b64_cert_len;
     tt_int_op(strcmpstart(pos, "-----END ED25519 CERT-----"), OP_EQ, 0);
     pos += strlen("-----END ED25519 CERT-----");
     tt_str_op(pos, OP_EQ, "");
+
+    /* Check that certificate expiry works properly and emits the right log
+       message */
+    const char *msg = "fire";
+    /* Move us forward 4 hours so that the the certificate is definitely
+       expired */
+    update_approx_time(approx_time() + 3600*4);
+    setup_full_capture_of_logs(LOG_PROTOCOL_WARN);
+    ret = cert_is_valid(parsed_cert, CERT_TYPE_SIGNING_AUTH, msg);
+    tt_int_op(ret, OP_EQ, 0);
+    /* Since the current time at the creation of the cert was "03-01-2002
+     * 23:36", and the expiration date of the cert was two hours, the Tor code
+     * will ceiling that and make it 02:00. Make sure that the right log
+     * message is emitted */
+    expect_log_msg_containing("Invalid signature for fire: expired"
+                              " (2002-01-04 02:00:00)");
+    teardown_capture_of_logs();
+
+    tor_cert_free(parsed_cert);
   }
 
  done:
@@ -132,7 +148,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -149,7 +165,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -166,7 +182,7 @@ test_descriptor_padding(void *arg)
     tt_assert(padded_plaintext);
     tor_free(plaintext);
     /* Make sure our padding has been zeroed. */
-    tt_int_op(tor_mem_is_zero((char *) padded_plaintext + plaintext_len,
+    tt_int_op(fast_mem_is_zero((char *) padded_plaintext + plaintext_len,
                               padded_len - plaintext_len), OP_EQ, 1);
     tor_free(padded_plaintext);
     /* Never never have a padded length smaller than the plaintext. */
@@ -179,115 +195,6 @@ test_descriptor_padding(void *arg)
 }
 
 static void
-test_link_specifier(void *arg)
-{
-  ssize_t ret;
-  hs_desc_link_specifier_t spec;
-  smartlist_t *link_specifiers = smartlist_new();
-  char buf[256];
-  char *b64 = NULL;
-  link_specifier_t *ls = NULL;
-
-  (void) arg;
-
-  /* Always this port. */
-  spec.u.ap.port = 42;
-  smartlist_add(link_specifiers, &spec);
-
-  /* Test IPv4 for starter. */
-  {
-    uint32_t ipv4;
-
-    spec.type = LS_IPV4;
-    ret = tor_addr_parse(&spec.u.ap.addr, "1.2.3.4");
-    tt_int_op(ret, OP_EQ, AF_INET);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 8);
-    /* Should be 2 bytes for port and 4 bytes for IPv4. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 6);
-    ipv4 = link_specifier_get_un_ipv4_addr(ls);
-    tt_int_op(tor_addr_to_ipv4h(&spec.u.ap.addr), OP_EQ, ipv4);
-    tt_int_op(link_specifier_get_un_ipv4_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test IPv6. */
-  {
-    uint8_t ipv6[16];
-
-    spec.type = LS_IPV6;
-    ret = tor_addr_parse(&spec.u.ap.addr, "[1:2:3:4::]");
-    tt_int_op(ret, OP_EQ, AF_INET6);
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    tt_int_op(ret, OP_EQ, 20);
-    /* Should be 2 bytes for port and 16 bytes for IPv6. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, 18);
-    for (unsigned int i = 0; i < sizeof(ipv6); i++) {
-      ipv6[i] = link_specifier_get_un_ipv6_addr(ls, i);
-    }
-    tt_mem_op(tor_addr_to_in6_addr8(&spec.u.ap.addr), OP_EQ, ipv6,
-              sizeof(ipv6));
-    tt_int_op(link_specifier_get_un_ipv6_port(ls), OP_EQ, spec.u.ap.port);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
-  /* Test legacy. */
-  {
-    uint8_t *id;
-
-    spec.type = LS_LEGACY_ID;
-    memset(spec.u.legacy_id, 'Y', sizeof(spec.u.legacy_id));
-    b64 = encode_link_specifiers(link_specifiers);
-    tt_assert(b64);
-
-    /* Decode it and validate the format. */
-    ret = base64_decode(buf, sizeof(buf), b64, strlen(b64));
-    tt_int_op(ret, OP_GT, 0);
-    /* First byte is the number of link specifier. */
-    tt_int_op(get_uint8(buf), OP_EQ, 1);
-    ret = link_specifier_parse(&ls, (uint8_t *) buf + 1, ret - 1);
-    /* 20 bytes digest + 1 byte type + 1 byte len. */
-    tt_int_op(ret, OP_EQ, 22);
-    tt_int_op(link_specifier_getlen_un_legacy_id(ls), OP_EQ, DIGEST_LEN);
-    /* Digest length is 20 bytes. */
-    tt_int_op(link_specifier_get_ls_len(ls), OP_EQ, DIGEST_LEN);
-    id = link_specifier_getarray_un_legacy_id(ls);
-    tt_mem_op(spec.u.legacy_id, OP_EQ, id, DIGEST_LEN);
-
-    link_specifier_free(ls);
-    ls = NULL;
-    tor_free(b64);
-  }
-
- done:
-  link_specifier_free(ls);
-  tor_free(b64);
-  smartlist_free(link_specifiers);
-}
-
-static void
 test_encode_descriptor(void *arg)
 {
   int ret;
@@ -336,7 +243,7 @@ test_decode_descriptor(void *arg)
   hs_descriptor_t *desc = NULL;
   hs_descriptor_t *decoded = NULL;
   hs_descriptor_t *desc_no_ip = NULL;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   (void) arg;
 
@@ -345,19 +252,19 @@ test_decode_descriptor(void *arg)
   desc = hs_helper_build_hs_desc_with_ip(&signing_kp);
 
   hs_helper_get_subcred_from_identity_keypair(&signing_kp,
-                                              subcredential);
+                                              &subcredential);
 
   /* Give some bad stuff to the decoding function. */
-  ret = hs_desc_decode_descriptor("hladfjlkjadf", subcredential,
+  ret = hs_desc_decode_descriptor("hladfjlkjadf", &subcredential,
                                   NULL, &decoded);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
 
   ret = hs_desc_encode_descriptor(desc, &signing_kp, NULL, &encoded);
-  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
   tt_assert(encoded);
 
-  ret = hs_desc_decode_descriptor(encoded, subcredential, NULL, &decoded);
-  tt_int_op(ret, OP_EQ, 0);
+  ret = hs_desc_decode_descriptor(encoded, &subcredential, NULL, &decoded);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
   tt_assert(decoded);
 
   hs_helper_desc_equal(desc, decoded);
@@ -368,7 +275,7 @@ test_decode_descriptor(void *arg)
     ret = ed25519_keypair_generate(&signing_kp_no_ip, 0);
     tt_int_op(ret, OP_EQ, 0);
     hs_helper_get_subcred_from_identity_keypair(&signing_kp_no_ip,
-                                                subcredential);
+                                                &subcredential);
     desc_no_ip = hs_helper_build_hs_desc_no_ip(&signing_kp_no_ip);
     tt_assert(desc_no_ip);
     tor_free(encoded);
@@ -377,8 +284,8 @@ test_decode_descriptor(void *arg)
     tt_int_op(ret, OP_EQ, 0);
     tt_assert(encoded);
     hs_descriptor_free(decoded);
-    ret = hs_desc_decode_descriptor(encoded, subcredential, NULL, &decoded);
-    tt_int_op(ret, OP_EQ, 0);
+    ret = hs_desc_decode_descriptor(encoded, &subcredential, NULL, &decoded);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(decoded);
   }
 
@@ -401,14 +308,14 @@ test_decode_descriptor(void *arg)
            &auth_ephemeral_kp.pubkey, CURVE25519_PUBKEY_LEN);
 
     hs_helper_get_subcred_from_identity_keypair(&signing_kp,
-                                                subcredential);
+                                                &subcredential);
 
     /* Build and add the auth client to the descriptor. */
     clients = desc->superencrypted_data.clients;
     if (!clients) {
       clients = smartlist_new();
     }
-    hs_desc_build_authorized_client(subcredential,
+    hs_desc_build_authorized_client(&subcredential,
                                     &client_kp.pubkey,
                                     &auth_ephemeral_kp.seckey,
                                     descriptor_cookie, client);
@@ -430,23 +337,23 @@ test_decode_descriptor(void *arg)
 
     /* If we do not have the client secret key, the decoding must fail. */
     hs_descriptor_free(decoded);
-    ret = hs_desc_decode_descriptor(encoded, subcredential,
+    ret = hs_desc_decode_descriptor(encoded, &subcredential,
                                     NULL, &decoded);
-    tt_int_op(ret, OP_LT, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_NEED_CLIENT_AUTH);
     tt_assert(!decoded);
 
     /* If we have an invalid client secret key, the decoding must fail. */
     hs_descriptor_free(decoded);
-    ret = hs_desc_decode_descriptor(encoded, subcredential,
+    ret = hs_desc_decode_descriptor(encoded, &subcredential,
                                     &invalid_client_kp.seckey, &decoded);
-    tt_int_op(ret, OP_LT, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_BAD_CLIENT_AUTH);
     tt_assert(!decoded);
 
     /* If we have the client secret key, the decoding must succeed and the
      * decoded descriptor must be correct. */
-    ret = hs_desc_decode_descriptor(encoded, subcredential,
+    ret = hs_desc_decode_descriptor(encoded, &subcredential,
                                     &client_kp.seckey, &decoded);
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_OK);
     tt_assert(decoded);
 
     hs_helper_desc_equal(desc, decoded);
@@ -682,7 +589,7 @@ test_decode_bad_signature(void *arg)
 
   setup_full_capture_of_logs(LOG_WARN);
   ret = hs_desc_decode_plaintext(HS_DESC_BAD_SIG, &desc_plaintext);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   expect_log_msg_containing("Malformed signature line. Rejecting.");
   teardown_capture_of_logs();
 
@@ -722,14 +629,14 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, bad_value, "180", "42", "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Missing fields. */
   {
     const char *plaintext = "hs-descriptor 3\n";
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Max length. */
@@ -742,7 +649,7 @@ test_decode_plaintext(void *arg)
     plaintext[big - 1] = '\0';
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Bad lifetime value. */
@@ -751,7 +658,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", bad_value, "42", "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Huge lifetime value. */
@@ -760,7 +667,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", "7181615", "42", "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Invalid encrypted section. */
@@ -769,7 +676,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", "180", "42", bad_value);
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
   /* Invalid revision counter. */
@@ -778,7 +685,7 @@ test_decode_plaintext(void *arg)
     tor_asprintf(&plaintext, template, "3", "180", bad_value, "MESSAGE");
     ret = hs_desc_decode_plaintext(plaintext, &desc_plaintext);
     tor_free(plaintext);
-    tt_int_op(ret, OP_EQ, -1);
+    tt_int_op(ret, OP_EQ, HS_DESC_DECODE_PLAINTEXT_ERROR);
   }
 
  done:
@@ -848,8 +755,7 @@ test_desc_signature(void *arg)
   ret = ed25519_sign_prefixed(&sig, (const uint8_t *) data, strlen(data),
                               "Tor onion service descriptor sig v3", &kp);
   tt_int_op(ret, OP_EQ, 0);
-  ret = ed25519_signature_to_base64(sig_b64, &sig);
-  tt_int_op(ret, OP_EQ, 0);
+  ed25519_signature_to_base64(sig_b64, &sig);
   /* Build the descriptor that should be valid. */
   tor_asprintf(&desc, "%ssignature %s\n", data, sig_b64);
   ret = desc_sig_is_valid(sig_b64, &kp.pubkey, desc, strlen(desc));
@@ -878,7 +784,7 @@ test_build_authorized_client(void *arg)
     "07d087f1d8c68393721f6e70316d3b29";
   const char client_pubkey_b16[] =
     "8c1298fa6050e372f8598f6deca32e27b0ad457741422c2629ebb132cf7fae37";
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
   char *mem_op_hex_tmp=NULL;
 
   (void) arg;
@@ -890,7 +796,7 @@ test_build_authorized_client(void *arg)
   tt_int_op(ret, OP_EQ, 0);
   curve25519_public_key_generate(&client_auth_pk, &client_auth_sk);
 
-  memset(subcredential, 42, sizeof(subcredential));
+  memset(subcredential.subcred, 42, sizeof(subcredential));
 
   desc_client = tor_malloc_zero(sizeof(hs_desc_authorized_client_t));
 
@@ -909,9 +815,9 @@ test_build_authorized_client(void *arg)
                 client_pubkey_b16,
                 strlen(client_pubkey_b16));
 
-  MOCK(crypto_strongest_rand_, mock_crypto_strongest_rand);
+  testing_enable_prefilled_rng("\x01", 1);
 
-  hs_desc_build_authorized_client(subcredential,
+  hs_desc_build_authorized_client(&subcredential,
                                   &client_auth_pk, &auth_ephemeral_sk,
                                   descriptor_cookie, desc_client);
 
@@ -925,15 +831,13 @@ test_build_authorized_client(void *arg)
  done:
   tor_free(desc_client);
   tor_free(mem_op_hex_tmp);
-  UNMOCK(crypto_strongest_rand_);
+  testing_disable_prefilled_rng();
 }
 
 struct testcase_t hs_descriptor[] = {
   /* Encoding tests. */
   { "cert_encoding", test_cert_encoding, TT_FORK,
     NULL, NULL },
-  { "link_specifier", test_link_specifier, TT_FORK,
-    NULL, NULL },
   { "encode_descriptor", test_encode_descriptor, TT_FORK,
     NULL, NULL },
   { "descriptor_padding", test_descriptor_padding, TT_FORK,
diff --git a/src/test/test_hs_dos.c b/src/test/test_hs_dos.c
new file mode 100644
index 0000000000..642513efce
--- /dev/null
+++ b/src/test/test_hs_dos.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_cell.c
+ * \brief Test hidden service cell functionality.
+ */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define HS_DOS_PRIVATE
+#define HS_INTROPOINT_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+
+#include "core/or/circuitlist.h"
+#include "core/or/circuituse.h"
+#include "core/or/or_circuit_st.h"
+
+#include "feature/hs/hs_dos.h"
+#include "feature/hs/hs_intropoint.h"
+#include "feature/nodelist/networkstatus.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_ns_consensus = tor_malloc_zero(sizeof(networkstatus_t));
+  current_ns_consensus->net_params = smartlist_new();
+  smartlist_add(current_ns_consensus->net_params,
+                (void *) "HiddenServiceEnableIntroDoSDefense=1");
+  hs_dos_consensus_has_changed(current_ns_consensus);
+}
+
+static void
+free_mock_consensus(void)
+{
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_can_send_intro2(void *arg)
+{
+  uint32_t now = (uint32_t) approx_time();
+  or_circuit_t *or_circ = NULL;
+
+  (void) arg;
+
+  hs_init();
+  hs_dos_init();
+
+  get_options_mutable()->ORPort_set = 1;
+  setup_mock_consensus();
+
+  or_circ =  or_circuit_new(1, NULL);
+
+  /* Make that circuit a service intro point. */
+  circuit_change_purpose(TO_CIRCUIT(or_circ), CIRCUIT_PURPOSE_INTRO_POINT);
+  hs_dos_setup_default_intro2_defenses(or_circ);
+  or_circ->introduce2_dos_defense_enabled = 1;
+
+  /* Brand new circuit, we should be able to send INTRODUCE2 cells. */
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+
+  /* Simulate that 10 cells have arrived in 1 second. There should be no
+   * refill since the bucket is already at maximum on the first cell. */
+  update_approx_time(++now);
+  for (int i = 0; i < 10; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 10);
+
+  /* Fully refill the bucket minus 1 cell. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Receive an INTRODUCE2 at each second. We should have the bucket full
+   * since at every second it gets refilled. */
+  for (int i = 0; i < 10; i++) {
+    update_approx_time(++now);
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  /* Last check if we can send the cell decrements the bucket so minus 1. */
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL) - 1);
+
+  /* Manually reset bucket for next test. */
+  token_bucket_ctr_reset(&or_circ->introduce2_bucket, now);
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_burst_consensus_param(NULL));
+
+  /* Do a full burst in the current second which should empty the bucket and
+   * we shouldn't be allowed to send one more cell after that. We go minus 1
+   * cell else the very last check if we can send the INTRO2 cell returns
+   * false because the bucket goes down to 0. */
+  for (uint32_t i = 0; i < get_intro2_burst_consensus_param(NULL) - 1; i++) {
+    tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  }
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 1);
+  /* Get the last remaining cell, we shouldn't be allowed to send it. */
+  tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+
+  /* Make sure the next 100 cells aren't allowed and bucket stays at 0. */
+  for (int i = 0; i < 100; i++) {
+    tt_int_op(false, OP_EQ, hs_dos_can_send_intro2(or_circ));
+    tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ, 0);
+  }
+
+  /* One second has passed, we should have the rate minus 1 cell added. */
+  update_approx_time(++now);
+  tt_int_op(true, OP_EQ, hs_dos_can_send_intro2(or_circ));
+  tt_uint_op(token_bucket_ctr_get(&or_circ->introduce2_bucket), OP_EQ,
+             get_intro2_rate_consensus_param(NULL) - 1);
+
+ done:
+  circuit_free_(TO_CIRCUIT(or_circ));
+
+  hs_free_all();
+  free_mock_consensus();
+}
+
+static void
+test_validate_dos_extension_params(void *arg)
+{
+  bool ret;
+
+  (void) arg;
+
+  /* Validate the default values. */
+  ret = cell_dos_extension_parameters_are_valid(
+                                      get_intro2_rate_consensus_param(NULL),
+                                      get_intro2_burst_consensus_param(NULL));
+  tt_assert(ret);
+
+  /* Valid custom rate/burst. */
+  ret = cell_dos_extension_parameters_are_valid(17, 42);
+  tt_assert(ret);
+  ret = cell_dos_extension_parameters_are_valid(INT32_MAX, INT32_MAX);
+  tt_assert(ret);
+
+  /* Invalid rate. */
+  ret = cell_dos_extension_parameters_are_valid(UINT64_MAX, 42);
+  tt_assert(!ret);
+
+  /* Invalid burst. */
+  ret = cell_dos_extension_parameters_are_valid(42, UINT64_MAX);
+  tt_assert(!ret);
+
+  /* Value of 0 is valid (but should disable defenses) */
+  ret = cell_dos_extension_parameters_are_valid(0, 0);
+  tt_assert(ret);
+
+  /* Can't have burst smaller than rate. */
+  ret = cell_dos_extension_parameters_are_valid(42, 40);
+  tt_assert(!ret);
+
+ done:
+  return;
+}
+
+struct testcase_t hs_dos_tests[] = {
+  { "can_send_intro2", test_can_send_intro2, TT_FORK,
+    NULL, NULL },
+  { "validate_dos_extension_params", test_validate_dos_extension_params,
+    TT_FORK, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_intropoint.c b/src/test/test_hs_intropoint.c
index 558fc32c54..e6b27d7a50 100644
--- a/src/test/test_hs_intropoint.c
+++ b/src/test/test_hs_intropoint.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -16,6 +16,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 
 #include "core/or/or.h"
+#include "core/or/channel.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuituse.h"
 #include "ht.h"
@@ -25,6 +26,8 @@
 #include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_common.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_dos.h"
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 
@@ -43,6 +46,9 @@ new_establish_intro_cell(const char *circ_nonce,
   uint8_t buf[RELAY_PAYLOAD_SIZE] = {0};
   trn_cell_establish_intro_t *cell = NULL;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Ensure that *cell_out is NULL such that we can use to check if we need to
    * free `cell` in case of an error. */
@@ -50,9 +56,9 @@ new_establish_intro_cell(const char *circ_nonce,
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, buf);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, buf);
   tt_i64_op(cell_len, OP_GT, 0);
 
   cell_len = trn_cell_establish_intro_parse(&cell, buf, sizeof(buf));
@@ -73,12 +79,15 @@ new_establish_intro_encoded_cell(const char *circ_nonce, uint8_t *cell_out)
 {
   ssize_t cell_len = 0;
   hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+
+  memset(&config, 0, sizeof(config));
 
   /* Auth key pair is generated in the constructor so we are all set for
    * using this IP object. */
-  ip = service_intro_point_new(NULL, 0, 0);
+  ip = service_intro_point_new(NULL);
   tt_assert(ip);
-  cell_len = hs_cell_build_establish_intro(circ_nonce, ip, cell_out);
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell_out);
   tt_i64_op(cell_len, OP_GT, 0);
 
  done:
@@ -118,6 +127,8 @@ helper_create_intro_circuit(void)
   or_circuit_t *circ = or_circuit_new(0, NULL);
   tt_assert(circ);
   circuit_change_purpose(TO_CIRCUIT(circ), CIRCUIT_PURPOSE_OR);
+  token_bucket_ctr_init(&circ->introduce2_bucket, 100, 100,
+                        (uint32_t) approx_time());
  done:
   return circ;
 }
@@ -693,6 +704,17 @@ test_introduce1_suitable_circuit(void *arg)
     tt_int_op(ret, OP_EQ, 0);
   }
 
+  /* Single hop circuit should not be allowed. */
+  {
+    circ = or_circuit_new(0, NULL);
+    circ->p_chan = tor_malloc_zero(sizeof(channel_t));
+    circ->p_chan->is_client = 1;
+    ret = circuit_is_suitable_for_introduce1(circ);
+    tor_free(circ->p_chan);
+    circuit_free_(TO_CIRCUIT(circ));
+    tt_int_op(ret, OP_EQ, 0);
+  }
+
  done:
   ;
 }
@@ -735,12 +757,15 @@ test_introduce1_validation(void *arg)
   cell = helper_create_introduce1_cell();
   tt_assert(cell);
 
+#ifndef ALL_BUGS_ARE_FATAL
   /* It should NOT be a legacy cell which will trigger a BUG(). */
   memset(cell->legacy_key_id, 'a', sizeof(cell->legacy_key_id));
   tor_capture_bugs_(1);
   ret = validate_introduce1_parsed_cell(cell);
   tor_end_capture_bugs_();
   tt_int_op(ret, OP_EQ, -1);
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
+
   /* Reset legacy ID and make sure it's correct. */
   memset(cell->legacy_key_id, 0, sizeof(cell->legacy_key_id));
   ret = validate_introduce1_parsed_cell(cell);
@@ -888,43 +913,213 @@ test_received_introduce1_handling(void *arg)
   UNMOCK(relay_send_command_from_edge_);
 }
 
+static void
+test_received_establish_intro_dos_ext(void *arg)
+{
+  int ret;
+  ssize_t cell_len = 0;
+  uint8_t cell[RELAY_PAYLOAD_SIZE] = {0};
+  char circ_nonce[DIGEST_LEN] = {0};
+  hs_service_intro_point_t *ip = NULL;
+  hs_service_config_t config;
+  or_circuit_t *intro_circ = or_circuit_new(0,NULL);
+
+  (void) arg;
+
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+
+  hs_circuitmap_init();
+
+  /* Setup. */
+  crypto_rand(circ_nonce, sizeof(circ_nonce));
+  ip = service_intro_point_new(NULL);
+  tt_assert(ip);
+  ip->support_intro2_dos_defense = 1;
+  memset(&config, 0, sizeof(config));
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 13;
+  config.intro_dos_burst_per_sec = 42;
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+  /* The INTRO2 bucket should be 0 at this point. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 0);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 0);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 0);
+
+  /* Case 1: Build encoded cell. Usable DoS parameters. */
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  /* Should be set to the burst value. */
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ, 42);
+  /* Validate the config of the intro2 bucket. */
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ, 13);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ, 42);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ, 1);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 2: Build encoded cell. Bad DoS parameters. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = UINT_MAX;
+  config.intro_dos_burst_per_sec = 13;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 3: Build encoded cell. Burst is smaller than rate. Not allowed. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 87;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 4: Build encoded cell. Rate is 0 but burst is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 0;
+  config.intro_dos_burst_per_sec = 45;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+  /* Need to reset the circuit in between test cases. */
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  intro_circ = or_circuit_new(0,NULL);
+  helper_prepare_circ_for_intro(intro_circ, circ_nonce);
+
+  /* Case 5: Build encoded cell. Burst is 0 but rate is not 0. Disables the
+   * defense. */
+  config.has_dos_defense_enabled = 1;
+  config.intro_dos_rate_per_sec = 45;
+  config.intro_dos_burst_per_sec = 0;
+  cell_len = hs_cell_build_establish_intro(circ_nonce, &config, ip, cell);
+  tt_size_op(cell_len, OP_GT, 0);
+  /* Pass it to the intro point. */
+  ret = hs_intro_received_establish_intro(intro_circ, cell, cell_len);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_u64_op(token_bucket_ctr_get(&intro_circ->introduce2_bucket), OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_u64_op(intro_circ->introduce2_bucket.cfg.rate, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_RATE_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_bucket.cfg.burst, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_BURST_PER_SEC_DEFAULT);
+  tt_int_op(intro_circ->introduce2_dos_defense_enabled, OP_EQ,
+            HS_CONFIG_V3_DOS_DEFENSE_DEFAULT);
+
+ done:
+  circuit_free_(TO_CIRCUIT(intro_circ));
+  service_intro_point_free(ip);
+  hs_circuitmap_free_all();
+  UNMOCK(relay_send_command_from_edge_);
+}
+
+static void *
+hs_subsystem_setup_fn(const struct testcase_t *tc)
+{
+  (void) tc;
+
+  return NULL;
+}
+
+static int
+hs_subsystem_cleanup_fn(const struct testcase_t *tc, void *arg)
+{
+  (void) tc;
+  (void) arg;
+
+  return 1;
+}
+
+static struct testcase_setup_t test_setup = {
+  hs_subsystem_setup_fn, hs_subsystem_cleanup_fn
+};
+
 struct testcase_t hs_intropoint_tests[] = {
   { "intro_point_registration",
-    test_intro_point_registration, TT_FORK, NULL, NULL },
+    test_intro_point_registration, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype",
-    test_establish_intro_wrong_keytype, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_keytype2",
-    test_establish_intro_wrong_keytype2, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_keytype2, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_purpose",
-    test_establish_intro_wrong_purpose, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_purpose, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig",
-    test_establish_intro_wrong_sig, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_sig_len",
-    test_establish_intro_wrong_sig_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_sig_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_auth_key_len",
-    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_auth_key_len, TT_FORK, NULL, &test_setup},
 
   { "receive_establish_intro_wrong_mac",
-    test_establish_intro_wrong_mac, TT_FORK, NULL, NULL },
+    test_establish_intro_wrong_mac, TT_FORK, NULL, &test_setup},
 
   { "introduce1_suitable_circuit",
-    test_introduce1_suitable_circuit, TT_FORK, NULL, NULL },
+    test_introduce1_suitable_circuit, TT_FORK, NULL, &test_setup},
 
   { "introduce1_is_legacy",
-    test_introduce1_is_legacy, TT_FORK, NULL, NULL },
+    test_introduce1_is_legacy, TT_FORK, NULL, &test_setup},
 
   { "introduce1_validation",
-    test_introduce1_validation, TT_FORK, NULL, NULL },
+    test_introduce1_validation, TT_FORK, NULL, &test_setup},
 
   { "received_introduce1_handling",
-    test_received_introduce1_handling, TT_FORK, NULL, NULL },
+    test_received_introduce1_handling, TT_FORK, NULL, &test_setup},
+
+  { "received_establish_intro_dos_ext",
+    test_received_establish_intro_dos_ext, TT_FORK, NULL, &test_setup},
 
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_hs_ntor.c b/src/test/test_hs_ntor.c
index 1c694e6040..7867740a1a 100644
--- a/src/test/test_hs_ntor.c
+++ b/src/test/test_hs_ntor.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -23,7 +23,7 @@ test_hs_ntor(void *arg)
 {
   int retval;
 
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   ed25519_keypair_t service_intro_auth_keypair;
   curve25519_keypair_t service_intro_enc_keypair;
@@ -42,7 +42,7 @@ test_hs_ntor(void *arg)
   /* Generate fake data for this unittest */
   {
     /* Generate fake subcredential */
-    memset(subcredential, 'Z', DIGEST256_LEN);
+    memset(subcredential.subcred, 'Z', DIGEST256_LEN);
 
     /* service */
     curve25519_keypair_generate(&service_intro_enc_keypair, 0);
@@ -57,7 +57,7 @@ test_hs_ntor(void *arg)
     hs_ntor_client_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
                                        &service_intro_enc_keypair.pubkey,
                                        &client_ephemeral_enc_keypair,
-                                       subcredential,
+                                       &subcredential,
                                        &client_hs_ntor_intro_cell_keys);
   tt_int_op(retval, OP_EQ, 0);
 
@@ -66,7 +66,7 @@ test_hs_ntor(void *arg)
     hs_ntor_service_get_introduce1_keys(&service_intro_auth_keypair.pubkey,
                                         &service_intro_enc_keypair,
                                         &client_ephemeral_enc_keypair.pubkey,
-                                        subcredential,
+                                        &subcredential,
                                         &service_hs_ntor_intro_cell_keys);
   tt_int_op(retval, OP_EQ, 0);
 
diff --git a/src/test/test_hs_ntor.sh b/src/test/test_hs_ntor.sh
index 8a0003d44a..ee7141cc9a 100755
--- a/src/test/test_hs_ntor.sh
+++ b/src/test/test_hs_ntor.sh
@@ -3,7 +3,7 @@
 
 exitcode=0
 
-# Run the python integration test sand return the exitcode of the python
+# Run the python integration tests and return the exitcode of the python
 # script.  The python script might ask the testsuite to skip it if not all
 # python dependencies are covered.
 "${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/hs_ntor_ref.py" || exitcode=$?
diff --git a/src/test/test_hs_ntor_cl.c b/src/test/test_hs_ntor_cl.c
index 6341b96d84..3acd7ef0bc 100644
--- a/src/test/test_hs_ntor_cl.c
+++ b/src/test/test_hs_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /** This is a wrapper over the little-t-tor HS ntor functions. The wrapper is
@@ -53,7 +53,7 @@ client1(int argc, char **argv)
   curve25519_public_key_t intro_enc_pubkey;
   ed25519_public_key_t intro_auth_pubkey;
   curve25519_keypair_t client_ephemeral_enc_keypair;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   /* Output */
   hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
@@ -65,7 +65,7 @@ client1(int argc, char **argv)
   BASE16(3, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
   BASE16(4, client_ephemeral_enc_keypair.seckey.secret_key,
          CURVE25519_SECKEY_LEN);
-  BASE16(5, subcredential, DIGEST256_LEN);
+  BASE16(5, subcredential.subcred, DIGEST256_LEN);
 
   /* Generate keypair */
   curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
@@ -74,7 +74,7 @@ client1(int argc, char **argv)
   retval = hs_ntor_client_get_introduce1_keys(&intro_auth_pubkey,
                                               &intro_enc_pubkey,
                                               &client_ephemeral_enc_keypair,
-                                              subcredential,
+                                              &subcredential,
                                               &hs_ntor_intro_cell_keys);
   if (retval < 0) {
     goto done;
@@ -106,7 +106,7 @@ server1(int argc, char **argv)
   curve25519_keypair_t intro_enc_keypair;
   ed25519_public_key_t intro_auth_pubkey;
   curve25519_public_key_t client_ephemeral_enc_pubkey;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   /* Output */
   hs_ntor_intro_cell_keys_t hs_ntor_intro_cell_keys;
@@ -119,7 +119,7 @@ server1(int argc, char **argv)
   BASE16(2, intro_auth_pubkey.pubkey, ED25519_PUBKEY_LEN);
   BASE16(3, intro_enc_keypair.seckey.secret_key, CURVE25519_SECKEY_LEN);
   BASE16(4, client_ephemeral_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
-  BASE16(5, subcredential, DIGEST256_LEN);
+  BASE16(5, subcredential.subcred, DIGEST256_LEN);
 
   /* Generate keypair */
   curve25519_public_key_generate(&intro_enc_keypair.pubkey,
@@ -130,7 +130,7 @@ server1(int argc, char **argv)
   retval = hs_ntor_service_get_introduce1_keys(&intro_auth_pubkey,
                                                &intro_enc_keypair,
                                                &client_ephemeral_enc_pubkey,
-                                               subcredential,
+                                               &subcredential,
                                                &hs_ntor_intro_cell_keys);
   if (retval < 0) {
     goto done;
@@ -188,7 +188,7 @@ client2(int argc, char **argv)
   ed25519_public_key_t intro_auth_pubkey;
   curve25519_keypair_t client_ephemeral_enc_keypair;
   curve25519_public_key_t service_ephemeral_rend_pubkey;
-  uint8_t subcredential[DIGEST256_LEN];
+  hs_subcredential_t subcredential;
 
   /* Output */
   hs_ntor_rend_cell_keys_t hs_ntor_rend_cell_keys;
@@ -201,7 +201,7 @@ client2(int argc, char **argv)
          CURVE25519_SECKEY_LEN);
   BASE16(4, intro_enc_pubkey.public_key, CURVE25519_PUBKEY_LEN);
   BASE16(5, service_ephemeral_rend_pubkey.public_key, CURVE25519_PUBKEY_LEN);
-  BASE16(6, subcredential, DIGEST256_LEN);
+  BASE16(6, subcredential.subcred, DIGEST256_LEN);
 
   /* Generate keypair */
   curve25519_public_key_generate(&client_ephemeral_enc_keypair.pubkey,
diff --git a/src/test/test_hs_ob.c b/src/test/test_hs_ob.c
new file mode 100644
index 0000000000..7f40187b5f
--- /dev/null
+++ b/src/test/test_hs_ob.c
@@ -0,0 +1,268 @@
+/* Copyright (c) 2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_hs_ob.c
+ * \brief Test hidden service onion balance functionality.
+ */
+
+#define CONFIG_PRIVATE
+#define HS_SERVICE_PRIVATE
+#define HS_OB_PRIVATE
+
+#include "test/test.h"
+#include "test/test_helpers.h"
+#include "test/log_test_helpers.h"
+
+#include "app/config/config.h"
+#include "feature/hs/hs_config.h"
+#include "feature/hs/hs_ob.h"
+#include "feature/hs/hs_service.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+static ed25519_keypair_t onion_addr_kp_1;
+static char onion_addr_1[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+static ed25519_keypair_t onion_addr_kp_2;
+static char onion_addr_2[HS_SERVICE_ADDR_LEN_BASE32 + 1];
+
+static bool config_is_good = true;
+
+static int
+helper_tor_config(const char *conf)
+{
+  int ret = -1;
+  or_options_t *options = helper_parse_options(conf);
+  tt_assert(options);
+  ret = hs_config_service_all(options, 0);
+ done:
+  or_options_free(options);
+  return ret;
+}
+
+static networkstatus_t mock_ns;
+
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void) now;
+  return &mock_ns;
+}
+
+static char *
+mock_read_file_to_str(const char *filename, int flags, struct stat *stat_out)
+{
+  char *ret = NULL;
+
+  (void) flags;
+  (void) stat_out;
+
+  if (!strcmp(filename, get_fname("hs3" PATH_SEPARATOR "ob_config"))) {
+    if (config_is_good) {
+      tor_asprintf(&ret, "MasterOnionAddress %s.onion\n"
+                         "MasterOnionAddress %s.onion\n",
+                         onion_addr_1, onion_addr_2);
+    } else {
+      tor_asprintf(&ret, "MasterOnionAddress JUNKJUNKJUNK.onion\n"
+                         "UnknownOption BLAH\n");
+    }
+    goto done;
+  }
+
+ done:
+  return ret;
+}
+
+static void
+test_parse_config_file(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+  const ed25519_public_key_t *pkey;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(read_file_to_str, mock_read_file_to_str);
+
+#define fmt_conf            \
+  "HiddenServiceDir %s\n"   \
+  "HiddenServicePort 22\n"  \
+  "HiddenServiceOnionBalanceInstance 1\n"
+  tor_asprintf(&conf, fmt_conf, get_fname("hs3"));
+#undef fmt_conf
+
+  /* Build the OB frontend onion addresses. */
+  ed25519_keypair_generate(&onion_addr_kp_1, 0);
+  hs_build_address(&onion_addr_kp_1.pubkey, HS_VERSION_THREE, onion_addr_1);
+  ed25519_keypair_generate(&onion_addr_kp_2, 0);
+  hs_build_address(&onion_addr_kp_2.pubkey, HS_VERSION_THREE, onion_addr_2);
+
+  ret = helper_tor_config(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, 0);
+
+  /* Load the keys for the service. After that, the v3 service should be
+   * registered in the global map and we'll be able to access it. */
+  tt_int_op(get_hs_service_staging_list_size(), OP_EQ, 1);
+  hs_service_load_all_keys();
+  tt_int_op(get_hs_service_map_size(), OP_EQ, 1);
+  const hs_service_t *s = get_first_service();
+  tt_assert(s);
+  tt_assert(s->config.ob_master_pubkeys);
+  tt_assert(hs_ob_service_is_instance(s));
+  tt_assert(smartlist_len(s->config.ob_master_pubkeys) == 2);
+
+  /* Test the public keys we've added. */
+  pkey = smartlist_get(s->config.ob_master_pubkeys, 0);
+  tt_mem_op(&onion_addr_kp_1.pubkey, OP_EQ, pkey, ED25519_PUBKEY_LEN);
+  pkey = smartlist_get(s->config.ob_master_pubkeys, 1);
+  tt_mem_op(&onion_addr_kp_2.pubkey, OP_EQ, pkey, ED25519_PUBKEY_LEN);
+
+ done:
+  hs_free_all();
+
+  UNMOCK(read_file_to_str);
+}
+
+static void
+test_parse_config_file_bad(void *arg)
+{
+  int ret;
+  char *conf = NULL;
+
+  (void) arg;
+
+  hs_init();
+
+  MOCK(read_file_to_str, mock_read_file_to_str);
+
+  /* Indicate mock_read_file_to_str() to use the bad config. */
+  config_is_good = false;
+
+#define fmt_conf            \
+  "HiddenServiceDir %s\n"   \
+  "HiddenServicePort 22\n"  \
+  "HiddenServiceOnionBalanceInstance 1\n"
+  tor_asprintf(&conf, fmt_conf, get_fname("hs3"));
+#undef fmt_conf
+
+  setup_full_capture_of_logs(LOG_INFO);
+  ret = helper_tor_config(conf);
+  tor_free(conf);
+  tt_int_op(ret, OP_EQ, -1);
+  expect_log_msg_containing("OnionBalance: MasterOnionAddress "
+                            "JUNKJUNKJUNK.onion is invalid");
+  expect_log_msg_containing("Found unrecognized option \'UnknownOption\'; "
+                            "saving it.");
+  teardown_capture_of_logs();
+
+ done:
+  hs_free_all();
+
+  UNMOCK(read_file_to_str);
+}
+
+static void
+test_get_subcredentials(void *arg)
+{
+  int ret;
+  hs_service_t *service = NULL;
+  hs_service_config_t config;
+  hs_subcredential_t *subcreds = NULL;
+
+  (void) arg;
+
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+
+  /* Setup consensus with proper time so we can compute the time period. */
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+
+  config.ob_master_pubkeys = smartlist_new();
+  tt_assert(config.ob_master_pubkeys);
+
+  /* Set up an instance */
+  service = tor_malloc_zero(sizeof(hs_service_t));
+  service->config = config;
+  /* Setup the service descriptors */
+  service->desc_current = service_descriptor_new();
+  service->desc_next = service_descriptor_new();
+
+  /* First try to compute subcredentials but with no OB keys. Make sure that
+   * subcreds get NULLed. To do this check we first poison subcreds. */
+  subcreds = (void*)999;
+  tt_ptr_op(subcreds, OP_NE, NULL);
+  size_t num = compute_subcredentials(service, &subcreds);
+  tt_ptr_op(subcreds, OP_EQ, NULL);
+
+  /* Generate a keypair to add to the OB keys list. */
+  ed25519_keypair_generate(&onion_addr_kp_1, 0);
+  smartlist_add(config.ob_master_pubkeys, &onion_addr_kp_1.pubkey);
+
+  /* Set up the instance subcredentials */
+  char current_subcred[SUBCRED_LEN];
+  char next_subcred[SUBCRED_LEN];
+  memset(current_subcred, 'C', SUBCRED_LEN);
+  memset(next_subcred, 'N', SUBCRED_LEN);
+  memcpy(service->desc_current->desc->subcredential.subcred, current_subcred,
+         SUBCRED_LEN);
+  memcpy(service->desc_next->desc->subcredential.subcred, next_subcred,
+         SUBCRED_LEN);
+
+  /* See that subcreds are computed properly */
+  num = compute_subcredentials(service, &subcreds);
+  /* 5 subcredentials: 3 for the frontend, 2 for the instance */
+  tt_uint_op(num, OP_EQ, 5);
+  tt_ptr_op(subcreds, OP_NE, NULL);
+
+  /* Validate the subcredentials we just got. We'll build them oursevles with
+   * the right time period steps and compare. */
+  const uint64_t tp = hs_get_time_period_num(0);
+  const int steps[3] = {0, -1, 1};
+
+  unsigned int i;
+  for (i = 0; i < 3; i++) {
+    hs_subcredential_t subcredential;
+    ed25519_public_key_t blinded_pubkey;
+    hs_build_blinded_pubkey(&onion_addr_kp_1.pubkey, NULL, 0, tp + steps[i],
+                            &blinded_pubkey);
+    hs_get_subcredential(&onion_addr_kp_1.pubkey, &blinded_pubkey,
+                         &subcredential);
+    tt_mem_op(subcreds[i].subcred, OP_EQ, subcredential.subcred,
+              SUBCRED_LEN);
+  }
+
+  tt_mem_op(subcreds[i++].subcred, OP_EQ, current_subcred, SUBCRED_LEN);
+  tt_mem_op(subcreds[i++].subcred, OP_EQ, next_subcred, SUBCRED_LEN);
+
+ done:
+  tor_free(subcreds);
+
+  smartlist_free(config.ob_master_pubkeys);
+  if (service) {
+    memset(&service->config, 0, sizeof(hs_service_config_t));
+    hs_service_free(service);
+  }
+
+  UNMOCK(networkstatus_get_live_consensus);
+}
+
+struct testcase_t hs_ob_tests[] = {
+  { "parse_config_file", test_parse_config_file, TT_FORK,
+    NULL, NULL },
+  { "parse_config_file_bad", test_parse_config_file_bad, TT_FORK,
+    NULL, NULL },
+
+  { "get_subcredentials", test_get_subcredentials, TT_FORK,
+    NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_hs_service.c b/src/test/test_hs_service.c
index c60ab6c930..8b94bb6cf1 100644
--- a/src/test/test_hs_service.c
+++ b/src/test/test_hs_service.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,8 +19,9 @@
 #define MAINLOOP_PRIVATE
 #define NETWORKSTATUS_PRIVATE
 #define STATEFILE_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define HS_CLIENT_PRIVATE
+#define CRYPT_PATH_PRIVATE
 
 #include "test/test.h"
 #include "test/test_helpers.h"
@@ -43,13 +44,15 @@
 #include "core/or/versions.h"
 #include "feature/dirauth/dirvote.h"
 #include "feature/dirauth/shared_random_state.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 #include "feature/hs/hs_circuit.h"
 #include "feature/hs/hs_circuitmap.h"
 #include "feature/hs/hs_client.h"
 #include "feature/hs/hs_common.h"
 #include "feature/hs/hs_config.h"
 #include "feature/hs/hs_ident.h"
+#include "feature/hs/hs_ob.h"
+#include "feature/hs/hs_cell.h"
 #include "feature/hs/hs_intropoint.h"
 #include "feature/hs/hs_service.h"
 #include "feature/nodelist/networkstatus.h"
@@ -60,6 +63,7 @@
 
 #include "core/or/cpath_build_state_st.h"
 #include "core/or/crypt_path_st.h"
+#include "core/or/crypt_path.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -86,6 +90,14 @@ mock_networkstatus_get_reasonably_live_consensus(time_t now, int flavor)
   return &mock_ns;
 }
 
+static networkstatus_t *
+mock_networkstatus_get_reasonably_live_consensus_null(time_t now, int flavor)
+{
+  (void) now;
+  (void) flavor;
+  return NULL;
+}
+
 static or_state_t *dummy_state = NULL;
 
 /* Mock function to get fake or state (used for rev counters) */
@@ -108,6 +120,9 @@ mock_circuit_mark_for_close(circuit_t *circ, int reason, int line,
   return;
 }
 
+static size_t relay_payload_len;
+static char relay_payload[RELAY_PAYLOAD_SIZE];
+
 static int
 mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
                                   uint8_t relay_command, const char *payload,
@@ -123,6 +138,24 @@ mock_relay_send_command_from_edge(streamid_t stream_id, circuit_t *circ,
   (void) cpath_layer;
   (void) filename;
   (void) lineno;
+
+  memcpy(relay_payload, payload, payload_len);
+  relay_payload_len = payload_len;
+
+  return 0;
+}
+
+static unsigned int num_intro_points = 0;
+static unsigned int
+mock_count_desc_circuit_established(const hs_service_descriptor_t *desc)
+{
+  (void) desc;
+  return num_intro_points;
+}
+
+static int
+mock_router_have_minimum_dir_info_false(void)
+{
   return 0;
 }
 
@@ -170,8 +203,7 @@ test_e2e_rend_circuit_setup(void *arg)
     tt_int_op(0, OP_EQ, ed25519_secret_key_generate(&sk, 0));
     tt_int_op(0, OP_EQ, ed25519_public_key_generate(&service_pk, &sk));
 
-    or_circ->hs_ident = hs_ident_circuit_new(&service_pk,
-                                             HS_IDENT_CIRCUIT_RENDEZVOUS);
+    or_circ->hs_ident = hs_ident_circuit_new(&service_pk);
 
     TO_CIRCUIT(or_circ)->state = CIRCUIT_STATE_OPEN;
   }
@@ -183,9 +215,8 @@ test_e2e_rend_circuit_setup(void *arg)
   /* Setup the circuit: do the ntor key exchange */
   {
     uint8_t ntor_key_seed[DIGEST256_LEN] = {2};
-    retval = hs_circuit_setup_e2e_rend_circ(or_circ,
-                                       ntor_key_seed, sizeof(ntor_key_seed),
-                                       1);
+    retval = hs_circuit_setup_e2e_rend_circ(or_circ, ntor_key_seed,
+                                            sizeof(ntor_key_seed), 1);
     tt_int_op(retval, OP_EQ, 0);
   }
 
@@ -194,12 +225,12 @@ test_e2e_rend_circuit_setup(void *arg)
   tt_int_op(retval, OP_EQ, 1);
 
   /* Check the digest algo */
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.f_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.f_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->crypto.b_digest),
+  tt_int_op(crypto_digest_get_algorithm(or_circ->cpath->pvt_crypto.b_digest),
             OP_EQ, DIGEST_SHA3_256);
-  tt_assert(or_circ->cpath->crypto.f_crypto);
-  tt_assert(or_circ->cpath->crypto.b_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.f_crypto);
+  tt_assert(or_circ->cpath->pvt_crypto.b_crypto);
 
   /* Ensure that circ purpose was changed */
   tt_int_op(or_circ->base_.purpose, OP_EQ, CIRCUIT_PURPOSE_S_REND_JOINED);
@@ -268,7 +299,7 @@ helper_clone_authorized_client(const hs_service_authorized_client_t *client)
 
 /* Helper: Return a newly allocated service object with the identity keypair
  * sets and the current descriptor. Then register it to the global map.
- * Caller should us hs_free_all() to free this service or remove it from the
+ * Caller should use hs_free_all() to free this service or remove it from the
  * global map before freeing. */
 static hs_service_t *
 helper_create_service(void)
@@ -290,6 +321,20 @@ helper_create_service(void)
   return service;
 }
 
+/* Helper: Deallocate a given service object, its child objects and
+ * remove it from onion service map.
+ * */
+static void
+helper_destroy_service(hs_service_t *service)
+{
+  if (!service)
+    return;
+
+  remove_service(get_hs_service_map(), service);
+
+  hs_service_free(service);
+}
+
 /* Helper: Return a newly allocated service object with clients. */
 static hs_service_t *
 helper_create_service_with_clients(int num_clients)
@@ -315,17 +360,18 @@ helper_create_service_with_clients(int num_clients)
 static hs_service_intro_point_t *
 helper_create_service_ip(void)
 {
-  hs_desc_link_specifier_t *ls;
-  hs_service_intro_point_t *ip = service_intro_point_new(NULL, 0, 0);
+  link_specifier_t *ls;
+  hs_service_intro_point_t *ip = service_intro_point_new(NULL);
   tor_assert(ip);
   /* Add a first unused link specifier. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_IPV4;
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_IPV4);
   smartlist_add(ip->base.link_specifiers, ls);
   /* Add a second link specifier used by a test. */
-  ls = tor_malloc_zero(sizeof(*ls));
-  ls->type = LS_LEGACY_ID;
-  memset(ls->u.legacy_id, 'A', sizeof(ls->u.legacy_id));
+  ls = link_specifier_new();
+  link_specifier_set_ls_type(ls, LS_LEGACY_ID);
+  memset(link_specifier_getarray_un_legacy_id(ls), 'A',
+         link_specifier_getlen_un_legacy_id(ls));
   smartlist_add(ip->base.link_specifiers, ls);
 
   return ip;
@@ -379,11 +425,11 @@ test_load_keys(void *arg)
   tt_assert(s);
 
   /* Ok we have the service object. Validate few things. */
-  tt_assert(!tor_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
+  tt_assert(!fast_mem_is_zero(s->onion_address, sizeof(s->onion_address)));
   tt_int_op(hs_address_is_valid(s->onion_address), OP_EQ, 1);
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_sk.seckey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_sk.seckey,
                              ED25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) s->keys.identity_pk.pubkey,
+  tt_assert(!fast_mem_is_zero((char *) s->keys.identity_pk.pubkey,
                              ED25519_PUBKEY_LEN));
   /* Check onion address from identity key. */
   hs_build_address(&s->keys.identity_pk, s->config.version, addr);
@@ -657,13 +703,15 @@ test_service_intro_point(void *arg)
 
   (void) arg;
 
+  update_approx_time(1481621834);
+
   /* Test simple creation of an object. */
   {
-    time_t now = time(NULL);
+    time_t now = approx_time();
     ip = helper_create_service_ip();
     tt_assert(ip);
     /* Make sure the authentication keypair is not zeroes. */
-    tt_int_op(tor_mem_is_zero((const char *) &ip->auth_key_kp,
+    tt_int_op(fast_mem_is_zero((const char *) &ip->auth_key_kp,
                               sizeof(ed25519_keypair_t)), OP_EQ, 0);
     /* The introduce2_max MUST be in that range. */
     tt_u64_op(ip->introduce2_max, OP_GE,
@@ -798,10 +846,11 @@ test_helper_functions(void *arg)
     const node_t *node = get_node_from_intro_point(ip);
     tt_ptr_op(node, OP_EQ, &mock_node);
     SMARTLIST_FOREACH_BEGIN(ip->base.link_specifiers,
-                            hs_desc_link_specifier_t *, ls) {
-      if (ls->type == LS_LEGACY_ID) {
+                            link_specifier_t *, ls) {
+      if (link_specifier_get_ls_type(ls) == LS_LEGACY_ID) {
         /* Change legacy id in link specifier which is not the mock node. */
-        memset(ls->u.legacy_id, 'B', sizeof(ls->u.legacy_id));
+        memset(link_specifier_getarray_un_legacy_id(ls), 'B',
+               link_specifier_getlen_un_legacy_id(ls));
       }
     } SMARTLIST_FOREACH_END(ls);
     node = get_node_from_intro_point(ip);
@@ -859,6 +908,10 @@ test_helper_functions(void *arg)
 
  done:
   /* This will free the service and all objects associated to it. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_service_free_all();
   UNMOCK(node_get_by_id);
 }
@@ -868,7 +921,7 @@ static void
 test_intro_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   origin_circuit_t *circ = NULL;
 
   (void) arg;
@@ -916,6 +969,10 @@ test_intro_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -930,7 +987,7 @@ test_intro_established(void *arg)
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -985,12 +1042,15 @@ test_intro_established(void *arg)
   /* Send an empty payload. INTRO_ESTABLISHED cells are basically zeroes. */
   ret = hs_service_receive_intro_established(circ, payload, sizeof(payload));
   tt_int_op(ret, OP_EQ, 0);
-  tt_u64_op(ip->circuit_established, OP_EQ, 1);
   tt_int_op(TO_CIRCUIT(circ)->purpose, OP_EQ, CIRCUIT_PURPOSE_S_INTRO);
 
  done:
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1002,7 +1062,7 @@ test_rdv_circuit_opened(void *arg)
 {
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
 
   (void) arg;
 
@@ -1033,6 +1093,10 @@ test_rdv_circuit_opened(void *arg)
 
  done:
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
   UNMOCK(relay_send_command_from_edge_);
@@ -1070,8 +1134,7 @@ test_closing_intro_circs(void *arg)
 
   /* Initialize intro circuit */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   /* Register circuit in the circuitmap . */
   hs_circuitmap_register_intro_circ_v3_service_side(intro_circ,
                                                     &ip->auth_key_kp.pubkey);
@@ -1097,8 +1160,7 @@ test_closing_intro_circs(void *arg)
   /* Now pretend that a new intro point circ was launched and opened. Check
    * that the intro point will be established correctly. */
   intro_circ = origin_circuit_init(CIRCUIT_PURPOSE_S_ESTABLISH_INTRO, flags);
-  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk,
-                                              HS_IDENT_CIRCUIT_INTRO);
+  intro_circ->hs_ident = hs_ident_circuit_new(&service->keys.identity_pk);
   ed25519_pubkey_copy(&intro_circ->hs_ident->intro_auth_pk,
                       &ip->auth_key_kp.pubkey);
   /* Register circuit in the circuitmap . */
@@ -1120,19 +1182,23 @@ test_closing_intro_circs(void *arg)
     circuit_free_(TO_CIRCUIT(intro_circ));
   }
   /* Frees the service object. */
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(assert_circuit_ok);
 }
 
 /** Test sending and receiving introduce2 cells */
 static void
-test_introduce2(void *arg)
+test_bad_introduce2(void *arg)
 {
   int ret;
   int flags = CIRCLAUNCH_NEED_UPTIME | CIRCLAUNCH_IS_INTERNAL;
   uint8_t payload[RELAY_PAYLOAD_SIZE] = {0};
   origin_circuit_t *circ = NULL;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip = NULL;
 
   (void) arg;
@@ -1142,7 +1208,7 @@ test_introduce2(void *arg)
   MOCK(get_or_state,
        get_or_state_replacement);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   circ = helper_create_origin_circuit(CIRCUIT_PURPOSE_S_INTRO, flags);
   tt_assert(circ);
@@ -1199,6 +1265,10 @@ test_introduce2(void *arg)
   dummy_state = NULL;
   if (circ)
     circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1222,6 +1292,7 @@ test_service_event(void *arg)
 
   /* Set a service for this circuit. */
   service = helper_create_service();
+  tt_assert(service);
   ed25519_pubkey_copy(&circ->hs_ident->identity_pk,
                       &service->keys.identity_pk);
 
@@ -1253,7 +1324,6 @@ test_service_event(void *arg)
      * descriptor map so we can retry it. */
     ip = helper_create_service_ip();
     service_intro_point_add(service->desc_current->intro_points.map, ip);
-    ip->circuit_established = 1;  /* We'll test that, it MUST be 0 after. */
     run_housekeeping_event(now);
     tt_int_op(digest256map_size(service->desc_current->intro_points.map),
               OP_EQ, 1);
@@ -1283,6 +1353,10 @@ test_service_event(void *arg)
  done:
   hs_circuitmap_remove_circuit(TO_CIRCUIT(circ));
   circuit_free_(TO_CIRCUIT(circ));
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(circuit_mark_for_close_);
 }
@@ -1293,12 +1367,12 @@ test_rotate_descriptors(void *arg)
 {
   int ret;
   time_t next_rotation_time, now;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_descriptor_t *desc_next;
 
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   hs_init();
   MOCK(get_or_state, get_or_state_replacement);
@@ -1314,7 +1388,7 @@ test_rotate_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
   now = mock_ns.valid_after+1;
@@ -1355,7 +1429,7 @@ test_rotate_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 27 Oct 1985 02:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
   now = mock_ns.valid_after+1;
@@ -1385,6 +1459,10 @@ test_rotate_descriptors(void *arg)
   tt_assert(service->desc_next);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   UNMOCK(get_or_state);
   UNMOCK(circuit_mark_for_close_);
@@ -1397,9 +1475,8 @@ static void
 test_build_update_descriptors(void *arg)
 {
   int ret;
-  time_t now = time(NULL);
   node_t *node;
-  hs_service_t *service;
+  hs_service_t *service = NULL;
   hs_service_intro_point_t *ip_cur, *ip_next;
   routerinfo_t ri;
 
@@ -1412,7 +1489,7 @@ test_build_update_descriptors(void *arg)
   MOCK(networkstatus_get_reasonably_live_consensus,
        mock_networkstatus_get_reasonably_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1420,10 +1497,11 @@ test_build_update_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
-  now = mock_ns.valid_after+1;
+
+  time_t now = mock_ns.valid_after+1;
 
   /* Create a service without a current descriptor to trigger a build. */
   service = helper_create_service();
@@ -1547,9 +1625,9 @@ test_build_update_descriptors(void *arg)
   tt_int_op(smartlist_len(ip_cur->base.link_specifiers), OP_EQ, 3);
   /* Make sure we have a valid encryption keypair generated when we pick an
    * intro point in the update process. */
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.seckey.secret_key,
                              CURVE25519_SECKEY_LEN));
-  tt_assert(!tor_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
+  tt_assert(!fast_mem_is_zero((char *) ip_cur->enc_key_kp.pubkey.public_key,
                              CURVE25519_PUBKEY_LEN));
   tt_u64_op(ip_cur->time_to_expire, OP_GE, now +
             INTRO_POINT_LIFETIME_MIN_SECONDS);
@@ -1615,6 +1693,10 @@ test_build_update_descriptors(void *arg)
   tt_u64_op(service->desc_next->next_upload_time, OP_EQ, 0);
 
  done:
+  if (service) {
+    remove_service(get_hs_service_map(), service);
+    hs_service_free(service);
+  }
   hs_free_all();
   nodelist_free_all();
 }
@@ -1627,6 +1709,7 @@ test_build_descriptors(void *arg)
 {
   int ret;
   time_t now = time(NULL);
+  hs_service_t *last_service = NULL;
 
   (void) arg;
 
@@ -1637,7 +1720,7 @@ test_build_descriptors(void *arg)
   MOCK(networkstatus_get_reasonably_live_consensus,
        mock_networkstatus_get_reasonably_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 03:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1645,25 +1728,33 @@ test_build_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 04:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   /* Generate a valid number of fake auth clients when a client authorization
    * is disabled. */
   {
     hs_service_t *service = helper_create_service();
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
     build_all_descriptors(now);
+    tt_assert(service->desc_current);
+    tt_assert(service->desc_current->desc);
+
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(0);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1671,12 +1762,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 16);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Generate a valid number of fake auth clients when the number of
    * clients is not a multiple of 16. */
   {
     hs_service_t *service = helper_create_service_with_clients(20);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1684,12 +1779,16 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
   /* Do not generate any fake desc client when the number of clients is
    * a multiple of 16 but not zero. */
   {
     hs_service_t *service = helper_create_service_with_clients(32);
+    last_service = service;
     service_descriptor_free(service->desc_current);
     service->desc_current = NULL;
 
@@ -1697,9 +1796,13 @@ test_build_descriptors(void *arg)
     hs_desc_superencrypted_data_t *superencrypted;
     superencrypted = &service->desc_current->desc->superencrypted_data;
     tt_int_op(smartlist_len(superencrypted->clients), OP_EQ, 32);
+
+    helper_destroy_service(service);
+    last_service = NULL;
   }
 
  done:
+  helper_destroy_service(last_service);
   hs_free_all();
 }
 
@@ -1718,7 +1821,7 @@ test_upload_descriptors(void *arg)
   MOCK(networkstatus_get_reasonably_live_consensus,
        mock_networkstatus_get_reasonably_live_consensus);
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
                            &mock_ns.valid_after);
@@ -1726,7 +1829,7 @@ test_upload_descriptors(void *arg)
   ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
                            &mock_ns.fresh_until);
   tt_int_op(ret, OP_EQ, 0);
-  voting_schedule_recalculate_timing(get_options(), mock_ns.valid_after);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
 
   update_approx_time(mock_ns.valid_after+1);
   now = mock_ns.valid_after+1;
@@ -1848,9 +1951,9 @@ test_rendezvous1_parsing(void *arg)
   }
 
   /* Send out the RENDEZVOUS1 and make sure that our mock func worked */
-  tt_assert(tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(fast_mem_is_zero(rend1_payload, 32));
   hs_circ_service_rp_has_opened(service, service_circ);
-  tt_assert(!tor_mem_is_zero(rend1_payload, 32));
+  tt_assert(!fast_mem_is_zero(rend1_payload, 32));
   tt_int_op(rend1_payload_len, OP_EQ, HS_LEGACY_RENDEZVOUS_CELL_SIZE);
 
   /******************************/
@@ -2098,6 +2201,490 @@ test_export_client_circuit_id(void *arg)
   tor_free(cp2);
 }
 
+static smartlist_t *
+mock_node_get_link_specifier_smartlist(const node_t *node, bool direct_conn)
+{
+  (void) node;
+  (void) direct_conn;
+
+  smartlist_t *lspecs = smartlist_new();
+  link_specifier_t *ls_legacy = link_specifier_new();
+  smartlist_add(lspecs, ls_legacy);
+
+  return lspecs;
+}
+
+static node_t *fake_node = NULL;
+
+static const node_t *
+mock_build_state_get_exit_node(cpath_build_state_t *state)
+{
+  (void) state;
+
+  if (!fake_node) {
+    curve25519_secret_key_t seckey;
+    curve25519_secret_key_generate(&seckey, 0);
+
+    fake_node = tor_malloc_zero(sizeof(node_t));
+    fake_node->ri = tor_malloc_zero(sizeof(routerinfo_t));
+    fake_node->ri->onion_curve25519_pkey =
+      tor_malloc_zero(sizeof(curve25519_public_key_t));
+    curve25519_public_key_generate(fake_node->ri->onion_curve25519_pkey,
+                                   &seckey);
+  }
+
+  return fake_node;
+}
+
+static void
+mock_launch_rendezvous_point_circuit(const hs_service_t *service,
+                                     const hs_service_intro_point_t *ip,
+                                     const hs_cell_introduce2_data_t *data)
+{
+  (void) service;
+  (void) ip;
+  (void) data;
+  return;
+}
+
+/**
+ *  Test that INTRO2 cells are handled well by onion services in the normal
+ *  case and also when onionbalance is enabled.
+ */
+static void
+test_intro2_handling(void *arg)
+{
+  (void)arg;
+
+  MOCK(build_state_get_exit_node, mock_build_state_get_exit_node);
+  MOCK(relay_send_command_from_edge_, mock_relay_send_command_from_edge);
+  MOCK(node_get_link_specifier_smartlist,
+       mock_node_get_link_specifier_smartlist);
+  MOCK(launch_rendezvous_point_circuit, mock_launch_rendezvous_point_circuit);
+
+  memset(relay_payload, 0, sizeof(relay_payload));
+
+  int retval;
+  time_t now = 0101010101;
+  update_approx_time(now);
+
+  /** OK this is the play:
+   *
+   *  In Act I, we have a standalone onion service X (without onionbalance
+   *  enabled). We test that X can properly handle INTRO2 cells sent by a
+   *  client Alice.
+   *
+   *  In Act II, we create an onionbalance setup with frontend being Z which
+   *  includes instances X and Y. We then setup onionbalance on X and test that
+   *  Alice who addresses Z can communicate with X through INTRO2 cells.
+   *
+   *  In Act III, we test that Alice can also communicate with X
+   *  directly even tho onionbalance is enabled.
+   *
+   *  And finally in Act IV, we check various cases where the INTRO2 cell
+   *  should not go through because the subcredentials don't line up
+   *  (e.g. Alice sends INTRO2 to X using Y's subcredential).
+   */
+
+  /** Let's start with some setup! Create the instances and the frontend
+      service, create Alice, etc:  */
+
+  /* Create instance X */
+  hs_service_t x_service;
+  memset(&x_service, 0, sizeof(hs_service_t));
+  /* Disable onionbalance */
+  x_service.config.ob_master_pubkeys = NULL;
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0,0);
+
+  /* Create subcredential for x: */
+  ed25519_keypair_t x_identity_keypair;
+  hs_subcredential_t x_subcred;
+  ed25519_keypair_generate(&x_identity_keypair, 0);
+  hs_helper_get_subcred_from_identity_keypair(&x_identity_keypair,
+                                              &x_subcred);
+
+  /* Create the x instance's intro point */
+  hs_service_intro_point_t *x_ip = NULL;
+  {
+    curve25519_secret_key_t seckey;
+    curve25519_public_key_t pkey;
+    curve25519_secret_key_generate(&seckey, 0);
+    curve25519_public_key_generate(&pkey, &seckey);
+
+    node_t intro_node;
+    memset(&intro_node, 0, sizeof(intro_node));
+    routerinfo_t ri;
+    memset(&ri, 0, sizeof(routerinfo_t));
+    ri.onion_curve25519_pkey = &pkey;
+    intro_node.ri = &ri;
+
+    x_ip = service_intro_point_new(&intro_node);
+  }
+
+  /* Create z frontend's subcredential */
+  ed25519_keypair_t z_identity_keypair;
+  hs_subcredential_t z_subcred;
+  ed25519_keypair_generate(&z_identity_keypair, 0);
+  hs_helper_get_subcred_from_identity_keypair(&z_identity_keypair,
+                                              &z_subcred);
+
+  /* Create y instance's subcredential */
+  ed25519_keypair_t y_identity_keypair;
+  hs_subcredential_t y_subcred;
+  ed25519_keypair_generate(&y_identity_keypair, 0);
+  hs_helper_get_subcred_from_identity_keypair(&y_identity_keypair,
+                                              &y_subcred);
+
+  /* Create Alice's intro point */
+  hs_desc_intro_point_t *alice_ip;
+  ed25519_keypair_t signing_kp;
+  ed25519_keypair_generate(&signing_kp, 0);
+  alice_ip = hs_helper_build_intro_point(&signing_kp, now, "1.2.3.4", 0,
+                                         &x_ip->auth_key_kp,
+                                         &x_ip->enc_key_kp);
+
+  /* Create Alice's intro and rend circuits */
+  origin_circuit_t *intro_circ = origin_circuit_new();
+  intro_circ->cpath = tor_malloc_zero(sizeof(crypt_path_t));
+  intro_circ->cpath->prev = intro_circ->cpath;
+  intro_circ->hs_ident = tor_malloc_zero(sizeof(*intro_circ->hs_ident));
+  origin_circuit_t rend_circ;
+  rend_circ.hs_ident = tor_malloc_zero(sizeof(*rend_circ.hs_ident));
+  curve25519_keypair_generate(&rend_circ.hs_ident->rendezvous_client_kp, 0);
+  memset(rend_circ.hs_ident->rendezvous_cookie, 'r', HS_REND_COOKIE_LEN);
+
+  /* ************************************************************ */
+
+  /* Act I:
+   *
+   * Where Alice connects to X without onionbalance in the picture */
+
+  /* Create INTRODUCE1 */
+  tt_assert(fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &x_subcred);
+
+  /* Check that the payload was written successfully */
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  /* Handle the cell */
+  retval = hs_circ_handle_introduce2(&x_service,
+                                    intro_circ, x_ip,
+                                    &x_subcred,
+                                    (uint8_t*)relay_payload,relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* ************************************************************ */
+
+  /* Act II:
+   *
+   * We now create an onionbalance setup with Z being the frontend and X and Y
+   * being the backend instances. Make sure that Alice can talk with the
+   * backend instance X even tho she thinks she is talking to the frontend Z.
+   */
+
+  /* Now configure the X instance to do onionbalance with Z as the frontend */
+  x_service.config.ob_master_pubkeys = smartlist_new();
+  smartlist_add(x_service.config.ob_master_pubkeys,
+                &z_identity_keypair.pubkey);
+
+  /* Create descriptors for x and load next descriptor with the x's
+   * subcredential so that it can accept connections for itself. */
+  x_service.desc_current = service_descriptor_new();
+  memset(x_service.desc_current->desc->subcredential.subcred, 'C',SUBCRED_LEN);
+  x_service.desc_next = service_descriptor_new();
+  memcpy(&x_service.desc_next->desc->subcredential, &x_subcred, SUBCRED_LEN);
+
+  /* Refresh OB keys */
+  hs_ob_refresh_keys(&x_service);
+
+  /* Create INTRODUCE1 from Alice to X through Z */
+  memset(relay_payload, 0, sizeof(relay_payload));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &z_subcred);
+
+  /* Check that the payload was written successfully */
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  /* Deliver INTRODUCE1 to X even tho it carries Z's subcredential */
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &z_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  replaycache_free(x_ip->replay_cache);
+  x_ip->replay_cache = replaycache_new(0, 0);
+
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+
+  /* ************************************************************ */
+
+  /* Act III:
+   *
+   * Now send a direct INTRODUCE cell from Alice to X using X's subcredential
+   * and check that it succeeds even with onionbalance enabled.
+   */
+
+  /* Refresh OB keys (just to check for memleaks) */
+  hs_ob_refresh_keys(&x_service);
+
+  /* Create INTRODUCE1 from Alice to X using X's subcred. */
+  memset(relay_payload, 0, sizeof(relay_payload));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &x_subcred);
+
+  /* Check that the payload was written successfully */
+  tt_int_op(retval, OP_EQ, 0);
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  /* Send INTRODUCE1 to X with X's subcredential (should succeed) */
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* ************************************************************ */
+
+  /* Act IV:
+   *
+   * Test cases where the INTRO2 cell should not be able to decode.
+   */
+
+  /* Try sending the exact same INTRODUCE2 cell again and see that the intro
+   * point replay cache triggers: */
+  setup_full_capture_of_logs(LOG_WARN);
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, -1);
+  expect_log_msg_containing("with the same ENCRYPTED section");
+  teardown_capture_of_logs();
+
+  /* Now cleanup the intro point replay cache but not the service replay cache
+     and see that this one triggers this time. */
+  replaycache_free(x_ip->replay_cache);
+  x_ip->replay_cache = replaycache_new(0, 0);
+  setup_full_capture_of_logs(LOG_INFO);
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, -1);
+  expect_log_msg_containing("with same REND_COOKIE");
+  teardown_capture_of_logs();
+
+  /* Now just to make sure cleanup both replay caches and make sure that the
+     cell gets through */
+  replaycache_free(x_ip->replay_cache);
+  x_ip->replay_cache = replaycache_new(0, 0);
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  x_service.state.replay_cache_rend_cookie = replaycache_new(0, 0);
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &x_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* As a final thing, create an INTRODUCE1 cell from Alice to X using Y's
+   * subcred (should fail since Y is just another instance and not the frontend
+   * service!) */
+  memset(relay_payload, 0, sizeof(relay_payload));
+  retval = hs_circ_send_introduce1(intro_circ, &rend_circ,
+                                   alice_ip, &y_subcred);
+  tt_int_op(retval, OP_EQ, 0);
+
+  /* Check that the payload was written successfully */
+  tt_assert(!fast_mem_is_zero(relay_payload, sizeof(relay_payload)));
+  tt_int_op(relay_payload_len, OP_NE, 0);
+
+  retval = hs_circ_handle_introduce2(&x_service,
+                                   intro_circ, x_ip,
+                                   &y_subcred,
+                                   (uint8_t*)relay_payload, relay_payload_len);
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  /* Start cleaning up X */
+  replaycache_free(x_service.state.replay_cache_rend_cookie);
+  smartlist_free(x_service.config.ob_master_pubkeys);
+  tor_free(x_service.state.ob_subcreds);
+  service_descriptor_free(x_service.desc_current);
+  service_descriptor_free(x_service.desc_next);
+  service_intro_point_free(x_ip);
+
+  /* Clean up Alice */
+  hs_desc_intro_point_free(alice_ip);
+  tor_free(rend_circ.hs_ident);
+
+  if (fake_node) {
+    tor_free(fake_node->ri->onion_curve25519_pkey);
+    tor_free(fake_node->ri);
+    tor_free(fake_node);
+  }
+
+  UNMOCK(build_state_get_exit_node);
+  UNMOCK(relay_send_command_from_edge_);
+  UNMOCK(node_get_link_specifier_smartlist);
+  UNMOCK(launch_rendezvous_point_circuit);
+}
+
+static void
+test_cannot_upload_descriptors(void *arg)
+{
+  int ret;
+  time_t now;
+  hs_service_t *service;
+
+  (void) arg;
+
+  hs_init();
+  MOCK(get_or_state,
+       get_or_state_replacement);
+  MOCK(networkstatus_get_reasonably_live_consensus,
+       mock_networkstatus_get_reasonably_live_consensus);
+
+  dummy_state = or_state_new();
+
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 13:00:00 UTC",
+                           &mock_ns.valid_after);
+  tt_int_op(ret, OP_EQ, 0);
+  ret = parse_rfc1123_time("Sat, 26 Oct 1985 14:00:00 UTC",
+                           &mock_ns.fresh_until);
+  tt_int_op(ret, OP_EQ, 0);
+  dirauth_sched_recalculate_timing(get_options(), mock_ns.valid_after);
+
+  update_approx_time(mock_ns.valid_after + 1);
+  now = mock_ns.valid_after + 1;
+
+  /* Create a service with no descriptor. It's added to the global map. */
+  service = hs_service_new(get_options());
+  tt_assert(service);
+  service->config.version = HS_VERSION_THREE;
+  ed25519_secret_key_generate(&service->keys.identity_sk, 0);
+  ed25519_public_key_generate(&service->keys.identity_pk,
+                              &service->keys.identity_sk);
+  /* Register service to global map. */
+  ret = register_service(get_hs_service_map(), service);
+  tt_int_op(ret, OP_EQ, 0);
+  /* But first, build our descriptor. */
+  build_all_descriptors(now);
+
+  /* 1. Testing missing intro points reason. */
+  {
+    digest256map_t *cur = service->desc_current->intro_points.map;
+    digest256map_t *tmp = digest256map_new();
+    service->desc_current->intro_points.map = tmp;
+    service->desc_current->missing_intro_points = 1;
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    digest256map_free(tmp, tor_free_);
+    service->desc_current->intro_points.map = cur;
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Missing intro points");
+    teardown_capture_of_logs();
+    /* Reset. */
+    service->desc_current->missing_intro_points = 0;
+  }
+
+  /* 2. Testing non established intro points. */
+  {
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Intro circuits aren't yet all established (0/3).");
+    teardown_capture_of_logs();
+  }
+
+  /* We need to pass the established circuit tests and thus from now on, we
+   * MOCK this to return 3 intro points. */
+  MOCK(count_desc_circuit_established, mock_count_desc_circuit_established);
+  num_intro_points = 3;
+
+  /* 3. Testing non established intro points. */
+  {
+    service->desc_current->next_upload_time = now + 1000;
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Next upload time is");
+    teardown_capture_of_logs();
+    /* Reset. */
+    service->desc_current->next_upload_time = 0;
+  }
+
+  /* 4. Testing missing live consensus. */
+  {
+    MOCK(networkstatus_get_reasonably_live_consensus,
+         mock_networkstatus_get_reasonably_live_consensus_null);
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "No reasonably live consensus");
+    teardown_capture_of_logs();
+    /* Reset. */
+    MOCK(networkstatus_get_reasonably_live_consensus,
+         mock_networkstatus_get_reasonably_live_consensus);
+  }
+
+  /* 5. Test missing minimum directory information. */
+  {
+    MOCK(router_have_minimum_dir_info,
+         mock_router_have_minimum_dir_info_false);
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Not enough directory information");
+    teardown_capture_of_logs();
+
+    /* Running it again shouldn't trigger anything due to rate limitation. */
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_no_log_entry();
+    teardown_capture_of_logs();
+    UNMOCK(router_have_minimum_dir_info);
+  }
+
+  /* Increase time and redo test (5) in order to test the rate limiting. */
+  update_approx_time(mock_ns.valid_after + 61);
+  {
+    MOCK(router_have_minimum_dir_info,
+         mock_router_have_minimum_dir_info_false);
+    setup_full_capture_of_logs(LOG_INFO);
+    run_upload_descriptor_event(now);
+    expect_log_msg_containing(
+      "Service [scrubbed] can't upload its current descriptor: "
+      "Not enough directory information");
+    teardown_capture_of_logs();
+    UNMOCK(router_have_minimum_dir_info);
+  }
+
+ done:
+  hs_free_all();
+  UNMOCK(count_desc_circuit_established);
+  UNMOCK(networkstatus_get_reasonably_live_consensus);
+  UNMOCK(get_or_state);
+}
+
 struct testcase_t hs_service_tests[] = {
   { "e2e_rend_circuit_setup", test_e2e_rend_circuit_setup, TT_FORK,
     NULL, NULL },
@@ -2123,7 +2710,7 @@ struct testcase_t hs_service_tests[] = {
     NULL, NULL },
   { "rdv_circuit_opened", test_rdv_circuit_opened, TT_FORK,
     NULL, NULL },
-  { "introduce2", test_introduce2, TT_FORK,
+  { "bad_introduce2", test_bad_introduce2, TT_FORK,
     NULL, NULL },
   { "service_event", test_service_event, TT_FORK,
     NULL, NULL },
@@ -2135,12 +2722,15 @@ struct testcase_t hs_service_tests[] = {
     NULL, NULL },
   { "upload_descriptors", test_upload_descriptors, TT_FORK,
     NULL, NULL },
+  { "cannot_upload_descriptors", test_cannot_upload_descriptors, TT_FORK,
+    NULL, NULL },
   { "rendezvous1_parsing", test_rendezvous1_parsing, TT_FORK,
     NULL, NULL },
   { "authorized_client_config_equal", test_authorized_client_config_equal,
     TT_FORK, NULL, NULL },
   { "export_client_circuit_id", test_export_client_circuit_id, TT_FORK,
     NULL, NULL },
+  { "intro2_handling", test_intro2_handling, TT_FORK, NULL, NULL },
 
   END_OF_TESTCASES
 };
diff --git a/src/test/test_introduce.c b/src/test/test_introduce.c
index 4a6d90d97e..0ae78496b2 100644
--- a/src/test/test_introduce.c
+++ b/src/test/test_introduce.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -383,8 +383,10 @@ make_intro_from_plaintext(
 
   /* Output the cell */
   *cell_out = cell;
+  cell = NULL;
 
  done:
+  tor_free(cell);
   return cell_len;
 }
 
@@ -535,4 +537,3 @@ struct testcase_t introduce_tests[] = {
   INTRODUCE_LEGACY(late_parse_v3),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_key_expiration.sh b/src/test/test_key_expiration.sh
index 3474210607..2238f7aa78 100755
--- a/src/test/test_key_expiration.sh
+++ b/src/test/test_key_expiration.sh
@@ -6,14 +6,28 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -21,11 +35,21 @@ if test "$UNAME_OS" = 'CYGWIN' || \
   exit 77
 fi
 
+# find the tor binary
 if [ $# -ge 1 ]; then
   TOR_BINARY="${1}"
   shift
 else
-  TOR_BINARY="${TESTING_TOR_BINARY}"
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "$TOR_BINARY" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
 fi
 
 if [ $# -ge 1 ]; then
@@ -47,11 +71,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_key_expiration_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_key_expiration_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -60,10 +84,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
diff --git a/src/test/test_keygen.sh b/src/test/test_keygen.sh
index 7afff271cb..6812f8883d 100755
--- a/src/test/test_keygen.sh
+++ b/src/test/test_keygen.sh
@@ -6,14 +6,28 @@
 umask 077
 set -e
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   if [ "$TESTING_TOR_BINARY" = "" ] ; then
     echo "Usage: ${0} PATH_TO_TOR [case-number]"
     exit 1
   fi
 fi
 
-UNAME_OS=`uname -s | cut -d_ -f1`
+UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
@@ -21,14 +35,22 @@ if test "$UNAME_OS" = 'CYGWIN' || \
   exit 77
 fi
 
+# find the tor binary
 if [ $# -ge 1 ]; then
   TOR_BINARY="${1}"
   shift
 else
-  TOR_BINARY="${TESTING_TOR_BINARY}"
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
 fi
 
+TOR_BINARY="$(abspath "$TOR_BINARY")"
 
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "$TOR_BINARY" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
 
   if [ $# -ge 1 ]; then
       dflt=0
@@ -64,11 +86,11 @@ dump() { xxd -p "$1" | tr -d '\n '; }
 die() { echo "$1" >&2 ; exit 5; }
 check_dir() { [ -d "$1" ] || die "$1 did not exist"; }
 check_file() { [ -e "$1" ] || die "$1 did not exist"; }
-check_no_file() { [ -e "$1" ] && die "$1 was not supposed to exist" || true; }
-check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: `dump $1` vs `dump $2`"; }
+check_no_file() { if [ -e "$1" ]; then die "$1 was not supposed to exist"; fi }
+check_files_eq() { cmp "$1" "$2" || die "$1 and $2 did not match: $(dump "$1") vs $(dump "$2")"; }
 check_keys_eq() { check_files_eq "${SRC}/keys/${1}" "${ME}/keys/${1}"; }
 
-DATA_DIR=`mktemp -d -t tor_keygen_tests.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_keygen_tests.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -77,10 +99,10 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 # Use an absolute path for this or Tor will complain
-DATA_DIR=`cd "${DATA_DIR}" && pwd`
+DATA_DIR=$(cd "${DATA_DIR}" && pwd)
 
 touch "${DATA_DIR}/empty_torrc"
 touch "${DATA_DIR}/empty_defaults_torrc"
@@ -144,7 +166,9 @@ ME="${DATA_DIR}/case2a"
 SRC="${DATA_DIR}/orig"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Somehow succeeded when missing secret key, certs: `cat ${ME}/stdout`" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Somehow succeeded when missing secret key, certs: $(cat "${ME}/stdout")"
+fi
 check_files_eq "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/ed25519_master_id_public_key"
 
 grep "We needed to load a secret key.*but couldn't find it" "${ME}/stdout" >/dev/null || die "Tor didn't declare that it was missing a secret key"
@@ -281,7 +305,9 @@ SRC="${DATA_DIR}/encrypted"
 mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_secret_key_encrypted" "${ME}/keys/"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
-${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout" && die "Tor started with encrypted secret key and no certs" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint > "${ME}/stdout"; then
+  die "Tor started with encrypted secret key and no certs"
+fi
 check_no_file "${ME}/keys/ed25519_signing_cert"
 check_no_file "${ME}/keys/ed25519_signing_secret_key"
 
@@ -370,7 +396,9 @@ mkdir -p "${ME}/keys"
 cp "${SRC}/keys/ed25519_master_id_public_key" "${ME}/keys/"
 cp "${OTHER}/keys/ed25519_master_id_secret_key" "${ME}/keys/"
 
-${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout" && die "Successfully started with mismatched keys!?" || true
+if ${TOR} --DataDirectory "${ME}" --list-fingerprint >"${ME}/stdout"; then
+  die "Successfully started with mismatched keys!?"
+fi
 
 grep "public_key does not match.*secret_key" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a key mismatch"
 
@@ -386,7 +414,9 @@ ME="${DATA_DIR}/case11a"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout" && die "Successfully started with passphrase-fd but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --passphrase-fd 1 > "${ME}/stdout"; then
+  die "Successfully started with passphrase-fd but no keygen?"
+fi
 
 grep "passphrase-fd specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -402,7 +432,9 @@ ME="${DATA_DIR}/case11b"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout" && die "Successfully started with no-passphrase but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with no-passphrase but no keygen?"
+fi
 
 grep "no-passphrase specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -418,7 +450,9 @@ ME="${DATA_DIR}/case11C"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout" && die "Successfully started with newpass but no keygen?" || true
+if ${TOR} --DataDirectory "${ME}" --newpass > "${ME}/stdout"; then
+  die "Successfully started with newpass but no keygen?"
+fi
 
 grep "newpass specified without --keygen" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -456,7 +490,9 @@ ME="${DATA_DIR}/case11E"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd ewigeblumenkraft > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd?"
+fi
 
 grep "Invalid --passphrase-fd value" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
@@ -473,7 +509,9 @@ ME="${DATA_DIR}/case11F"
 
 mkdir -p "${ME}/keys"
 
-${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout" && die "Successfully started with bogus passphrase-fd combination?" || true
+if ${TOR} --DataDirectory "${ME}" --keygen --passphrase-fd 1 --no-passphrase > "${ME}/stdout"; then
+  die "Successfully started with bogus passphrase-fd combination?"
+fi
 
 grep "no-passphrase specified with --passphrase-fd" "${ME}/stdout" >/dev/null || die "Tor didn't declare that there was a problem with the arguments."
 
diff --git a/src/test/test_keypin.c b/src/test/test_keypin.c
index e7beef8609..ff6397f8c7 100644
--- a/src/test/test_keypin.c
+++ b/src/test/test_keypin.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_link_handshake.c b/src/test/test_link_handshake.c
index 34f59f26cd..1566b349ed 100644
--- a/src/test/test_link_handshake.c
+++ b/src/test/test_link_handshake.c
@@ -1,11 +1,11 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #define CHANNELTLS_PRIVATE
 #define CONNECTION_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
+#define CHANNEL_OBJECT_PRIVATE
 #define TORTLS_PRIVATE
 
 #include "core/or/or.h"
@@ -18,6 +18,7 @@
 #include "feature/relay/routerkeys.h"
 #include "core/or/scheduler.h"
 #include "feature/nodelist/torcert.h"
+#include "feature/relay/relay_handshake.h"
 
 #include "core/or/or_connection_st.h"
 #include "core/or/or_handshake_certs_st.h"
@@ -263,7 +264,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_assert(c1->handshake_state->authenticated_rsa);
     tt_assert(! c1->handshake_state->authenticated_ed25519);
   }
-  tt_assert(! tor_mem_is_zero(
+  tt_assert(! fast_mem_is_zero(
                 (char*)c1->handshake_state->authenticated_rsa_peer_id, 20));
 
   chan2 = tor_malloc_zero(sizeof(*chan2));
@@ -290,7 +291,7 @@ test_link_handshake_certs_ok(void *arg)
     tt_ptr_op(c2->handshake_state->certs->ed_id_sign, OP_EQ, NULL);
   }
   tt_assert(c2->handshake_state->certs->id_cert);
-  tt_assert(tor_mem_is_zero(
+  tt_assert(fast_mem_is_zero(
               (char*)c2->handshake_state->authenticated_rsa_peer_id, 20));
   /* no authentication has happened yet, since we haen't gotten an AUTH cell.
    */
@@ -325,7 +326,7 @@ test_link_handshake_certs_ok(void *arg)
   crypto_pk_free(key2);
 }
 
-typedef struct certs_data_s {
+typedef struct certs_data_t {
   int is_ed;
   int is_link_cert;
   or_connection_t *c;
@@ -948,7 +949,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_int_op(36, OP_EQ, cell1->payload_len);
   tt_int_op(36, OP_EQ, cell2->payload_len);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_int_op(0, OP_EQ, cell1->circ_id);
   tt_int_op(0, OP_EQ, cell2->circ_id);
   tt_int_op(CELL_AUTH_CHALLENGE, OP_EQ, cell1->command);
@@ -960,7 +961,7 @@ test_link_handshake_send_authchallenge(void *arg)
 #else
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell1->payload + 32, 4);
   tt_mem_op("\x00\x01\x00\x03", OP_EQ, cell2->payload + 32, 4);
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   tt_mem_op(cell1->payload, OP_NE, cell2->payload, 32);
 
  done:
@@ -972,7 +973,7 @@ test_link_handshake_send_authchallenge(void *arg)
   crypto_pk_free(rsa1);
 }
 
-typedef struct authchallenge_data_s {
+typedef struct authchallenge_data_t {
   or_connection_t *c;
   channel_tls_t *chan;
   var_cell_t *cell;
@@ -1171,7 +1172,7 @@ mock_set_circid_type(channel_t *chan,
   (void) consider_identity;
 }
 
-typedef struct authenticate_data_s {
+typedef struct authenticate_data_t {
   int is_ed;
   or_connection_t *c1, *c2;
   channel_tls_t *chan2;
@@ -1492,6 +1493,7 @@ AUTHENTICATE_FAIL(missing_ed_auth,
                     "authentication certificate";
                 })
 
+#ifndef COCCI
 #define TEST_RSA(name, flags)                                           \
   { #name , test_link_handshake_ ## name, (flags),                      \
       &passthrough_setup, (void*)"RSA" }
@@ -1527,6 +1529,7 @@ AUTHENTICATE_FAIL(missing_ed_auth,
 #define TEST_AUTHENTICATE_ED(name)                                      \
   { "authenticate/" #name "_ed25519" , test_link_handshake_auth_ ## name, \
       TT_FORK, &setup_authenticate, (void*)3 }
+#endif /* !defined(COCCI) */
 
 struct testcase_t link_handshake_tests[] = {
   TEST_RSA(certs_ok, TT_FORK),
diff --git a/src/test/test_logging.c b/src/test/test_logging.c
index 95a2fce757..e09f7a21cd 100644
--- a/src/test/test_logging.c
+++ b/src/test/test_logging.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
@@ -9,14 +9,13 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 
 static void
-dummy_cb_fn(int severity, uint32_t domain, const char *msg)
+dummy_cb_fn(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)severity; (void)domain; (void)msg;
 }
@@ -36,7 +35,7 @@ test_get_sigsafe_err_fds(void *arg)
 
   set_log_severity_config(LOG_WARN, LOG_ERR, &include_bug);
   set_log_severity_config(LOG_WARN, LOG_ERR, &no_bug);
-  no_bug.masks[0] &= ~(LD_BUG|LD_GENERAL);
+  no_bug.masks[SEVERITY_MASK_IDX(LOG_ERR)] &= ~(LD_BUG|LD_GENERAL);
   set_log_severity_config(LOG_INFO, LOG_NOTICE, &no_bug2);
 
   /* Add some logs; make sure the output is as expected. */
@@ -117,22 +116,27 @@ test_sigsafe_err(void *arg)
   content = read_file_to_str(fn, 0, NULL);
 
   tt_ptr_op(content, OP_NE, NULL);
-  tor_split_lines(lines, content, (int)strlen(content));
+  smartlist_split_string(lines, content, "\n", 0, 0);
   tt_int_op(smartlist_len(lines), OP_GE, 5);
 
-  if (strstr(smartlist_get(lines, 0), "opening new log file"))
+  if (strstr(smartlist_get(lines, 0), "opening new log file")) {
+    void *item = smartlist_get(lines, 0);
     smartlist_del_keeporder(lines, 0);
+    tor_free(item);
+  }
+
   tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool"));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 1), "=============="));
-  tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal."));
-  /* Next line is blank. */
-  tt_assert(!strcmpstart(smartlist_get(lines, 3), "=============="));
-  tt_str_op(smartlist_get(lines, 4), OP_EQ,
+  tt_str_op(smartlist_get(lines, 1), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 2), "=============="));
+  tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal."));
+  tt_str_op(smartlist_get(lines, 4), OP_EQ, "");
+  tt_assert(!strcmpstart(smartlist_get(lines, 5), "=============="));
+  tt_str_op(smartlist_get(lines, 6), OP_EQ,
             "Testing any attempt to manually log from a signal.");
 
  done:
   tor_free(content);
+  SMARTLIST_FOREACH(lines, char *, x, tor_free(x));
   smartlist_free(lines);
 }
 
diff --git a/src/test/test_mainloop.c b/src/test/test_mainloop.c
index 089ea812cf..c4e60d9da5 100644
--- a/src/test/test_mainloop.c
+++ b/src/test/test_mainloop.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,11 +6,29 @@
  * \brief Tests for functions closely related to the Tor main loop
  */
 
+#define CONFIG_PRIVATE
+#define MAINLOOP_PRIVATE
+#define STATEFILE_PRIVATE
+
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
+#include "lib/confmgt/confmgt.h"
+
 #include "core/or/or.h"
+#include "core/mainloop/connection.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/mainloop_state_st.h"
+#include "core/mainloop/mainloop_sys.h"
+#include "core/mainloop/netstatus.h"
+
+#include "feature/hs/hs_service.h"
+
+#include "app/config/config.h"
+#include "app/config/statefile.h"
+#include "app/config/or_state_st.h"
+
+#include "app/main/subsysmgr.h"
 
 static const uint64_t BILLION = 1000000000;
 
@@ -131,12 +149,233 @@ test_mainloop_update_time_jumps(void *arg)
   monotime_disable_test_mocking();
 }
 
+static int schedule_rescan_called = 0;
+static void
+mock_schedule_rescan_periodic_events(void)
+{
+  ++schedule_rescan_called;
+}
+
+static void
+test_mainloop_user_activity(void *arg)
+{
+  (void)arg;
+  const time_t start = 1542658829;
+  update_approx_time(start);
+
+  MOCK(schedule_rescan_periodic_events, mock_schedule_rescan_periodic_events);
+
+  reset_user_activity(start);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  set_network_participation(false);
+
+  // reset can move backwards and forwards, but does not change network
+  // participation.
+  reset_user_activity(start-10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-10);
+  reset_user_activity(start+10);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  tt_int_op(schedule_rescan_called, OP_EQ, 0);
+  tt_int_op(false, OP_EQ, is_participating_on_network());
+
+  // "note" can only move forward.  Calling it from a non-participating
+  // state makes us rescan the periodic callbacks and set participation.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+20);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // Calling it again will move us forward, but not call rescan again.
+  note_user_activity(start+25);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We won't move backwards.
+  note_user_activity(start+20);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+25);
+  tt_int_op(true, OP_EQ, is_participating_on_network());
+  tt_int_op(schedule_rescan_called, OP_EQ, 1);
+
+  // We _will_ adjust if the clock jumps though.
+  netstatus_note_clock_jumped(500);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+525);
+
+  netstatus_note_clock_jumped(-400);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+125);
+
+ done:
+  UNMOCK(schedule_rescan_periodic_events);
+}
+
+static unsigned int
+mock_get_num_services(void)
+{
+  return 1;
+}
+
+static connection_t *
+mock_connection_gbtu(int type)
+{
+  (void) type;
+  return (void *)"hello fellow connections";
+}
+
+static void
+test_mainloop_check_participation(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  const time_t start = 1542658829;
+  const time_t ONE_DAY = 24*60*60;
+
+  // Suppose we've been idle for a day or two
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  check_network_participation_callback(start, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+  // suppose we've been idle for 2 days... but we are a server.
+  reset_user_activity(start - 2*ONE_DAY);
+  options->ORPort_set = 1;
+  set_network_participation(true);
+  check_network_participation_callback(start+2, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+2);
+  options->ORPort_set = 0;
+
+  // idle for 2 days, but we have a hidden service.
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  MOCK(hs_service_get_num_services, mock_get_num_services);
+  check_network_participation_callback(start+3, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+3);
+  UNMOCK(hs_service_get_num_services);
+
+  // idle for 2 days but we have at least one user connection
+  MOCK(connection_get_by_type_nonlinked, mock_connection_gbtu);
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+  check_network_participation_callback(start+10, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start+10);
+
+  // as above, but DormantTimeoutDisabledByIdleStreams is not set
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantTimeoutDisabledByIdleStreams = 0;
+  check_network_participation_callback(start+13, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, false);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+  UNMOCK(connection_get_by_type_nonlinked);
+  options->DormantTimeoutDisabledByIdleStreams = 1;
+
+  // idle for 2 days but DormantClientTimeout is 3 days
+  reset_user_activity(start - 2*ONE_DAY);
+  set_network_participation(true);
+  options->DormantClientTimeout = ONE_DAY * 3;
+  check_network_participation_callback(start+30, options);
+  tt_int_op(is_participating_on_network(), OP_EQ, true);
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start-2*ONE_DAY);
+
+ done:
+  or_options_free(options);
+  UNMOCK(hs_service_get_num_services);
+  UNMOCK(connection_get_by_type_nonlinked);
+}
+
+static void
+test_mainloop_dormant_load_state(void *arg)
+{
+  (void)arg;
+  or_state_t *or_state = or_state_new();
+  mainloop_state_t *state;
+  {
+    int idx = subsystems_get_state_idx(&sys_mainloop);
+    tor_assert(idx >= 0);
+    state = config_mgr_get_obj_mutable(get_state_mgr(), or_state, idx);
+  }
+  const time_t start = 1543956575;
+
+  reset_user_activity(0);
+  set_network_participation(false);
+
+  // When we construct a new state, it starts out in "auto" mode.
+  tt_int_op(state->Dormant, OP_EQ, -1);
+
+  // Initializing from "auto" makes us start out (by default) non-Dormant,
+  // with activity right now.
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+  // Initializing from dormant clears the last user activity time, and
+  // makes us dormant.
+  state->Dormant = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(! is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, 0);
+
+  // Initializing from non-dormant sets the last user activity time, and
+  // makes us non-dormant.
+  state->Dormant = 0;
+  state->MinutesSinceUserActivity = 123;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start - 123*60);
+
+  // If we would start dormant, but DormantCanceledByStartup is set, then
+  // we start up non-dormant.
+  state->Dormant = 1;
+  get_options_mutable()->DormantCanceledByStartup = 1;
+  netstatus_load_from_state(state, start);
+  tt_assert(is_participating_on_network());
+  tt_i64_op(get_last_user_activity_time(), OP_EQ, start);
+
+ done:
+  or_state_free(or_state);
+}
+
+static void
+test_mainloop_dormant_save_state(void *arg)
+{
+  (void)arg;
+  mainloop_state_t *state = tor_malloc_zero(sizeof(mainloop_state_t));
+  const time_t start = 1543956575;
+
+  // Can we save a non-dormant state correctly?
+  reset_user_activity(start - 1000);
+  set_network_participation(true);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 0);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 1000 / 60);
+
+  // Can we save a dormant state correctly?
+  set_network_participation(false);
+  netstatus_flush_to_state(state, start);
+
+  tt_int_op(state->Dormant, OP_EQ, 1);
+  tt_int_op(state->MinutesSinceUserActivity, OP_EQ, 0);
+
+ done:
+  tor_free(state);
+}
+
 #define MAINLOOP_TEST(name) \
   { #name, test_mainloop_## name , TT_FORK, NULL, NULL }
 
 struct testcase_t mainloop_tests[] = {
   MAINLOOP_TEST(update_time_normal),
   MAINLOOP_TEST(update_time_jumps),
+  MAINLOOP_TEST(user_activity),
+  MAINLOOP_TEST(check_participation),
+  MAINLOOP_TEST(dormant_load_state),
+  MAINLOOP_TEST(dormant_save_state),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_microdesc.c b/src/test/test_microdesc.c
index 4c4317d81a..f89025aa6c 100644
--- a/src/test/test_microdesc.c
+++ b/src/test/test_microdesc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -11,6 +11,7 @@
 #include "feature/dirparse/routerparse.h"
 #include "feature/nodelist/microdesc.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/routerlist.h"
 #include "feature/nodelist/torcert.h"
 
@@ -20,6 +21,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #ifdef HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -70,6 +72,7 @@ test_md_cache(void *data)
   const char *test_md3_noannotation = strchr(test_md3, '\n')+1;
   time_t time1, time2, time3;
   char *fn = NULL, *s = NULL;
+  char *encoded_family = NULL;
   (void)data;
 
   options = get_options_mutable();
@@ -172,8 +175,9 @@ test_md_cache(void *data)
 
   tt_ptr_op(md1->family, OP_EQ, NULL);
   tt_ptr_op(md3->family, OP_NE, NULL);
-  tt_int_op(smartlist_len(md3->family), OP_EQ, 3);
-  tt_str_op(smartlist_get(md3->family, 0), OP_EQ, "nodeX");
+
+  encoded_family = nodefamily_format(md3->family);
+  tt_str_op(encoded_family, OP_EQ, "nodex nodey nodez");
 
   /* Now rebuild the cache! */
   tt_int_op(microdesc_cache_rebuild(mc, 1), OP_EQ, 0);
@@ -254,6 +258,7 @@ test_md_cache(void *data)
   smartlist_free(wanted);
   tor_free(s);
   tor_free(fn);
+  tor_free(encoded_family);
 }
 
 static const char truncated_md[] =
@@ -417,6 +422,28 @@ static const char test_md2_21[] =
   "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
   "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
 
+static const char test_md2_withfamily_28[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family OtherNode !Strange\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
+static const char test_md2_withfamily_29[] =
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAL2R8EfubUcahxha4u02P4VAR0llQIMwFAmrHPjzcK7apcQgDOf2ovOA\n"
+  "+YQnJFxlpBmCoCZC6ssCi+9G0mqo650lFuTMP5I90BdtjotfzESfTykHLiChyvhd\n"
+  "l0dlqclb2SU/GKem/fLRXH16aNi72CdSUu/1slKs/70ILi34QixRAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "ntor-onion-key hbxdRnfVUJJY7+KcT4E3Rs7/zuClbN3hJrjSBiEGMgI=\n"
+  "family !Strange $B7E27F104213C36F13E7E9829182845E495997A0 othernode\n"
+  "id ed25519 wqfLzgfCtRfYNg88LsL1QpzxS0itapJ1aj6TbnByx/Q\n";
+
 static void
 test_md_generate(void *arg)
 {
@@ -447,13 +474,24 @@ test_md_generate(void *arg)
   tt_assert(ed25519_pubkey_eq(md->ed25519_identity_pkey,
                               &ri->cache_info.signing_key_cert->signing_key));
 
+  // Try family encoding.
+  microdesc_free(md);
+  ri->declared_family = smartlist_new();
+  smartlist_add_strdup(ri->declared_family, "OtherNode !Strange");
+  md = dirvote_create_microdescriptor(ri, 28);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_28);
+
+  microdesc_free(md);
+  md = dirvote_create_microdescriptor(ri, 29);
+  tt_str_op(md->body, OP_EQ, test_md2_withfamily_29);
+
  done:
   microdesc_free(md);
   routerinfo_free(ri);
 }
 
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-DISABLE_GCC_WARNING(overlength-strings)
+DISABLE_GCC_WARNING("-Woverlength-strings")
 /* We allow huge string constants in the unit tests, but not in the code
  * at large. */
 #endif
@@ -611,9 +649,44 @@ static const char MD_PARSE_TEST_DATA[] =
   "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
   "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   "p6 allow 80\n"
+  /* Good 11: Normal, non-exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 12: Normal, exit relay with ipv6 address */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
+  /* Good 13: Normal, exit relay with only ipv6 exit policy */
+  "onion-key\n"
+  "-----BEGIN RSA PUBLIC KEY-----\n"
+  "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+  "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+  "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+  "-----END RSA PUBLIC KEY-----\n"
+  "a [::1:2:3:4]:9090\n"
+  "a 18.0.0.1:9999\n"
+  "ntor-onion-key k2yFqTU2vzMCQDEiE/j9UcEHxKrXMLpB3IL0or09sik=\n"
+  "p6 accept 20-23,43,53,79-81,88,110,143,194,220,389,443,464,531,543-544\n"
+  "id rsa1024 2A8wYpHxnkKJ92orocvIQBzeHlE\n"
   ;
 #ifdef HAVE_CFLAG_WOVERLENGTH_STRINGS
-ENABLE_GCC_WARNING(overlength-strings)
+ENABLE_GCC_WARNING("-Woverlength-strings")
 #endif
 
 /** More tests for parsing different kinds of microdescriptors, and getting
@@ -628,7 +701,7 @@ test_md_parse(void *arg)
   smartlist_t *mds = microdescs_parse_from_string(MD_PARSE_TEST_DATA,
                                                   NULL, 1, SAVED_NOWHERE,
                                                   invalid);
-  tt_int_op(smartlist_len(mds), OP_EQ, 11);
+  tt_int_op(smartlist_len(mds), OP_EQ, 14);
   tt_int_op(smartlist_len(invalid), OP_EQ, 4);
 
   test_memeq_hex(smartlist_get(invalid,0),
@@ -675,6 +748,21 @@ test_md_parse(void *arg)
   tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
   tt_int_op(md->ipv6_orport, OP_EQ, 9090);
 
+  md = smartlist_get(mds, 11);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 1);
+
+  md = smartlist_get(mds, 12);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
+  md = smartlist_get(mds, 13);
+  tt_assert(tor_addr_family(&md->ipv6_addr) == AF_INET6);
+  tt_int_op(md->ipv6_orport, OP_EQ, 9090);
+  tt_int_op(md->policy_is_reject_star, OP_EQ, 0);
+
  done:
   SMARTLIST_FOREACH(mds, microdesc_t *, mdsc, microdesc_free(mdsc));
   smartlist_free(mds);
@@ -683,6 +771,80 @@ test_md_parse(void *arg)
   tor_free(mem_op_hex_tmp);
 }
 
+static void
+test_md_parse_id_ed25519(void *arg)
+{
+  (void)arg;
+
+  /* A correct MD with an ed25519 ID ... and an unspecified ID type,
+   * which is permitted. */
+  const char GOOD_MD[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id wumpus dodecahedron\n";
+
+  smartlist_t *mds = NULL;
+  const microdesc_t *md;
+
+  mds = microdescs_parse_from_string(GOOD_MD,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 1);
+  md = smartlist_get(mds, 0);
+  tt_mem_op(md->ed25519_identity_pkey, OP_EQ,
+            "This isn't actually a public key", ED25519_PUBKEY_LEN);
+  SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key appears twice. */
+  const char DUPLICATE_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyBrZXk\n";
+
+  setup_capture_of_logs(LOG_WARN);
+  mds = microdescs_parse_from_string(DUPLICATE_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Extra ed25519 key");
+  mock_clean_saved_logs();
+  smartlist_free(mds);
+
+  /* As above, but ed25519 ID key is invalid. */
+  const char BOGUS_KEY[] =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAM7uUtq5F6h63QNYIvC+4NcWaD0DjtnrOORZMkdpJhinXUOwce3cD5Dj\n"
+    "sgdN1wJpWpTQMXJ2DssfSgmOVXETP7qJuZyRprxalQhaEATMDNJA/66Ml1jSO9mZ\n"
+    "+8Xb7m/4q778lNtkSbsvMaYD2Dq6k2QQ3kMhr9z8oUtX0XA23+pfAgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n"
+    "id ed25519 VGhpcyBpc24ndCBhY3R1YWxseSBhIHB1YmxpYyZZZZZZZZZZZ\n";
+
+  mds = microdescs_parse_from_string(BOGUS_KEY,
+                                     NULL, 1, SAVED_NOWHERE, NULL);
+  tt_assert(mds);
+  tt_int_op(smartlist_len(mds), OP_EQ, 0); // no entries.
+  expect_single_log_msg_containing("Bogus ed25519 key");
+
+ done:
+  if (mds) {
+    SMARTLIST_FOREACH(mds, microdesc_t *, m, microdesc_free(m));
+    smartlist_free(mds);
+  }
+  teardown_capture_of_logs();
+}
+
 static int mock_rgsbd_called = 0;
 static routerstatus_t *mock_rgsbd_val_a = NULL;
 static routerstatus_t *mock_rgsbd_val_b = NULL;
@@ -816,6 +978,7 @@ struct testcase_t microdesc_tests[] = {
   { "broken_cache", test_md_cache_broken, TT_FORK, NULL, NULL },
   { "generate", test_md_generate, 0, NULL, NULL },
   { "parse", test_md_parse, 0, NULL, NULL },
+  { "parse_id_ed25519", test_md_parse_id_ed25519, 0, NULL, NULL },
   { "reject_cache", test_md_reject_cache, TT_FORK, NULL, NULL },
   { "corrupt_desc", test_md_corrupt_desc, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
diff --git a/src/test/test_namemap.c b/src/test/test_namemap.c
new file mode 100644
index 0000000000..e93d3fbc3c
--- /dev/null
+++ b/src/test/test_namemap.c
@@ -0,0 +1,174 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/container/namemap.h"
+#include "lib/container/namemap_st.h"
+#include "lib/malloc/malloc.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static void
+test_namemap_empty(void *arg)
+{
+  (void)arg;
+
+  namemap_t m;
+  namemap_init(&m);
+  namemap_t m2 = NAMEMAP_INIT();
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m2));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello128"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, ""));
+  tt_uint_op(0, OP_EQ, namemap_get_size(&m));
+
+ done:
+  namemap_clear(&m);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_toolong(void *arg)
+{
+  (void)arg;
+  namemap_t m;
+  char *ok = NULL;
+  char *toolong = NULL;
+  namemap_init(&m);
+
+  ok = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+1);
+  memset(ok, 'x', MAX_NAMEMAP_NAME_LEN);
+
+  toolong = tor_malloc_zero(MAX_NAMEMAP_NAME_LEN+2);
+  memset(toolong, 'x', MAX_NAMEMAP_NAME_LEN+1);
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+  unsigned u1 = namemap_get_or_create_id(&m, toolong);
+  unsigned u2 = namemap_get_or_create_id(&m, ok);
+  tt_uint_op(u1, OP_EQ, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m, ok));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m, toolong));
+
+  tt_str_op(ok, OP_EQ, namemap_get_name(&m, u2));
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m, u1));
+
+ done:
+  tor_free(ok);
+  tor_free(toolong);
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_blackbox(void *arg)
+{
+  (void)arg;
+
+  namemap_t m1, m2;
+  namemap_init(&m1);
+  namemap_init(&m2);
+
+  unsigned u1 = namemap_get_or_create_id(&m1, "hello");
+  unsigned u2 = namemap_get_or_create_id(&m1, "world");
+  tt_uint_op(u1, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u2, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u1, OP_NE, u2);
+
+  tt_uint_op(u1, OP_EQ, namemap_get_id(&m1, "hello"));
+  tt_uint_op(u1, OP_EQ, namemap_get_or_create_id(&m1, "hello"));
+  tt_uint_op(u2, OP_EQ, namemap_get_id(&m1, "world"));
+  tt_uint_op(u2, OP_EQ, namemap_get_or_create_id(&m1, "world"));
+
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "HELLO"));
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m2, "hello"));
+
+  unsigned u3 = namemap_get_or_create_id(&m2, "hola");
+  tt_uint_op(u3, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(NAMEMAP_ERR, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_or_create_id(&m2, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  unsigned int u4 = namemap_get_or_create_id(&m1, "hola");
+  tt_uint_op(u4, OP_NE, NAMEMAP_ERR);
+  tt_uint_op(u4, OP_EQ, namemap_get_id(&m1, "hola"));
+  tt_uint_op(u3, OP_EQ, namemap_get_id(&m2, "hola"));
+
+  tt_str_op("hello", OP_EQ, namemap_get_name(&m1, u1));
+  tt_str_op("world", OP_EQ, namemap_get_name(&m1, u2));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m2, u3));
+  tt_str_op("hola", OP_EQ, namemap_get_name(&m1, u4));
+
+  tt_ptr_op(NULL, OP_EQ, namemap_get_name(&m2, u3 + 10));
+
+ done:
+  namemap_clear(&m1);
+  namemap_clear(&m2);
+}
+
+static void
+test_namemap_internals(void *arg)
+{
+  (void)arg;
+  // This test actually assumes know something about the identity layout.
+  namemap_t m;
+  namemap_init(&m);
+
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_or_create_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(0, OP_EQ, namemap_get_id(&m, "that"));
+  tt_uint_op(1, OP_EQ, namemap_get_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+  tt_uint_op(1, OP_EQ, namemap_get_or_create_id(&m, "is"));
+  tt_uint_op(2, OP_EQ, namemap_get_or_create_id(&m, "not"));
+
+ done:
+  namemap_clear(&m);
+}
+
+static void
+test_namemap_fmt(void *arg)
+{
+  (void)arg;
+  namemap_t m = NAMEMAP_INIT();
+
+  unsigned a = namemap_get_or_create_id(&m, "greetings");
+  unsigned b = namemap_get_or_create_id(&m, "earthlings");
+
+  tt_str_op(namemap_fmt_name(&m, a), OP_EQ, "greetings");
+  tt_str_op(namemap_fmt_name(&m, b), OP_EQ, "earthlings");
+  tt_int_op(a, OP_NE, 100);
+  tt_int_op(b, OP_NE, 100);
+  tt_str_op(namemap_fmt_name(&m, 100), OP_EQ, "{100}");
+
+ done:
+  namemap_clear(&m);
+}
+
+#define T(name) \
+  { #name, test_namemap_ ## name , 0, NULL, NULL }
+
+struct testcase_t namemap_tests[] = {
+  T(empty),
+  T(toolong),
+  T(blackbox),
+  T(internals),
+  T(fmt),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_netinfo.c b/src/test/test_netinfo.c
new file mode 100644
index 0000000000..93892978dc
--- /dev/null
+++ b/src/test/test_netinfo.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "trunnel/netinfo.h"
+#include "test/test.h"
+
+static void
+test_netinfo_unsupported_addr(void *arg)
+{
+  const uint8_t wire_data[] =
+    { // TIME
+      0x00, 0x00, 0x00, 0x01,
+      // OTHERADDR
+      0x04, // ATYPE
+      0x04, // ALEN
+      0x08, 0x08, 0x08, 0x08, // AVAL
+      0x01, // NMYADDR
+      0x03, // ATYPE (unsupported)
+      0x05, // ALEN
+      'a', 'd', 'r', 'r', '!' // AVAL (unsupported)
+    };
+
+  (void)arg;
+
+  netinfo_cell_t *parsed_cell = NULL;
+
+  ssize_t parsed = netinfo_cell_parse(&parsed_cell, wire_data,
+                  sizeof(wire_data));
+
+  tt_assert(parsed == sizeof(wire_data));
+
+  netinfo_addr_t *addr = netinfo_cell_get_my_addrs(parsed_cell, 0);
+  tt_assert(addr);
+
+  tt_int_op(3, OP_EQ, netinfo_addr_get_addr_type(addr));
+  tt_int_op(5, OP_EQ, netinfo_addr_get_len(addr));
+
+ done:
+  netinfo_cell_free(parsed_cell);
+}
+
+struct testcase_t netinfo_tests[] = {
+  { "unsupported_addr", test_netinfo_unsupported_addr, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
+
diff --git a/src/test/test_nodelist.c b/src/test/test_nodelist.c
index 53eb0413e5..fbbbf0a99f 100644
--- a/src/test/test_nodelist.c
+++ b/src/test/test_nodelist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -6,15 +6,24 @@
  * \brief Unit tests for nodelist related functions.
  **/
 
+#define NODELIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+
 #include "core/or/or.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "feature/nodelist/describe.h"
 #include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/nodefamily.h"
 #include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/torcert.h"
 
+#include "core/or/extend_info_st.h"
+#include "feature/dirauth/dirvote.h"
+#include "feature/nodelist/fmt_routerstatus.h"
 #include "feature/nodelist/microdesc_st.h"
 #include "feature/nodelist/networkstatus_st.h"
 #include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodefamily_st.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerstatus_st.h"
 
@@ -72,7 +81,7 @@ test_nodelist_node_get_verbose_nickname_not_named(void *arg)
 }
 
 /** A node should be considered a directory server if it has an open dirport
- * of it accepts tunnelled directory requests.
+ * or it accepts tunnelled directory requests.
  */
 static void
 test_nodelist_node_is_dir(void *arg)
@@ -231,6 +240,1176 @@ test_nodelist_ed_id(void *arg)
 #undef N_NODES
 }
 
+static void
+test_nodelist_nodefamily(void *arg)
+{
+  (void)arg;
+  /* hex ID digests */
+  const char h1[] = "5B435D6869206861206C65207363617270652070";
+  const char h2[] = "75C3B220616E6461726520696E206769726F2061";
+  const char h3[] = "2074726F766172206461206D616E67696172652C";
+  const char h4[] = "206D656E747265206E6F6E2076616C65206C2769";
+  const char h5[] = "6E766572736F2E202D2D5072696D6F204C657669";
+
+  /* binary ID digests */
+  uint8_t d1[DIGEST_LEN], d2[DIGEST_LEN], d3[DIGEST_LEN], d4[DIGEST_LEN],
+    d5[DIGEST_LEN];
+  base16_decode((char*)d1, sizeof(d1), h1, strlen(h1));
+  base16_decode((char*)d2, sizeof(d2), h2, strlen(h2));
+  base16_decode((char*)d3, sizeof(d3), h3, strlen(h3));
+  base16_decode((char*)d4, sizeof(d4), h4, strlen(h4));
+  base16_decode((char*)d5, sizeof(d5), h5, strlen(h5));
+
+  char *enc=NULL, *enc2=NULL;
+
+  nodefamily_t *nf1 = NULL;
+  nodefamily_t *nf2 = NULL;
+  nodefamily_t *nf3 = NULL;
+
+  enc = nodefamily_format(NULL);
+  tt_str_op(enc, OP_EQ, "");
+  tor_free(enc);
+
+  /* Make sure that sorting and de-duplication work. */
+  tor_asprintf(&enc, "$%s hello", h1);
+  nf1 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf1);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "hello hello $%s hello", h1);
+  nf2 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf2);
+  tt_ptr_op(nf1, OP_EQ, nf2);
+  tor_free(enc);
+
+  tor_asprintf(&enc, "%s $%s hello", h1, h1);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(! nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_nickname(nf1, "hello"));
+  tt_assert(nodefamily_contains_nickname(nf1, "HELLO"));
+  tt_assert(! nodefamily_contains_nickname(nf1, "goodbye"));
+
+  tt_int_op(nf1->refcnt, OP_EQ, 3);
+  nodefamily_free(nf3);
+  tt_int_op(nf1->refcnt, OP_EQ, 2);
+
+  /* Try parsing with a provided self RSA digest. */
+  nf3 = nodefamily_parse("hello ", d1, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_EQ, nf3);
+
+  /* Do we get the expected result when we re-encode? */
+  tor_asprintf(&enc, "$%s hello", h1);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc2, OP_EQ, enc);
+  tor_free(enc2);
+  tor_free(enc);
+
+  /* Make sure that we get a different result if we give a different digest. */
+  nodefamily_free(nf3);
+  tor_asprintf(&enc, "hello $%s hello", h3);
+  nf3 = nodefamily_parse(enc, NULL, 0);
+  tt_assert(nf3);
+  tt_ptr_op(nf1, OP_NE, nf3);
+  tor_free(enc);
+
+  tt_assert(nodefamily_contains_rsa_id(nf3, d3));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d2));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d1));
+  tt_assert(nodefamily_contains_nickname(nf3, "hello"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "goodbye"));
+
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+
+  /* Try one with several digests, all with nicknames appended, in different
+     formats. */
+  tor_asprintf(&enc, "%s $%s $%s=res $%s~ist", h1, h2, h3, h4);
+  nf1 = nodefamily_parse(enc, d5, 0);
+  tt_assert(nf1);
+  tt_assert(nodefamily_contains_rsa_id(nf1, d1));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d2));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d3));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf1, d5));
+  /* Nicknames aren't preserved when ids are present, since node naming is
+   * deprecated */
+  tt_assert(! nodefamily_contains_nickname(nf3, "res"));
+  tor_free(enc);
+  tor_asprintf(&enc, "$%s $%s $%s $%s $%s", h4, h3, h1, h5, h2);
+  enc2 = nodefamily_format(nf1);
+  tt_str_op(enc, OP_EQ, enc2);
+  tor_free(enc);
+  tor_free(enc2);
+
+  /* Try ones where we parse the empty string. */
+  nf2 = nodefamily_parse("", NULL, 0);
+  nf3 = nodefamily_parse("", d4, 0);
+  tt_assert(nf2);
+  tt_assert(nf3);
+  tt_ptr_op(nf2, OP_NE, nf3);
+
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d4));
+  tt_assert(nodefamily_contains_rsa_id(nf3, d4));
+  tt_assert(! nodefamily_contains_rsa_id(nf2, d5));
+  tt_assert(! nodefamily_contains_rsa_id(nf3, d5));
+  tt_assert(! nodefamily_contains_nickname(nf2, "fred"));
+  tt_assert(! nodefamily_contains_nickname(nf3, "bosco"));
+
+  /* The NULL family should contain nothing. */
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d4));
+  tt_assert(! nodefamily_contains_rsa_id(NULL, d5));
+
+ done:
+  tor_free(enc);
+  tor_free(enc2);
+  nodefamily_free(nf1);
+  nodefamily_free(nf2);
+  nodefamily_free(nf3);
+  nodefamily_free_all();
+}
+
+static void
+test_nodelist_nodefamily_parse_err(void *arg)
+{
+  (void)arg;
+  nodefamily_t *nf1 = NULL;
+  char *enc = NULL;
+  const char *semibogus =
+    "sdakljfdslkfjdsaklfjdkl9sdf " // too long for nickname
+    "$jkASDFLkjsadfjhkl " // not hex
+    "$7468696e67732d696e2d7468656d73656c766573 " // ok
+    "reticulatogranulate "// ok
+    "$73656d69616e7468726f706f6c6f676963616c6c79 " // too long for hex
+    "$616273656e746d696e6465646e6573736573" // too short for hex
+    ;
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // We only get two items when we parse this.
+  for (int reject = 0; reject <= 1; ++reject) {
+    for (int log_at_warn = 0; log_at_warn <= 1; ++log_at_warn) {
+      unsigned flags = log_at_warn ? NF_WARN_MALFORMED : 0;
+      flags |= reject ? NF_REJECT_MALFORMED : 0;
+      nf1 = nodefamily_parse(semibogus, NULL, flags);
+      if (reject) {
+        tt_assert(nf1 == NULL);
+      } else {
+        tt_assert(nf1);
+        enc = nodefamily_format(nf1);
+        tt_str_op(enc, OP_EQ,
+                  "$7468696E67732D696E2D7468656D73656C766573 "
+                  "reticulatogranulate");
+        tor_free(enc);
+      }
+
+      if (log_at_warn) {
+        expect_log_msg_containing("$616273656e746d696e6465646e6573736573");
+        expect_log_msg_containing("sdakljfdslkfjdsaklfjdkl9sdf");
+      } else {
+        tt_int_op(mock_saved_log_n_entries(), OP_EQ, 0);
+      }
+      mock_clean_saved_logs();
+    }
+  }
+
+ done:
+  tor_free(enc);
+  nodefamily_free(nf1);
+  teardown_capture_of_logs();
+}
+
+static const node_t *
+mock_node_get_by_id(const char *id)
+{
+  if (fast_memeq(id, "!!!!!!!!!!!!!!!!!!!!", DIGEST_LEN))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  memcpy(fake_node->identity, id, DIGEST_LEN);
+  return fake_node;
+}
+
+static const node_t *
+mock_node_get_by_nickname(const char *nn, unsigned flags)
+{
+  (void)flags;
+  if (!strcmp(nn, "nonesuch"))
+    return NULL;
+
+  // use tor_free, not node_free.
+  node_t *fake_node = tor_malloc_zero(sizeof(node_t));
+  strlcpy(fake_node->identity, nn, DIGEST_LEN);
+  return fake_node;
+}
+
+static void
+test_nodelist_nodefamily_lookup(void *arg)
+{
+  (void)arg;
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+  smartlist_t *sl = smartlist_new();
+  nodefamily_t *nf1 = NULL;
+  char *mem_op_hex_tmp = NULL;
+
+  // 'null' is allowed.
+  nodefamily_add_nodes_to_smartlist(NULL, sl);
+  tt_int_op(smartlist_len(sl), OP_EQ, 0);
+
+  // Try a real family
+  nf1 = nodefamily_parse("$EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE "
+                         "$2121212121212121212121212121212121212121 "
+                         "$3333333333333333333333333333333333333333 "
+                         "erewhon nonesuch", NULL, 0);
+  tt_assert(nf1);
+  nodefamily_add_nodes_to_smartlist(nf1, sl);
+  // There were 5 elements; 2 were dropped because the mocked lookup failed.
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+
+  const node_t *n = smartlist_get(sl, 0);
+  test_memeq_hex(n->identity, "3333333333333333333333333333333333333333");
+  n = smartlist_get(sl, 1);
+  test_memeq_hex(n->identity, "EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE");
+  n = smartlist_get(sl, 2);
+  tt_str_op(n->identity, OP_EQ, "erewhon");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  SMARTLIST_FOREACH(sl, node_t *, fake_node, tor_free(fake_node));
+  smartlist_free(sl);
+  nodefamily_free(nf1);
+  tor_free(mem_op_hex_tmp);
+}
+
+static void
+test_nodelist_nickname_matches(void *arg)
+{
+  (void)arg;
+  node_t mock_node;
+  routerstatus_t mock_rs;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+
+  strlcpy(mock_rs.nickname, "evilgeniuses", sizeof(mock_rs.nickname));
+  mock_node.rs = &mock_rs;
+  memcpy(mock_node.identity, ".forabettertomorrow.", DIGEST_LEN);
+
+#define match(x) tt_assert(node_nickname_matches(&mock_node, (x)))
+#define no_match(x) tt_assert(! node_nickname_matches(&mock_node, (x)))
+
+  match("evilgeniuses");
+  match("EvilGeniuses");
+  match("EvilGeniuses");
+  match("2e666f7261626574746572746f6d6f72726f772e");
+  match("2E666F7261626574746572746F6D6F72726F772E");
+  match("$2e666f7261626574746572746f6d6f72726f772e");
+  match("$2E666F7261626574746572746F6D6F72726F772E");
+  match("$2E666F7261626574746572746F6D6F72726F772E~evilgeniuses");
+  match("$2E666F7261626574746572746F6D6F72726F772E~EVILGENIUSES");
+
+  no_match("evilgenius");
+  no_match("evilgeniuseses");
+  no_match("evil.genius");
+  no_match("$2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("2E666F7261626574746572746F6D6F72726FFFFF");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E~fred");
+  no_match("$2E666F7261626574746572746F6D6F72726F772E=EVILGENIUSES");
+ done:
+  ;
+}
+
+static void
+test_nodelist_node_nodefamily(void *arg)
+{
+  (void)arg;
+  node_t mock_node1;
+  routerstatus_t mock_rs;
+  microdesc_t mock_md;
+
+  node_t mock_node2;
+  routerinfo_t mock_ri;
+
+  smartlist_t *nodes=smartlist_new();
+
+  memset(&mock_node1, 0, sizeof(mock_node1));
+  memset(&mock_node2, 0, sizeof(mock_node2));
+  memset(&mock_rs, 0, sizeof(mock_rs));
+  memset(&mock_md, 0, sizeof(mock_md));
+  memset(&mock_ri, 0, sizeof(mock_ri));
+
+  mock_node1.rs = &mock_rs;
+  mock_node1.md = &mock_md;
+
+  mock_node2.ri = &mock_ri;
+
+  strlcpy(mock_rs.nickname, "nodeone", sizeof(mock_rs.nickname));
+  mock_ri.nickname = tor_strdup("nodetwo");
+
+  memcpy(mock_node1.identity, "NodeOneNode1NodeOne1", DIGEST_LEN);
+  memcpy(mock_node2.identity, "SecondNodeWe'reTestn", DIGEST_LEN);
+
+  // empty families.
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain nodes, but not these nodes
+  mock_ri.declared_family = smartlist_new();
+  smartlist_add(mock_ri.declared_family, (char*)"NodeThree");
+  mock_md.family = nodefamily_parse("NodeFour", NULL, 0);
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(! node_family_contains(&mock_node2, &mock_node1));
+
+  // Families contain one another.
+  smartlist_add(mock_ri.declared_family, (char*)
+                "4e6f64654f6e654e6f6465314e6f64654f6e6531");
+  tt_assert(! node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  nodefamily_free(mock_md.family);
+  mock_md.family = nodefamily_parse(
+            "NodeFour "
+            "5365636f6e644e6f64655765277265546573746e", NULL, 0);
+  tt_assert(node_family_contains(&mock_node1, &mock_node2));
+  tt_assert(node_family_contains(&mock_node2, &mock_node1));
+
+  // Try looking up families now.
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+  MOCK(node_get_by_id, mock_node_get_by_id);
+
+  node_lookup_declared_family(nodes, &mock_node1);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  const node_t *n = smartlist_get(nodes, 0);
+  tt_mem_op(n->identity, OP_EQ, "SecondNodeWe'reTestn", DIGEST_LEN);
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "nodefour");
+
+  // free, try the other one.
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_clear(nodes);
+
+  node_lookup_declared_family(nodes, &mock_node2);
+  tt_int_op(smartlist_len(nodes), OP_EQ, 2);
+  n = smartlist_get(nodes, 0);
+  // This gets a truncated hex hex ID since it was looked up by name
+  tt_str_op(n->identity, OP_EQ, "NodeThree");
+  n = smartlist_get(nodes, 1);
+  tt_str_op(n->identity, OP_EQ, "4e6f64654f6e654e6f6");
+
+ done:
+  UNMOCK(node_get_by_nickname);
+  UNMOCK(node_get_by_id);
+  smartlist_free(mock_ri.declared_family);
+  nodefamily_free(mock_md.family);
+  tor_free(mock_ri.nickname);
+  // use tor_free, these aren't real nodes
+  SMARTLIST_FOREACH(nodes, node_t *, x, tor_free(x));
+  smartlist_free(nodes);
+}
+
+static void
+test_nodelist_nodefamily_canonicalize(void *arg)
+{
+  (void)arg;
+  char *c = NULL;
+
+  c = nodefamily_canonicalize("", NULL, 0);
+  tt_str_op(c, OP_EQ, "");
+  tor_free(c);
+
+  uint8_t own_id[20];
+  memset(own_id, 0, sizeof(own_id));
+  c = nodefamily_canonicalize(
+           "alice BOB caroL %potrzebie !!!@#@# "
+           "$bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb=fred "
+           "ffffffffffffffffffffffffffffffffffffffff "
+           "$cccccccccccccccccccccccccccccccccccccccc ", own_id, 0);
+  tt_str_op(c, OP_EQ,
+           "!!!@#@# "
+           "$0000000000000000000000000000000000000000 "
+           "$BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB "
+           "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC "
+           "$FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF "
+           "%potrzebie "
+           "alice bob carol");
+
+ done:
+  tor_free(c);
+}
+
+/** format_node_description() should return
+ * "Fingerprint~Nickname at IPv4 and [IPv6]".
+ * The nickname and addresses are optional.
+ */
+static void
+test_nodelist_format_node_description(void *arg)
+{
+  char mock_digest[DIGEST_LEN];
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_null_ip;
+  tor_addr_t mock_ipv4;
+  tor_addr_t mock_ipv6;
+
+  char ndesc[NODE_DESC_BUF_LEN];
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(ndesc, 0, sizeof(ndesc));
+  memset(mock_digest, 0, sizeof(mock_digest));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_null_ip, 0, sizeof(mock_null_ip));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ipv6, 0, sizeof(mock_ipv6));
+
+  /* Set variables */
+  memcpy(mock_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  tor_addr_parse(&mock_ipv6, "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Test function with variables */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  /* format node description should use ~ because named is deprecated */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               NULL,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~""TestOR7890123456789");
+
+  /* Try a null IP address, rather than NULL */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_null_ip,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789");
+
+  /* Try some real IP addresses */
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               NULL,
+                               &mock_ipv4,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA at 111.222.233.244");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = format_node_description(ndesc,
+                               mock_digest,
+                               mock_nickname,
+                               &mock_ipv6,
+                               tor_addr_to_ipv4h(&mock_ipv4));
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(ndesc, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = format_node_description(NULL, NULL, NULL, NULL, 0);
+  tt_str_op(rv, OP_EQ, "<NULL BUFFER>");
+
+  rv = format_node_description(ndesc, NULL, NULL, NULL, 0);
+  tt_ptr_op(rv, OP_EQ, ndesc);
+  tt_str_op(rv, OP_EQ, "<NULL ID DIGEST>");
+
+ done:
+  return;
+}
+
+/** router_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_router_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+  routerinfo_t mock_ri_ipv4;
+  routerinfo_t mock_ri_ipv6;
+  routerinfo_t mock_ri_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_ipv4, 0, sizeof(mock_ri_ipv4));
+  memset(&mock_ri_ipv6, 0, sizeof(mock_ri_ipv6));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerinfos.
+   * mock_nickname is referenced from all 3 routerinfos.
+   * That's ok, all their memory is static. */
+  memcpy(&mock_ri_ipv4, &mock_ri_dual, sizeof(mock_ri_ipv4));
+  memcpy(&mock_ri_ipv6, &mock_ri_dual, sizeof(mock_ri_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_ri_ipv4.ipv6_addr, 0, sizeof(mock_ri_ipv4.ipv6_addr));
+  mock_ri_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = router_describe(&mock_ri_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = router_describe(&mock_ri_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = router_describe(&mock_ri_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = router_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerinfos */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.ri = &mock_ri_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.ri = &mock_ri_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = &mock_ri_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** node_describe() is a wrapper for format_node_description(), see that
+ * test for details.
+ *
+ * The routerinfo-only and routerstatus-only node_describe() tests are in
+ * test_nodelist_router_describe() and test_nodelist_routerstatus_describe(),
+ * so we can re-use their mocked variables.
+ */
+static void
+test_nodelist_node_describe(void *arg)
+{
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+  tor_addr_t mock_ipv4;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Routerinfos */
+  routerinfo_t mock_ri_dual;
+
+  /* Clear variables */
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_ri_dual, 0, sizeof(mock_ri_dual));
+
+  /* Set up the dual-stack routerinfo */
+  memcpy(mock_ri_dual.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ri_dual.cache_info.identity_digest));
+  strlcpy(mock_nickname, "TestOR7890123456789", sizeof(mock_nickname));
+  mock_ri_dual.nickname = mock_nickname;
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_ri_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_ri_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Routerstatuses */
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_dual;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB"
+         "\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB\xBB",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "Bbb",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "2.2.2.2");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[bbbb::bbbb]");
+
+  /* Create and modify the other routerstatus. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  /* Clear the unnecessary IPv6 address */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+
+  /* Microdescs */
+  microdesc_t mock_md_null;
+  microdesc_t mock_md_ipv6;
+
+  /* Clear variables */
+  memset(&mock_md_null, 0, sizeof(mock_md_null));
+  memset(&mock_md_ipv6, 0, sizeof(mock_md_ipv6));
+
+  /* Set up the microdesc */
+  tor_addr_parse(&mock_md_ipv6.ipv6_addr,
+                 "[eeee::6000:6000]");
+
+  /* Set up the node */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC"
+         "\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC\xCC",
+         sizeof(mock_node.identity));
+
+  /* Test that the routerinfo and routerstatus work separately, but the
+   * identity comes from the node */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerstatus overrides the routerinfo */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the microdesc IPv6 is used if the routerinfo doesn't have IPv6
+   */
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = NULL;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* Test that the routerinfo doesn't change the results above
+   */
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [eeee::6000:6000]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_ipv4;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  mock_node.ri = &mock_ri_dual;
+  mock_node.rs = &mock_rs_dual;
+  mock_node.md = &mock_md_null;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC~Bbb at "
+            "2.2.2.2 and [bbbb::bbbb]");
+
+  /* test NULL handling */
+  rv = node_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  mock_node.ri = NULL;
+  mock_node.rs = NULL;
+  mock_node.md = NULL;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "<null rs and ri>");
+
+ done:
+  return;
+}
+
+/** routerstatus_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ *
+ * The routerstatus-only node_describe() tests are in this function,
+ * so we can re-use the same mocked variables.
+ */
+static void
+test_nodelist_routerstatus_describe(void *arg)
+{
+  tor_addr_t mock_ipv4;
+  routerstatus_t mock_rs_ipv4;
+  routerstatus_t mock_rs_ipv6;
+  routerstatus_t mock_rs_dual;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ipv4, 0, sizeof(mock_ipv4));
+  memset(&mock_rs_ipv4, 0, sizeof(mock_rs_ipv4));
+  memset(&mock_rs_ipv6, 0, sizeof(mock_rs_ipv6));
+  memset(&mock_rs_dual, 0, sizeof(mock_rs_dual));
+
+  /* Set up the dual-stack routerstatus */
+  memcpy(mock_rs_dual.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_rs_dual.identity_digest));
+  strlcpy(mock_rs_dual.nickname, "TestOR7890123456789",
+          sizeof(mock_rs_dual.nickname));
+  tor_addr_parse(&mock_ipv4, "111.222.233.244");
+  mock_rs_dual.addr = tor_addr_to_ipv4h(&mock_ipv4);
+  tor_addr_parse(&mock_rs_dual.ipv6_addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Create and modify the other routerstatuses. */
+  memcpy(&mock_rs_ipv4, &mock_rs_dual, sizeof(mock_rs_ipv4));
+  memcpy(&mock_rs_ipv6, &mock_rs_dual, sizeof(mock_rs_ipv6));
+  /* Clear the unnecessary addresses */
+  memset(&mock_rs_ipv4.ipv6_addr, 0, sizeof(mock_rs_ipv4.ipv6_addr));
+  mock_rs_ipv6.addr = 0;
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = routerstatus_describe(&mock_rs_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = routerstatus_describe(&mock_rs_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  rv = routerstatus_describe(&mock_rs_dual);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* test NULL handling */
+  rv = routerstatus_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+  /* Now test a node with only these routerstatuses */
+  node_t mock_node;
+  memset(&mock_node, 0, sizeof(mock_node));
+  memcpy(mock_node.identity,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_node.identity));
+
+  /* Try some real IP addresses */
+  mock_node.rs = &mock_rs_ipv4;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  mock_node.rs = &mock_rs_ipv6;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  mock_node.rs = &mock_rs_dual;
+  rv = node_describe(&mock_node);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244 and [1111:2222:3333:4444:5555:6666:7777:8888]");
+
+ done:
+  return;
+}
+
+/** extend_info_describe() is a wrapper for format_node_description(), see
+ * that test for details.
+ */
+static void
+test_nodelist_extend_info_describe(void *arg)
+{
+  extend_info_t mock_ei_ipv4;
+  extend_info_t mock_ei_ipv6;
+
+  const char *rv = NULL;
+
+  (void) arg;
+
+  /* Clear variables */
+  memset(&mock_ei_ipv4, 0, sizeof(mock_ei_ipv4));
+  memset(&mock_ei_ipv6, 0, sizeof(mock_ei_ipv6));
+
+  /* Set up the IPv4 extend info */
+  memcpy(mock_ei_ipv4.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         sizeof(mock_ei_ipv4.identity_digest));
+  strlcpy(mock_ei_ipv4.nickname, "TestOR7890123456789",
+          sizeof(mock_ei_ipv4.nickname));
+  tor_addr_parse(&mock_ei_ipv4.addr, "111.222.233.244");
+
+  /* Create and modify the other extend info. */
+  memcpy(&mock_ei_ipv6, &mock_ei_ipv4, sizeof(mock_ei_ipv6));
+  tor_addr_parse(&mock_ei_ipv6.addr,
+                 "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* We don't test the no-nickname and no-IP cases, because they're covered by
+   * format_node_description(), and we don't expect to see them in Tor code. */
+
+  /* Try some real IP addresses */
+  rv = extend_info_describe(&mock_ei_ipv4);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "111.222.233.244");
+
+  rv = extend_info_describe(&mock_ei_ipv6);
+  tt_str_op(rv, OP_EQ,
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR7890123456789 at "
+            "[1111:2222:3333:4444:5555:6666:7777:8888]");
+
+  /* Extend infos only have one IP address, so there is no dual case */
+
+  /* test NULL handling */
+  rv = extend_info_describe(NULL);
+  tt_str_op(rv, OP_EQ, "<null>");
+
+ done:
+  return;
+}
+
+/** router_get_verbose_nickname() should return "Fingerprint~Nickname"
+ */
+static void
+test_nodelist_router_get_verbose_nickname(void *arg)
+{
+  routerinfo_t mock_ri;
+  char mock_nickname[MAX_NICKNAME_LEN+1];
+
+  char vname[MAX_VERBOSE_NICKNAME_LEN+1];
+
+  (void) arg;
+
+  memset(&mock_ri, 0, sizeof(routerinfo_t));
+  memset(mock_nickname, 0, sizeof(mock_nickname));
+  mock_ri.nickname = mock_nickname;
+
+  /* verbose nickname should use ~ because named is deprecated */
+  strlcpy(mock_nickname, "TestOR", sizeof(mock_nickname));
+  memcpy(mock_ri.cache_info.identity_digest,
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"
+         "\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA",
+         DIGEST_LEN);
+  router_get_verbose_nickname(vname, &mock_ri);
+  tt_str_op(vname, OP_EQ, "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA~TestOR");
+
+  /* test NULL router handling */
+  router_get_verbose_nickname(vname, NULL);
+  tt_str_op(vname, OP_EQ, "<null>");
+
+  router_get_verbose_nickname(NULL, &mock_ri);
+  router_get_verbose_nickname(NULL, NULL);
+
+ done:
+  return;
+}
+
+static void
+test_nodelist_routerstatus_has_visibly_changed(void *arg)
+{
+  (void)arg;
+  routerstatus_t rs_orig, rs;
+  char *fmt_orig = NULL, *fmt = NULL;
+  memset(&rs_orig, 0, sizeof(rs_orig));
+  strlcpy(rs_orig.nickname, "friendly", sizeof(rs_orig.nickname));
+  memcpy(rs_orig.identity_digest, "abcdefghijklmnopqrst", 20);
+  memcpy(rs_orig.descriptor_digest, "abcdefghijklmnopqrst", 20);
+  rs_orig.addr = 0x7f000001;
+  rs_orig.or_port = 3;
+  rs_orig.published_on = time(NULL);
+  rs_orig.has_bandwidth = 1;
+  rs_orig.bandwidth_kb = 20;
+
+#define COPY() memcpy(&rs, &rs_orig, sizeof(rs))
+#define FORMAT() \
+  STMT_BEGIN \
+    tor_free(fmt_orig);                                                   \
+    tor_free(fmt);                                                        \
+    fmt_orig = routerstatus_format_entry(&rs_orig, NULL, NULL,            \
+                          NS_CONTROL_PORT,                                \
+                          NULL);                                          \
+    fmt = routerstatus_format_entry(&rs, NULL, NULL, NS_CONTROL_PORT,     \
+                          NULL);                                          \
+    tt_assert(fmt_orig);                                                  \
+    tt_assert(fmt);                                                       \
+  STMT_END
+#define ASSERT_SAME() \
+  STMT_BEGIN                                                    \
+    tt_assert(! routerstatus_has_visibly_changed(&rs_orig, &rs));       \
+    FORMAT();                                                   \
+    tt_str_op(fmt_orig, OP_EQ, fmt);                            \
+    COPY();                                                     \
+  STMT_END
+#define ASSERT_CHANGED() \
+  STMT_BEGIN                                                    \
+    tt_assert(routerstatus_has_visibly_changed(&rs_orig, &rs));         \
+    FORMAT();                                                   \
+    tt_str_op(fmt_orig, OP_NE, fmt);                            \
+    COPY();                                                     \
+  STMT_END
+#define ASSERT_CHANGED_NO_FORMAT() \
+  STMT_BEGIN                                                    \
+    tt_assert(routerstatus_has_visibly_changed(&rs_orig, &rs));         \
+    COPY();                                                     \
+  STMT_END
+
+  COPY();
+  ASSERT_SAME();
+
+  rs.addr = 0x7f000002;
+  ASSERT_CHANGED();
+
+  strlcpy(rs.descriptor_digest, "hello world", sizeof(rs.descriptor_digest));
+  ASSERT_CHANGED();
+
+  strlcpy(rs.nickname, "fr1end1y", sizeof(rs.nickname));
+  ASSERT_CHANGED();
+
+  rs.published_on += 3600;
+  ASSERT_CHANGED();
+
+  rs.or_port = 55;
+  ASSERT_CHANGED();
+
+  rs.dir_port = 9999;
+  ASSERT_CHANGED();
+
+  tor_addr_parse(&rs.ipv6_addr, "1234::56");
+  ASSERT_CHANGED();
+
+  tor_addr_parse(&rs_orig.ipv6_addr, "1234::56");
+  rs_orig.ipv6_orport = 99;
+  COPY();
+  rs.ipv6_orport = 22;
+  ASSERT_CHANGED();
+
+  rs.is_authority = 1;
+  ASSERT_CHANGED();
+
+  rs.is_exit = 1;
+  ASSERT_CHANGED();
+
+  rs.is_stable = 1;
+  ASSERT_CHANGED();
+
+  rs.is_fast = 1;
+  ASSERT_CHANGED();
+
+  rs.is_flagged_running = 1;
+  ASSERT_CHANGED();
+
+  // This option is obsolete and not actually formatted.
+  rs.is_named = 1;
+  ASSERT_CHANGED_NO_FORMAT();
+
+  // This option is obsolete and not actually formatted.
+  rs.is_unnamed = 1;
+  ASSERT_CHANGED_NO_FORMAT();
+
+  rs.is_valid = 1;
+  ASSERT_CHANGED();
+
+  rs.is_possible_guard = 1;
+  ASSERT_CHANGED();
+
+  rs.is_bad_exit = 1;
+  ASSERT_CHANGED();
+
+  rs.is_hs_dir = 1;
+  ASSERT_CHANGED();
+
+  rs.is_v2_dir = 1;
+  ASSERT_CHANGED();
+
+  rs.is_staledesc = 1;
+  ASSERT_CHANGED();
+
+  // Setting this to zero crashes us with an assertion failure in
+  // routerstatus_format_entry() if we don't have a descriptor.
+  rs.has_bandwidth = 0;
+  ASSERT_CHANGED_NO_FORMAT();
+
+  // Does not actually matter; not visible to controller.
+  rs.has_exitsummary = 1;
+  ASSERT_SAME();
+
+  // Does not actually matter; not visible to the controller.
+  rs.bw_is_unmeasured = 1;
+  ASSERT_SAME();
+
+  rs.bandwidth_kb = 2000;
+  ASSERT_CHANGED();
+
+  // not visible to the controller.
+  rs.has_guardfraction = 1;
+  rs.guardfraction_percentage = 22;
+  ASSERT_SAME();
+
+  // not visible to the controller.
+  rs_orig.has_guardfraction = 1;
+  rs_orig.guardfraction_percentage = 20;
+  COPY();
+  rs.guardfraction_percentage = 25;
+  ASSERT_SAME();
+
+  // not visible to the controller.
+  rs.exitsummary = (char*)"accept 1-2";
+  ASSERT_SAME();
+
+ done:
+#undef COPY
+#undef ASSERT_SAME
+#undef ASSERT_CHANGED
+  tor_free(fmt_orig);
+  tor_free(fmt);
+  return;
+}
+
 #define NODE(name, flags) \
   { #name, test_nodelist_##name, (flags), NULL, NULL }
 
@@ -239,6 +1418,18 @@ struct testcase_t nodelist_tests[] = {
   NODE(node_get_verbose_nickname_not_named, TT_FORK),
   NODE(node_is_dir, TT_FORK),
   NODE(ed_id, TT_FORK),
+  NODE(nodefamily, TT_FORK),
+  NODE(nodefamily_parse_err, TT_FORK),
+  NODE(nodefamily_lookup, TT_FORK),
+  NODE(nickname_matches, 0),
+  NODE(node_nodefamily, TT_FORK),
+  NODE(nodefamily_canonicalize, 0),
+  NODE(format_node_description, 0),
+  NODE(router_describe, 0),
+  NODE(node_describe, 0),
+  NODE(routerstatus_describe, 0),
+  NODE(extend_info_describe, 0),
+  NODE(router_get_verbose_nickname, 0),
+  NODE(routerstatus_has_visibly_changed, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_ntor_cl.c b/src/test/test_ntor_cl.c
index 68b6927f56..a1508d0afc 100644
--- a/src/test/test_ntor_cl.c
+++ b/src/test/test_ntor_cl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_oom.c b/src/test/test_oom.c
index b813fa43a3..51c237ec2e 100644
--- a/src/test/test_oom.c
+++ b/src/test/test_oom.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOM handling logic */
@@ -8,7 +8,7 @@
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "lib/evloop/compat_libevent.h"
 #include "core/mainloop/connection.h"
diff --git a/src/test/test_oos.c b/src/test/test_oos.c
index 815feda7ce..f8c712a6b6 100644
--- a/src/test/test_oos.c
+++ b/src/test/test_oos.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for OOS handler */
diff --git a/src/test/test_options.c b/src/test/test_options.c
index 0e52967a23..8e0d19f126 100644
--- a/src/test/test_options.c
+++ b/src/test/test_options.c
@@ -1,19 +1,29 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
+#define RELAY_CONFIG_PRIVATE
+#define LOG_PRIVATE
+#define ROUTERSET_PRIVATE
 #include "core/or/or.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "app/config/config.h"
+#include "feature/dirauth/dirauth_config.h"
+#include "feature/dirauth/dirauth_options_st.h"
+#include "feature/dirauth/dirauth_sys.h"
+#include "feature/relay/relay_config.h"
 #include "test/test.h"
 #include "lib/geoip/geoip.h"
 
-#define ROUTERSET_PRIVATE
 #include "feature/nodelist/routerset.h"
 #include "core/mainloop/mainloop.h"
+#include "app/main/subsysmgr.h"
 #include "test/log_test_helpers.h"
+#include "test/resolve_test_helpers.h"
+#include "lib/crypt_ops/crypto_options_st.h"
+#include "lib/crypt_ops/crypto_sys.h"
 
 #include "lib/sandbox/sandbox.h"
 #include "lib/memarea/memarea.h"
@@ -21,24 +31,23 @@
 #include "lib/encoding/confline.h"
 #include "core/or/policies.h"
 #include "test/test_helpers.h"
+#include "test/opts_test_helpers.h"
 #include "lib/net/resolve.h"
 
 #ifdef HAVE_SYS_PARAM_H
 #include <sys/param.h>
 #endif
 
-#define NS_MODULE test_options
-
 typedef struct {
   int severity;
-  uint32_t domain;
+  log_domain_mask_t domain;
   char *msg;
 } logmsg_t;
 
 static smartlist_t *messages = NULL;
 
 static void
-log_cback(int severity, uint32_t domain, const char *msg)
+log_cback(int severity, log_domain_mask_t domain, const char *msg)
 {
   logmsg_t *x = tor_malloc(sizeof(*x));
   x->severity = severity;
@@ -54,9 +63,9 @@ setup_log_callback(void)
 {
   log_severity_list_t lst;
   memset(&lst, 0, sizeof(lst));
-  lst.masks[LOG_ERR - LOG_ERR] = ~0;
-  lst.masks[LOG_WARN - LOG_ERR] = ~0;
-  lst.masks[LOG_NOTICE - LOG_ERR] = ~0;
+  lst.masks[SEVERITY_MASK_IDX(LOG_ERR)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_WARN)] = LD_ALL_DOMAINS;
+  lst.masks[SEVERITY_MASK_IDX(LOG_NOTICE)] = LD_ALL_DOMAINS;
   add_callback_log(&lst, log_cback);
   mark_logs_temp();
 }
@@ -90,16 +99,57 @@ clear_log_messages(void)
   messages = NULL;
 }
 
-#define setup_options(opt,dflt)              \
+#define setup_options(opt)                   \
   do {                                       \
     opt = options_new();                     \
     opt->command = CMD_RUN_TOR;              \
     options_init(opt);                       \
-                                             \
-    dflt = config_dup(&options_format, opt); \
-    clear_log_messages();                    \
   } while (0)
 
+#ifdef COCCI
+
+#define ENABLE_AUTHORITY_MIN ""
+#define ENABLE_AUTHORITY_V3_MIN ""
+#define ENABLE_AUTHORITY_BRIDGE_MIN ""
+#define AUTHORITY_OPT_REQ_ ""
+#define ENABLE_AUTHORITY ""
+#define ENABLE_AUTHORITY_V3 ""
+#define ENABLE_AUTHORITY_BRIDGE ""
+
+#else /* !defined(COCCI) */
+
+#define ENABLE_AUTHORITY_MIN \
+  "AuthoritativeDirectory 1\n"
+
+#define ENABLE_AUTHORITY_V3_MIN \
+  ENABLE_AUTHORITY_MIN \
+  "V3AuthoritativeDir 1\n"
+
+#define ENABLE_AUTHORITY_BRIDGE_MIN \
+  ENABLE_AUTHORITY_MIN \
+  "BridgeAuthoritativeDir 1\n"
+
+#define AUTHORITY_OPT_REQ_ \
+  "Address 192.0.2.111\n" \
+  "ContactInfo a@example.org\n" \
+  "DirPort 1025\n" \
+  "ORPort 1026\n"
+
+/* Not actually valid: requires v3 / bridge */
+#define ENABLE_AUTHORITY \
+  ENABLE_AUTHORITY_MIN \
+  AUTHORITY_OPT_REQ_
+
+#define ENABLE_AUTHORITY_V3 \
+  ENABLE_AUTHORITY_V3_MIN \
+  AUTHORITY_OPT_REQ_
+
+#define ENABLE_AUTHORITY_BRIDGE \
+  ENABLE_AUTHORITY_BRIDGE_MIN \
+  AUTHORITY_OPT_REQ_
+
+#endif /* defined(COCCI) */
+
 #define VALID_DIR_AUTH "DirAuthority dizum orport=443 v3ident=E8A9C45"  \
   "EDE6D711294FADF8E7951F4DE6CA56B58 194.109.206.212:80 7EA6 EAD6 FD83" \
   " 083C 538F 4403 8BBF A077 587D D755\n"
@@ -179,12 +229,11 @@ test_options_validate_impl(const char *configuration,
                            int phase)
 {
   or_options_t *opt=NULL;
-  or_options_t *dflt;
   config_line_t *cl=NULL;
   char *msg=NULL;
   int r;
 
-  setup_options(opt, dflt);
+  setup_options(opt);
 
   r = config_get_lines(configuration, &cl, 1);
   if (phase == PH_GETLINES) {
@@ -196,7 +245,7 @@ test_options_validate_impl(const char *configuration,
   if (r)
     goto done;
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   if (phase == PH_ASSIGN) {
     if (test_options_checkmsgs(configuration, expect_errmsg,
                                expect_log_severity,
@@ -207,7 +256,7 @@ test_options_validate_impl(const char *configuration,
   if (r)
     goto done;
 
-  r = options_validate(NULL, opt, dflt, 0, &msg);
+  r = options_validate(NULL, opt, &msg);
   if (phase == PH_VALIDATE) {
     if (test_options_checkmsgs(configuration, expect_errmsg,
                                expect_log_severity,
@@ -221,7 +270,6 @@ test_options_validate_impl(const char *configuration,
   policies_free_all();
   config_free_lines(cl);
   or_options_free(opt);
-  or_options_free(dflt);
   tor_free(msg);
   clear_log_messages();
 }
@@ -241,6 +289,7 @@ test_options_validate(void *arg)
   (void)arg;
   setup_log_callback();
   sandbox_disable_getaddrinfo_cache();
+  mock_hostname_resolver();
 
   WANT_ERR("ExtORPort 500000", "Invalid ExtORPort", PH_VALIDATE);
 
@@ -258,14 +307,10 @@ test_options_validate(void *arg)
   WANT_ERR("BridgeRelay 1\nDirCache 0",
            "We're a bridge but DirCache is disabled.", PH_VALIDATE);
 
-  WANT_ERR_LOG("HeartbeatPeriod 21 snarks",
-               "Interval 'HeartbeatPeriod 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
-               PH_ASSIGN);
-  WANT_ERR_LOG("LogTimeGranularity 21 snarks",
-               "Msec interval 'LogTimeGranularity 21 snarks' is malformed or"
-               " out of bounds.", LOG_WARN, "Unknown unit 'snarks'.",
-               PH_ASSIGN);
+  WANT_ERR("HeartbeatPeriod 21 snarks",
+           "Unknown unit in 21 snarks", PH_ASSIGN);
+  WANT_ERR("LogTimeGranularity 21 snarks",
+           "Unknown unit in 21 snarks", PH_ASSIGN);
   OK("HeartbeatPeriod 1 hour", PH_VALIDATE);
   OK("LogTimeGranularity 100 milliseconds", PH_VALIDATE);
 
@@ -278,6 +323,7 @@ test_options_validate(void *arg)
 
   close_temp_logs();
   clear_log_messages();
+  unmock_hostname_resolver();
   return;
 }
 
@@ -287,20 +333,18 @@ test_have_enough_mem_for_dircache(void *arg)
 {
   (void)arg;
   or_options_t *opt=NULL;
-  or_options_t *dflt=NULL;
   config_line_t *cl=NULL;
   char *msg=NULL;
   int r;
   const char *configuration = "ORPort 8080\nDirCache 1", *expect_errmsg;
 
-  setup_options(opt, dflt);
+  setup_options(opt);
   setup_log_callback();
-  (void)dflt;
 
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled */
@@ -323,7 +367,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 300 MB RAM available, DirCache enabled, Bridge */
@@ -346,7 +390,7 @@ test_have_enough_mem_for_dircache(void *arg)
   r = config_get_lines(configuration, &cl, 1);
   tt_int_op(r, OP_EQ, 0);
 
-  r = config_assign(&options_format, opt, cl, 0, &msg);
+  r = config_assign(get_options_mgr(), opt, cl, 0, &msg);
   tt_int_op(r, OP_EQ, 0);
 
   /* 200 MB RAM available, DirCache disabled */
@@ -369,7 +413,6 @@ test_have_enough_mem_for_dircache(void *arg)
  done:
   if (msg)
     tor_free(msg);
-  or_options_free(dflt);
   or_options_free(opt);
   config_free_lines(cl);
   return;
@@ -383,33 +426,8 @@ fixed_get_uname(void)
   return fixed_get_uname_result;
 }
 
-#define TEST_OPTIONS_OLD_VALUES   "TestingV3AuthInitialVotingInterval 1800\n" \
-  "ClientBootstrapConsensusMaxInProgressTries 3\n" \
-  "TestingV3AuthInitialVoteDelay 300\n"   \
-  "TestingV3AuthInitialDistDelay 300\n" \
-  "TestingClientMaxIntervalWithoutRequest 600\n" \
-  "TestingDirConnectionMaxStall 600\n" \
-
-#define TEST_OPTIONS_DEFAULT_VALUES TEST_OPTIONS_OLD_VALUES \
-  "MaxClientCircuitsPending 1\n"                                        \
-  "RendPostPeriod 1000\n"                                               \
-  "KeepAlivePeriod 1\n"                                                 \
-  "ConnLimit 1\n"                                                       \
-  "V3AuthVotingInterval 300\n"                                          \
-  "V3AuthVoteDelay 20\n"                                                \
-  "V3AuthDistDelay 20\n"                                                \
-  "V3AuthNIntervalsValid 3\n"                                           \
-  "ClientUseIPv4 1\n"                                                   \
-  "VirtualAddrNetworkIPv4 127.192.0.0/10\n"                             \
-  "VirtualAddrNetworkIPv6 [FE80::]/10\n"                                \
-  "UseEntryGuards 1\n"                                                  \
-  "Schedulers Vanilla\n"                                                \
-  "ClientDNSRejectInternalAddresses 1\n"
-
 typedef struct {
-  or_options_t *old_opt;
   or_options_t *opt;
-  or_options_t *def_opt;
 } options_test_data_t;
 
 static void free_options_test_data(options_test_data_t *td);
@@ -422,17 +440,12 @@ get_options_test_data(const char *conf)
   config_line_t *cl=NULL;
   options_test_data_t *result = tor_malloc(sizeof(options_test_data_t));
   result->opt = options_new();
-  result->old_opt = options_new();
-  result->def_opt = options_new();
 
-  // XXX: Really, all of these options should be set to defaults
-  // with options_init(), but about a dozen tests break when I do that.
-  // Being kinda lame and just fixing the immedate breakage for now..
-  result->opt->ConnectionPadding = -1; // default must be "auto"
+  options_init(result->opt);
 
   rv = config_get_lines(conf, &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->opt, cl, 0, &msg);
+  rv = config_assign(get_options_mgr(), result->opt, cl, 0, &msg);
   if (msg) {
     /* Display the parse error message by comparing it with an empty string */
     tt_str_op(msg, OP_EQ, "");
@@ -441,13 +454,7 @@ get_options_test_data(const char *conf)
   config_free_lines(cl);
   result->opt->LogTimeGranularity = 1;
   result->opt->TokenBucketRefillInterval = 1;
-  rv = config_get_lines(TEST_OPTIONS_OLD_VALUES, &cl, 1);
-  tt_int_op(rv, OP_EQ, 0);
-  rv = config_assign(&options_format, result->def_opt, cl, 0, &msg);
-  if (msg) {
-    /* Display the parse error message by comparing it with an empty string */
-    tt_str_op(msg, OP_EQ, "");
-  }
+  rv = config_get_lines("", &cl, 1);
   tt_int_op(rv, OP_EQ, 0);
 
  done:
@@ -466,9 +473,7 @@ static void
 free_options_test_data(options_test_data_t *td)
 {
   if (!td) return;
-  or_options_free(td->old_opt);
   or_options_free(td->opt);
-  or_options_free(td->def_opt);
   tor_free(td);
 }
 
@@ -491,7 +496,7 @@ test_options_validate__uname_for_server(void *ignored)
 
   MOCK(get_uname, fixed_get_uname);
   fixed_get_uname_result = "Windows 95";
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Tor is running as a server, but you"
            " are running Windows 95; this probably won't work. See https://www"
            ".torproject.org/docs/faq.html#BestOSForRelay for details.\n");
@@ -499,7 +504,7 @@ test_options_validate__uname_for_server(void *ignored)
 
   fixed_get_uname_result = "Windows 98";
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Tor is running as a server, but you"
            " are running Windows 98; this probably won't work. See https://www"
            ".torproject.org/docs/faq.html#BestOSForRelay for details.\n");
@@ -507,7 +512,7 @@ test_options_validate__uname_for_server(void *ignored)
 
   fixed_get_uname_result = "Windows Me";
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Tor is running as a server, but you"
            " are running Windows Me; this probably won't work. See https://www"
            ".torproject.org/docs/faq.html#BestOSForRelay for details.\n");
@@ -515,8 +520,8 @@ test_options_validate__uname_for_server(void *ignored)
 
   fixed_get_uname_result = "Windows 2000";
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  expect_no_log_entry();
+  options_validate(NULL, tdata->opt, &msg);
+  expect_no_log_msg("Tor is running as a server, but you ");
   tor_free(msg);
 
  done:
@@ -539,7 +544,7 @@ test_options_validate__outbound_addresses(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                     "OutboundBindAddress xxyy!!!sdfaf");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Multiple outbound bind addresses configured: "
                         "xxyy!!!sdfaf");
@@ -576,7 +581,7 @@ test_options_validate__data_directory(void *ignored)
                                                 "ONGLONGlongreallylongLONG"
                                                 "LONG"); // 440 characters
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Invalid DataDirectory");
 
@@ -594,7 +599,7 @@ test_options_validate__nickname(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                         "Nickname ThisNickNameIsABitTooLong");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Nickname 'ThisNickNameIsABitTooLong', nicknames must be between "
@@ -604,16 +609,14 @@ test_options_validate__nickname(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("Nickname AMoreValidNick");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("DataDirectory /tmp/somewhere");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
 
  done:
   free_options_test_data(tdata);
@@ -631,8 +634,8 @@ test_options_validate__contactinfo(void *ignored)
   setup_capture_of_logs(LOG_DEBUG);
   tdata->opt->ContactInfo = NULL;
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
            "Your ContactInfo config option is not set. Please strongly "
            "consider setting it, so we can contact you if your relay is "
@@ -645,8 +648,8 @@ test_options_validate__contactinfo(void *ignored)
   tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "ContactInfo hella@example.org");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
            "Your ContactInfo config option is not set. Please strongly "
            "consider setting it, so we can contact you if your relay is "
@@ -673,50 +676,53 @@ test_options_validate__logs(void *ignored)
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
-  tt_str_op(tdata->opt->Logs->value, OP_EQ, "notice stdout");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
   quiet_level = 1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->Logs->key, OP_EQ, "Log");
-  tt_str_op(tdata->opt->Logs->value, OP_EQ, "warn stdout");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
   quiet_level = 2;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 0;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 1, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
-  tt_int_op(ret, OP_EQ, -1);
+  tt_int_op(ret, OP_EQ, 0);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
   tdata->opt->Logs = NULL;
   tdata->opt->RunAsDaemon = 1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_assert(!tdata->opt->Logs);
   tor_free(msg);
+#ifdef _WIN32
+  /* Can't RunAsDaemon on Windows. */
   tt_int_op(ret, OP_EQ, -1);
+#else
+  tt_int_op(ret, OP_EQ, 0);
+#endif /* defined(_WIN32) */
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
@@ -724,7 +730,7 @@ test_options_validate__logs(void *ignored)
   config_line_t *cl=NULL;
   config_get_lines("Log foo", &cl, 1);
   tdata->opt->Logs = cl;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op((intptr_t)tdata->opt->Logs, OP_EQ, (intptr_t)cl);
   tt_int_op(ret, OP_EQ, -1);
 
@@ -752,13 +758,14 @@ test_options_validate__authdir(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_INFO);
   options_test_data_t *tdata = get_options_test_data(
-                                 "AuthoritativeDirectory 1\n"
+                                 ENABLE_AUTHORITY_V3_MIN
                                  "Address this.should.not!exist!.example.org");
+  const dirauth_options_t *da_opt;
 
   sandbox_disable_getaddrinfo_cache();
 
   MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   UNMOCK(tor_addr_lookup);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to resolve/guess local address. See logs for"
@@ -768,246 +775,217 @@ test_options_validate__authdir(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Authoritative directory servers must set "
                         "ContactInfo");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Authoritative directory servers must set ContactInfo");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_MIN
                                 "Address 100.200.10.1\n"
                                 "TestingTorNetwork 1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
             "AuthoritativeDir is set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY);
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
             "AuthoritativeDir is set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "RecommendedVersions 1.2, 3.14\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "RecommendedVersions 1.2, 3.14\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "1.2, 3.14");
-  tt_str_op(tdata->opt->RecommendedServerVersions->value, OP_EQ, "1.2, 3.14");
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
+  tt_str_op(da_opt->RecommendedClientVersions->value, OP_EQ, "1.2, 3.14");
+  tt_str_op(da_opt->RecommendedServerVersions->value, OP_EQ, "1.2, 3.14");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
-                                "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedServerVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(tdata->opt->RecommendedClientVersions->value, OP_EQ, "25");
-  tt_str_op(tdata->opt->RecommendedServerVersions->value, OP_EQ, "4.18");
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
+  tt_str_op(da_opt->RecommendedClientVersions->value, OP_EQ, "25");
+  tt_str_op(da_opt->RecommendedServerVersions->value, OP_EQ, "4.18");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY
                                 "VersioningAuthoritativeDirectory 1\n"
                                 "RecommendedVersions 1.2, 3.14\n"
                                 "RecommendedClientVersions 25\n"
-                                "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedServerVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
   tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
             "AuthoritativeDir is set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "VersioningAuthoritativeDirectory 1\n"
-                                "RecommendedServerVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedServerVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
             "Recommended*Versions.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "VersioningAuthoritativeDirectory 1\n"
-                                "RecommendedClientVersions 4.18\n"
-                                "ContactInfo hello@hello.com\n");
+                                "RecommendedClientVersions 4.18\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
+  da_opt = get_dirauth_options(tdata->opt);
   tt_str_op(msg, OP_EQ, "Versioning authoritative dir servers must set "
             "Recommended*Versions.");
   tor_free(msg);
+  da_opt = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "UseEntryGuards 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "UseEntryGuards 1\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Authoritative directory servers "
             "can't set UseEntryGuards. Disabling.\n");
   tt_int_op(tdata->opt->UseEntryGuards, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "DownloadExtraInfo 0\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("Authoritative directories always try"
             " to download extra-info documents. Setting DownloadExtraInfo.\n");
   tt_int_op(tdata->opt->DownloadExtraInfo, OP_EQ, 1);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "DownloadExtraInfo 1\n"
-                                "V3AuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3BandwidthsFile non-existent-file\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  expect_no_log_msg("Authoritative directories always try"
-            " to download extra-info documents. Setting DownloadExtraInfo.\n");
-  tt_int_op(tdata->opt->DownloadExtraInfo, OP_EQ, 1);
+  options_validate(NULL, tdata->opt, &msg);
+  expect_log_msg("Can't open bandwidth file at configured location: "
+                 "non-existent-file\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "GuardfractionFile non-existent-file\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_str_op(msg, OP_EQ, "AuthoritativeDir is set, but none of (Bridge/V3)"
-            "AuthoritativeDir is set.");
+  options_validate(NULL, tdata->opt, &msg);
+  expect_log_msg("Cannot open guardfraction file 'non-existent-file'. "
+                 "Failing.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existent-file\n");
+                                "ORPort 2000\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Running as authoritative directory, but no DirPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_BRIDGE_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "V3BandwidthsFile non-existent-file\n");
+                                "ORPort 2000\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Running as authoritative directory, but no DirPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existent-file\n");
+                                "DirPort 999\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no DirPort set.");
+            "Running as authoritative directory, but no ORPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
+  tdata = get_options_test_data(ENABLE_AUTHORITY_BRIDGE_MIN
                                 "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n"
-                                "GuardfractionFile non-existent-file\n");
+                                "DirPort 999\n"
+                                "ContactInfo hello@hello.com\n");
   mock_clean_saved_logs();
-  options_validate(NULL, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no DirPort set.");
+            "Running as authoritative directory, but no ORPort set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "ClientOnly 1\n");
+  /* We have to call the dirauth-specific function, and fake port parsing,
+   * to hit this case */
+  tdata->opt->DirPort_set = 1;
+  tdata->opt->ORPort_set = 1;
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate_dirauth_mode(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no DirPort set.");
+  tt_str_op(msg, OP_EQ, "Running as authoritative directory, "
+            "but ClientOnly also set.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("AuthoritativeDirectory 1\n"
-                                "Address 100.200.10.1\n"
-                                "DirPort 999\n"
-                                "BridgeAuthoritativeDir 1\n"
-                                "ContactInfo hello@hello.com\n");
+  tdata = get_options_test_data(ENABLE_AUTHORITY_BRIDGE
+                                "ClientOnly 1\n");
+  /* We have to call the dirauth-specific function, and fake port parsing,
+   * to hit this case */
+  tdata->opt->DirPort_set = 1;
+  tdata->opt->ORPort_set = 1;
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate_dirauth_mode(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ,
-            "Running as authoritative directory, but no ORPort set.");
+  tt_str_op(msg, OP_EQ, "Running as authoritative directory, "
+            "but ClientOnly also set.");
   tor_free(msg);
 
-  // TODO: This case can't be reached, since clientonly is used to
-  // check when parsing port lines as well.
-  /* free_options_test_data(tdata); */
-  /* tdata = get_options_test_data("AuthoritativeDirectory 1\n" */
-  /*                               "Address 100.200.10.1\n" */
-  /*                               "DirPort 999\n" */
-  /*                               "ORPort 888\n" */
-  /*                               "ClientOnly 1\n" */
-  /*                               "BridgeAuthoritativeDir 1\n" */
-  /*                               "ContactInfo hello@hello.com\n" ); */
-  /* mock_clean_saved_logs(); */
-  /* ret = options_validate(tdata->old_opt, tdata->opt, */
-  /*                        tdata->def_opt, 0, &msg); */
-  /* tt_int_op(ret, OP_EQ, -1); */
-  /* tt_str_op(msg, OP_EQ, "Running as authoritative directory, " */
-  /*           "but ClientOnly also set."); */
-
  done:
   teardown_capture_of_logs();
   //  sandbox_free_getaddrinfo_cache();
@@ -1020,6 +998,7 @@ test_options_validate__relay_with_hidden_services(void *ignored)
 {
   (void)ignored;
   char *msg;
+  int ret;
   setup_capture_of_logs(LOG_DEBUG);
   options_test_data_t *tdata = get_options_test_data(
                                   "ORPort 127.0.0.1:5555\n"
@@ -1028,7 +1007,8 @@ test_options_validate__relay_with_hidden_services(void *ignored)
                                   "HiddenServicePort 80 127.0.0.1:8080\n"
                                                      );
 
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "Tor is currently configured as a relay and a hidden service. "
             "That's not very secure: you should probably run your hidden servi"
@@ -1041,27 +1021,25 @@ test_options_validate__relay_with_hidden_services(void *ignored)
   tor_free(msg);
 }
 
-// TODO: it doesn't seem possible to hit the case of having no port lines at
-// all, since there will be a default created for SocksPort
-/* static void */
-/* test_options_validate__ports(void *ignored) */
-/* { */
-/*   (void)ignored; */
-/*   int ret; */
-/*   char *msg; */
-/*   setup_capture_of_logs(LOG_WARN); */
-/*   options_test_data_t *tdata = get_options_test_data(""); */
-/*   ret = options_validate(tdata->old_opt, tdata->opt, */
-/*                          tdata->def_opt, 0, &msg); */
-/*   expect_log_msg("SocksPort, TransPort, NATDPort, DNSPort, and ORPort " */
-/*           "are all undefined, and there aren't any hidden services " */
-/*           "configured. " */
-/*           " Tor will still run, but probably won't do anything.\n"); */
-/*  done: */
-/*   teardown_capture_of_logs(); */
-/*   free_options_test_data(tdata); */
-/*   tor_free(msg); */
-/* } */
+static void
+test_options_validate__listen_ports(void *ignored)
+{
+  (void)ignored;
+  int ret;
+  char *msg;
+  setup_capture_of_logs(LOG_WARN);
+  options_test_data_t *tdata = get_options_test_data("SOCKSPort 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  expect_log_msg("SocksPort, TransPort, NATDPort, DNSPort, and ORPort "
+                 "are all undefined, and there aren't any hidden services "
+                 "configured. "
+                 " Tor will still run, but probably won't do anything.\n");
+ done:
+  teardown_capture_of_logs();
+  free_options_test_data(tdata);
+  tor_free(msg);
+}
 
 static void
 test_options_validate__transproxy(void *ignored)
@@ -1075,31 +1053,31 @@ test_options_validate__transproxy(void *ignored)
   // Test default trans proxy
   tdata = get_options_test_data("TransProxyType default\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_DEFAULT);
   tor_free(msg);
 
   // Test pf-divert trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType pf-divert\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
-#if !defined(OpenBSD) && !defined( DARWIN )
+#if !defined(OpenBSD) && !defined(DARWIN)
   tt_str_op(msg, OP_EQ,
           "pf-divert is a OpenBSD-specific and OS X/Darwin-specific feature.");
 #else
   tt_int_op(tdata->opt->TransProxyType_parsed, OP_EQ, TPT_PF_DIVERT);
   tt_str_op(msg, OP_EQ, "Cannot use TransProxyType without "
             "any valid TransPort.");
-#endif /* !defined(OpenBSD) && !defined( DARWIN ) */
+#endif /* !defined(OpenBSD) && !defined(DARWIN) */
   tor_free(msg);
 
   // Test tproxy trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType tproxy\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
 #if !defined(__linux__)
@@ -1114,7 +1092,7 @@ test_options_validate__transproxy(void *ignored)
   // Test ipfw trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType ipfw\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
 
 #ifndef KERNEL_MAY_SUPPORT_IPFW
@@ -1130,7 +1108,7 @@ test_options_validate__transproxy(void *ignored)
   // Test unknown trans proxy
   free_options_test_data(tdata);
   tdata = get_options_test_data("TransProxyType non-existent\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value for TransProxyType");
   tor_free(msg);
@@ -1142,39 +1120,35 @@ test_options_validate__transproxy(void *ignored)
 #if defined(__linux__)
   tdata = get_options_test_data("TransProxyType tproxy\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
 #elif defined(KERNEL_MAY_SUPPORT_IPFW)
   tdata = get_options_test_data("TransProxyType ipfw\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 #elif defined(OpenBSD)
   tdata = get_options_test_data("TransProxyType pf-divert\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 #elif defined(__NetBSD__)
   tdata = get_options_test_data("TransProxyType default\n"
                                 "TransPort 127.0.0.1:123\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 #endif /* defined(__linux__) || ... */
 
   // Assert that a test has run for some TransProxyType
   tt_assert(tdata);
 
-#else /* !(defined(USE_TRANSPARENT)) */
+#else /* !defined(USE_TRANSPARENT) */
   tdata = get_options_test_data("TransPort 127.0.0.1:555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TransPort is disabled in this build.");
   tor_free(msg);
@@ -1185,13 +1159,14 @@ test_options_validate__transproxy(void *ignored)
   tor_free(msg);
 }
 
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static country_t opt_tests_geoip_get_country(const char *country);
+ATTR_UNUSED static int opt_tests_geoip_get_country_called = 0;
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+opt_tests_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  opt_tests_geoip_get_country_called++;
 
   return 1;
 }
@@ -1201,7 +1176,8 @@ test_options_validate__exclude_nodes(void *ignored)
 {
   (void)ignored;
 
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       opt_tests_geoip_get_country);
 
   int ret;
   char *msg;
@@ -1209,8 +1185,8 @@ test_options_validate__exclude_nodes(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                                   "ExcludeExitNodes {us}\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(tdata->opt->ExcludeExitNodesUnion_->list), OP_EQ, 1);
   tt_str_op((char *)
             (smartlist_get(tdata->opt->ExcludeExitNodesUnion_->list, 0)),
@@ -1219,8 +1195,8 @@ test_options_validate__exclude_nodes(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(tdata->opt->ExcludeExitNodesUnion_->list), OP_EQ, 1);
   tt_str_op((char *)
             (smartlist_get(tdata->opt->ExcludeExitNodesUnion_->list, 0)),
@@ -1230,8 +1206,8 @@ test_options_validate__exclude_nodes(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n"
                                 "ExcludeExitNodes {us} {cn}\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(smartlist_len(tdata->opt->ExcludeExitNodesUnion_->list), OP_EQ, 2);
   tt_str_op((char *)
             (smartlist_get(tdata->opt->ExcludeExitNodesUnion_->list, 0)),
@@ -1245,8 +1221,8 @@ test_options_validate__exclude_nodes(void *ignored)
   tdata = get_options_test_data("ExcludeNodes {cn}\n"
                                 "StrictNodes 1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "You have asked to exclude certain relays from all positions "
             "in your circuits. Expect hidden services and other Tor "
@@ -1256,8 +1232,8 @@ test_options_validate__exclude_nodes(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("ExcludeNodes {cn}\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "You have asked to exclude certain relays from all positions "
             "in your circuits. Expect hidden services and other Tor "
@@ -1265,7 +1241,7 @@ test_options_validate__exclude_nodes(void *ignored)
   tor_free(msg);
 
  done:
-  NS_UNMOCK(geoip_get_country);
+  UNMOCK(geoip_get_country);
   teardown_capture_of_logs();
   free_options_test_data(tdata);
   tor_free(msg);
@@ -1281,8 +1257,8 @@ test_options_validate__node_families(void *ignored)
                                      "NodeFamily flux, flax\n"
                                      "NodeFamily somewhere\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->NodeFamilySets);
   tt_int_op(smartlist_len(tdata->opt->NodeFamilySets), OP_EQ, 2);
   tt_str_op((char *)(smartlist_get(
@@ -1299,15 +1275,15 @@ test_options_validate__node_families(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(!tdata->opt->NodeFamilySets);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("NodeFamily !flux\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_assert(tdata->opt->NodeFamilySets);
   tt_int_op(smartlist_len(tdata->opt->NodeFamilySets), OP_EQ, 0);
@@ -1327,14 +1303,14 @@ test_options_validate__token_bucket(void *ignored)
   options_test_data_t *tdata = get_options_test_data("");
 
   tdata->opt->TokenBucketRefillInterval = 0;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TokenBucketRefillInterval must be between 1 and 1000 inclusive.");
   tor_free(msg);
 
   tdata->opt->TokenBucketRefillInterval = 1001;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TokenBucketRefillInterval must be between 1 and 1000 inclusive.");
@@ -1346,29 +1322,6 @@ test_options_validate__token_bucket(void *ignored)
 }
 
 static void
-test_options_validate__recommended_packages(void *ignored)
-{
-  (void)ignored;
-  int ret;
-  char *msg;
-  setup_capture_of_logs(LOG_WARN);
-  options_test_data_t *tdata = get_options_test_data(
-            "RecommendedPackages foo 1.2 http://foo.com sha1=123123123123\n"
-            "RecommendedPackages invalid-package-line\n");
-
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  expect_no_log_msg("Invalid RecommendedPackage line "
-            "invalid-package-line will be ignored\n");
-
- done:
-  escaped(NULL); // This will free the leaking memory from the previous escaped
-  teardown_capture_of_logs();
-  free_options_test_data(tdata);
-  tor_free(msg);
-}
-
-static void
 test_options_validate__fetch_dir(void *ignored)
 {
   (void)ignored;
@@ -1378,7 +1331,7 @@ test_options_validate__fetch_dir(void *ignored)
                                             "FetchDirInfoExtraEarly 1\n"
                                             "FetchDirInfoEarly 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "FetchDirInfoExtraEarly requires that you"
             " also set FetchDirInfoEarly");
@@ -1388,10 +1341,8 @@ test_options_validate__fetch_dir(void *ignored)
   tdata = get_options_test_data("FetchDirInfoExtraEarly 1\n"
                                 "FetchDirInfoEarly 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_NE, "FetchDirInfoExtraEarly requires that you"
-            " also set FetchDirInfoEarly");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1408,7 +1359,7 @@ test_options_validate__conn_limit(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                             "ConnLimit 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConnLimit must be greater than 0, but was set to 0");
   tor_free(msg);
@@ -1416,10 +1367,8 @@ test_options_validate__conn_limit(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("ConnLimit 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "MaxClientCircuitsPending must be between 1 and 1024, "
-            "but was set to 0");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1442,11 +1391,10 @@ test_options_validate__paths_needed(void *ignored)
 
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
-                                      "PathsNeededToBuildCircuits 0.1\n"
-                                      "ConnLimit 1\n");
+                                      "PathsNeededToBuildCircuits 0.1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->PathsNeededToBuildCircuits > 0.24 &&
             tdata->opt->PathsNeededToBuildCircuits < 0.26);
   expect_log_msg("PathsNeededToBuildCircuits is too low. "
@@ -1455,11 +1403,10 @@ test_options_validate__paths_needed(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.99\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.99\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->PathsNeededToBuildCircuits > 0.94 &&
             tdata->opt->PathsNeededToBuildCircuits < 0.96);
   expect_log_msg("PathsNeededToBuildCircuits is "
@@ -1468,14 +1415,13 @@ test_options_validate__paths_needed(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.91\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("PathsNeededToBuildCircuits 0.91\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_assert(tdata->opt->PathsNeededToBuildCircuits > 0.90 &&
             tdata->opt->PathsNeededToBuildCircuits < 0.92);
-  expect_no_log_entry();
+  expect_no_log_msg_containing("PathsNeededToBuildCircuits");
   tor_free(msg);
 
  done:
@@ -1495,32 +1441,28 @@ test_options_validate__max_client_circuits(void *ignored)
   int ret;
   char *msg;
   options_test_data_t *tdata = get_options_test_data(
-                                           "MaxClientCircuitsPending 0\n"
-                                           "ConnLimit 1\n");
+                                             "MaxClientCircuitsPending 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "MaxClientCircuitsPending must be between 1 and 1024,"
             " but was set to 0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("MaxClientCircuitsPending 1025\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("MaxClientCircuitsPending 1025\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "MaxClientCircuitsPending must be between 1 and 1024,"
             " but was set to 1025");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("MaxClientCircuitsPending 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "KeepalivePeriod option must be positive.");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1534,23 +1476,18 @@ test_options_validate__ports(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                      "FirewallPorts 65537\n"
-                                      "MaxClientCircuitsPending 1\n"
-                                      "ConnLimit 1\n");
+  options_test_data_t *tdata = get_options_test_data("FirewallPorts 65537\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '65537' out of range in FirewallPorts");
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("FirewallPorts 1\n"
-                                "LongLivedPorts 124444\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "LongLivedPorts 124444\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '124444' out of range in LongLivedPorts");
   tor_free(msg);
@@ -1558,11 +1495,9 @@ test_options_validate__ports(void *ignored)
   free_options_test_data(tdata);
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 2\n"
-                                "RejectPlaintextPorts 112233\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "RejectPlaintextPorts 112233\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '112233' out of range in RejectPlaintextPorts");
   tor_free(msg);
@@ -1571,11 +1506,9 @@ test_options_validate__ports(void *ignored)
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 2\n"
                                 "RejectPlaintextPorts 3\n"
-                                "WarnPlaintextPorts 65536\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "WarnPlaintextPorts 65536\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Port '65536' out of range in WarnPlaintextPorts");
   tor_free(msg);
@@ -1584,13 +1517,10 @@ test_options_validate__ports(void *ignored)
   tdata = get_options_test_data("FirewallPorts 1\n"
                                 "LongLivedPorts 2\n"
                                 "RejectPlaintextPorts 3\n"
-                                "WarnPlaintextPorts 4\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "WarnPlaintextPorts 4\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "KeepalivePeriod option must be positive.");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
@@ -1605,13 +1535,10 @@ test_options_validate__reachable_addresses(void *ignored)
   int ret;
   char *msg;
   setup_capture_of_logs(LOG_NOTICE);
-  options_test_data_t *tdata = get_options_test_data(
-                                     "FascistFirewall 1\n"
-                                     "MaxClientCircuitsPending 1\n"
-                                     "ConnLimit 1\n");
+  options_test_data_t *tdata = get_options_test_data("FascistFirewall 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Converting FascistFirewall config "
             "option to new format: \"ReachableDirAddresses *:80\"\n");
   tt_str_op(tdata->opt->ReachableDirAddresses->value, OP_EQ, "*:80");
@@ -1624,13 +1551,17 @@ test_options_validate__reachable_addresses(void *ignored)
   mock_clean_saved_logs();
   tdata = get_options_test_data("FascistFirewall 1\n"
                                 "ReachableDirAddresses *:81\n"
-                                "ReachableORAddresses *:444\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
-  tdata->opt->FirewallPorts = smartlist_new();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+                                "ReachableORAddresses *:444\n");
+  tt_assert(tdata->opt->FirewallPorts);
+  SMARTLIST_FOREACH(tdata->opt->FirewallPorts, char *, cp, tor_free(cp));
+  smartlist_clear(tdata->opt->FirewallPorts);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+#if 0
+  /* This does not actually produce any logs, and did not produce any relevant
+   * logs before. */
   expect_log_entry();
+#endif
   tt_str_op(tdata->opt->ReachableDirAddresses->value, OP_EQ, "*:81");
   tt_str_op(tdata->opt->ReachableORAddresses->value, OP_EQ, "*:444");
   tor_free(msg);
@@ -1638,12 +1569,10 @@ test_options_validate__reachable_addresses(void *ignored)
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("FascistFirewall 1\n"
-                                "FirewallPort 123\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "FirewallPort 123\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Converting FascistFirewall and "
             "FirewallPorts config options to new format: "
             "\"ReachableAddresses *:123\"\n");
@@ -1655,25 +1584,25 @@ test_options_validate__reachable_addresses(void *ignored)
   tdata = get_options_test_data("FascistFirewall 1\n"
                                 "ReachableAddresses *:82\n"
                                 "ReachableAddresses *:83\n"
-                                "ReachableAddresses reject *:*\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ReachableAddresses reject *:*\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+#if 0
+  /* This does not actually produce any logs, and did not produce any relevant
+   * logs before. */
   expect_log_entry();
+#endif
   tt_str_op(tdata->opt->ReachableAddresses->value, OP_EQ, "*:82");
   tor_free(msg);
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
   tdata = get_options_test_data("FascistFirewall 1\n"
-                                "ReachableAddresses *:82\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ReachableAddresses *:82\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(tdata->opt->ReachableAddresses->next, OP_EQ, NULL);
   tor_free(msg);
 
@@ -1683,44 +1612,36 @@ test_options_validate__reachable_addresses(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableAddresses *:82\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableORAddresses *:82\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ReachableDirAddresses *:82\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("ClientUseIPv4 0\n"
-                                "ORPort 127.0.0.1:5555\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, SERVERS_REACHABLE_MSG);
   tor_free(msg);
@@ -1728,74 +1649,68 @@ test_options_validate__reachable_addresses(void *ignored)
   /* Test IPv4-only clients setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 1\n"
+  tdata = get_options_test_data("ClientUseIPv4 1\n"
                                 "ClientUseIPv6 0\n"
                                 "UseBridges 0\n"
                                 "ClientPreferIPv6ORPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 1\n"
+  tdata = get_options_test_data("ClientUseIPv4 1\n"
                                 "ClientUseIPv6 0\n"
                                 "UseBridges 0\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   /* Now test an IPv4/IPv6 client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 1\n"
+  tdata = get_options_test_data("ClientUseIPv4 1\n"
                                 "ClientUseIPv6 1\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
   /* Now test an IPv6 client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv6 1\n"
+  tdata = get_options_test_data("ClientUseIPv6 1\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
   /* And an implicit (IPv4 disabled) IPv6 client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientUseIPv4 0\n"
+  tdata = get_options_test_data("ClientUseIPv4 0\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
   /* And an implicit (bridge) client setting IPv6 preferences */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n"
+  tdata = get_options_test_data("UseBridges 1\n"
                                 "Bridge 127.0.0.1:12345\n"
                                 "ClientPreferIPv6ORPort 1\n"
                                 "ClientPreferIPv6DirPort 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
@@ -1814,76 +1729,68 @@ test_options_validate__use_bridges(void *ignored)
   options_test_data_t *tdata = get_options_test_data(
                                    "UseBridges 1\n"
                                    "ClientUseIPv4 1\n"
-                                   "ORPort 127.0.0.1:5555\n"
-                                   "MaxClientCircuitsPending 1\n"
-                                   "ConnLimit 1\n");
+                                   "ORPort 127.0.0.1:5555\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Servers must be able to freely connect to the rest of"
             " the Internet, so they must not set UseBridges.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("UseBridges 1\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("UseBridges 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_NE, "Servers must be able to freely connect to the rest of"
             " the Internet, so they must not set UseBridges.");
   tor_free(msg);
 
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       opt_tests_geoip_get_country);
   free_options_test_data(tdata);
   tdata = get_options_test_data("UseBridges 1\n"
-                                "EntryNodes {cn}\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "EntryNodes {cn}\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "You cannot set both UseBridges and EntryNodes.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n");
+  tdata = get_options_test_data("UseBridges 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "If you set UseBridges, you must specify at least one bridge.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n"
+  tdata = get_options_test_data("UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
                                 "UseEntryGuards 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Setting UseBridges requires also setting UseEntryGuards.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "UseBridges 1\n"
+  tdata = get_options_test_data("UseBridges 1\n"
                                 "Bridge 10.0.0.1\n"
                                 "Bridge !!!\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bridge line did not parse. See logs for details.");
   tor_free(msg);
 
  done:
-  NS_UNMOCK(geoip_get_country);
+  UNMOCK(geoip_get_country);
   policies_free_all();
   free_options_test_data(tdata);
   tor_free(msg);
@@ -1895,14 +1802,13 @@ test_options_validate__entry_nodes(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       opt_tests_geoip_get_country);
   options_test_data_t *tdata = get_options_test_data(
                                          "EntryNodes {cn}\n"
-                                         "UseEntryGuards 0\n"
-                                         "MaxClientCircuitsPending 1\n"
-                                         "ConnLimit 1\n");
+                                         "UseEntryGuards 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "If EntryNodes is set, UseEntryGuards must be enabled.");
@@ -1910,17 +1816,14 @@ test_options_validate__entry_nodes(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("EntryNodes {cn}\n"
-                                "UseEntryGuards 1\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+                                "UseEntryGuards 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
-  tt_str_op(msg, OP_EQ, "KeepalivePeriod option must be positive.");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
  done:
-  NS_UNMOCK(geoip_get_country);
+  UNMOCK(geoip_get_country);
   free_options_test_data(tdata);
   tor_free(msg);
 }
@@ -1931,51 +1834,41 @@ test_options_validate__safe_logging(void *ignored)
   (void)ignored;
   int ret;
   char *msg;
-  options_test_data_t *tdata = get_options_test_data(
-                                            "MaxClientCircuitsPending 1\n"
-                                            "ConnLimit 1\n");
+  options_test_data_t *tdata = get_options_test_data("SafeLogging 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_NONE);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging 0\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging 0\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_NONE);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging Relay\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging Relay\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_RELAY);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging 1\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, -1);
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->SafeLogging_, OP_EQ, SAFELOG_SCRUB_ALL);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("SafeLogging stuffy\n"
-                                "MaxClientCircuitsPending 1\n"
-                                "ConnLimit 1\n");
+  tdata = get_options_test_data("SafeLogging stuffy\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value '\"stuffy\"' in SafeLogging");
   tor_free(msg);
@@ -1994,27 +1887,24 @@ test_options_validate__publish_server_descriptor(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_WARN);
   options_test_data_t *tdata = get_options_test_data(
-             "PublishServerDescriptor bridge\n" TEST_OPTIONS_DEFAULT_VALUES
-                                                     );
+             "PublishServerDescriptor bridge\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("PublishServerDescriptor humma\n"
-                                TEST_OPTIONS_DEFAULT_VALUES);
+  tdata = get_options_test_data("PublishServerDescriptor humma\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Unrecognized value in PublishServerDescriptor");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("PublishServerDescriptor bridge, v3\n"
-                                TEST_OPTIONS_DEFAULT_VALUES);
+  tdata = get_options_test_data("PublishServerDescriptor bridge, v3\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bridges are not supposed to publish router "
             "descriptors to the directory authorities. Please correct your "
@@ -2023,10 +1913,9 @@ test_options_validate__publish_server_descriptor(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("BridgeRelay 1\n"
-                                "PublishServerDescriptor v3\n"
-                                TEST_OPTIONS_DEFAULT_VALUES);
+                                "PublishServerDescriptor v3\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bridges are not supposed to publish router "
             "descriptors to the directory authorities. Please correct your "
@@ -2034,9 +1923,9 @@ test_options_validate__publish_server_descriptor(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data("BridgeRelay 1\n" TEST_OPTIONS_DEFAULT_VALUES);
+  tdata = get_options_test_data("BridgeRelay 1\n");
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_NE, "Bridges are not supposed to publish router "
             "descriptors to the directory authorities. Please correct your "
@@ -2045,10 +1934,10 @@ test_options_validate__publish_server_descriptor(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data("BridgeRelay 1\n"
-                                "DirPort 999\n" TEST_OPTIONS_DEFAULT_VALUES);
+                                "DirPort 999\n");
 
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   expect_log_msg("Can't set a DirPort on a bridge "
             "relay; disabling DirPort\n");
@@ -2073,21 +1962,19 @@ test_options_validate__testing(void *ignored)
 #define ENSURE_DEFAULT(varname, varval)                     \
   STMT_BEGIN                                                \
     free_options_test_data(tdata);                          \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES \
-                                #varname " " #varval "\n"); \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  tdata = get_options_test_data(#varname " " #varval "\n"); \
+  ret = options_validate(NULL, tdata->opt, &msg); \
   tt_str_op(msg, OP_EQ, \
             #varname " may only be changed in testing Tor networks!");  \
   tt_int_op(ret, OP_EQ, -1);                                            \
   tor_free(msg);                                                        \
                                                 \
   free_options_test_data(tdata); \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES \
-                                #varname " " #varval "\n"               \
+  tdata = get_options_test_data(#varname " " #varval "\n"               \
                                 VALID_DIR_AUTH                          \
                                 "TestingTorNetwork 1\n");               \
                                                                         \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  ret = options_validate(NULL, tdata->opt, &msg);             \
   if (msg) { \
     tt_str_op(msg, OP_NE, \
               #varname " may only be changed in testing Tor networks!"); \
@@ -2095,11 +1982,10 @@ test_options_validate__testing(void *ignored)
   } \
                                                                         \
   free_options_test_data(tdata);          \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES \
-                                #varname " " #varval "\n"           \
+  tdata = get_options_test_data(#varname " " #varval "\n"           \
                                 "___UsingTestNetworkDefaults 1\n"); \
                                                                         \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  ret = options_validate(NULL, tdata->opt, &msg);\
   if (msg) { \
     tt_str_op(msg, OP_NE, \
               #varname " may only be changed in testing Tor networks!"); \
@@ -2112,7 +1998,6 @@ test_options_validate__testing(void *ignored)
   ENSURE_DEFAULT(TestingV3AuthInitialDistDelay, 3000);
   ENSURE_DEFAULT(TestingV3AuthVotingStartOffset, 3000);
   ENSURE_DEFAULT(TestingAuthDirTimeToLearnReachability, 3000);
-  ENSURE_DEFAULT(TestingEstimatedDescriptorPropagationTime, 3000);
   ENSURE_DEFAULT(TestingServerDownloadInitialDelay, 3000);
   ENSURE_DEFAULT(TestingClientDownloadInitialDelay, 3000);
   ENSURE_DEFAULT(TestingServerConsensusDownloadInitialDelay, 3000);
@@ -2142,21 +2027,12 @@ test_options_validate__hidserv(void *ignored)
   char *msg;
   setup_capture_of_logs(LOG_WARN);
 
-  options_test_data_t *tdata = get_options_test_data(
-                                                TEST_OPTIONS_DEFAULT_VALUES);
-  tdata->opt->MinUptimeHidServDirectoryV2 = -1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
-  tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("MinUptimeHidServDirectoryV2 "
-            "option must be at least 0 seconds. Changing to 0.\n");
-  tt_int_op(tdata->opt->MinUptimeHidServDirectoryV2, OP_EQ, 0);
-  tor_free(msg);
+  options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RendPostPeriod 1\n" );
+  tdata = get_options_test_data("RendPostPeriod 1\n" );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("RendPostPeriod option is too short;"
             " raising to 600 seconds.\n");
@@ -2164,10 +2040,9 @@ test_options_validate__hidserv(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RendPostPeriod 302401\n" );
+  tdata = get_options_test_data("RendPostPeriod 302401\n" );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("RendPostPeriod is too large; "
             "clipping to 302400s.\n");
@@ -2189,45 +2064,40 @@ test_options_validate__path_bias(void *ignored)
   char *msg;
 
   options_test_data_t *tdata = get_options_test_data(
-                                            TEST_OPTIONS_DEFAULT_VALUES
                                             "PathBiasNoticeRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasNoticeRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasWarnRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasWarnRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasWarnRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasExtremeRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasExtremeRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasExtremeRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasNoticeUseRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasNoticeUseRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "PathBiasNoticeUseRate is too high. It must be between 0 and 1.0");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "PathBiasExtremeUseRate 1.1\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("PathBiasExtremeUseRate 1.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
            "PathBiasExtremeUseRate is too high. It must be between 0 and 1.0");
@@ -2246,130 +2116,141 @@ test_options_validate__bandwidth(void *ignored)
   char *msg;
   options_test_data_t *tdata = NULL;
 
-#define ENSURE_BANDWIDTH_PARAM(p) \
-  STMT_BEGIN                                                \
+#define ENSURE_BANDWIDTH_PARAM(p, EXTRA_OPT_STR) \
+  STMT_BEGIN \
   free_options_test_data(tdata); \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES #p " 3Gb\n"); \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  tdata = get_options_test_data(EXTRA_OPT_STR \
+                                #p " 3Gb\n"); \
+  ret = options_validate(NULL, tdata->opt, &msg); \
   tt_int_op(ret, OP_EQ, -1); \
   tt_mem_op(msg, OP_EQ, #p " (3221225471) must be at most 2147483647", 40); \
   tor_free(msg); \
   STMT_END
 
-  ENSURE_BANDWIDTH_PARAM(BandwidthRate);
-  ENSURE_BANDWIDTH_PARAM(BandwidthBurst);
-  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth);
-  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate);
-  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst);
-  ENSURE_BANDWIDTH_PARAM(PerConnBWRate);
-  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst);
-  ENSURE_BANDWIDTH_PARAM(AuthDirFastGuarantee);
-  ENSURE_BANDWIDTH_PARAM(AuthDirGuardBWGuarantee);
+  ENSURE_BANDWIDTH_PARAM(BandwidthRate, "");
+  ENSURE_BANDWIDTH_PARAM(BandwidthBurst, "");
+
+  ENSURE_BANDWIDTH_PARAM(BandwidthRate, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(BandwidthBurst, ENABLE_AUTHORITY_V3);
+
+  ENSURE_BANDWIDTH_PARAM(BandwidthRate, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(BandwidthBurst, ENABLE_AUTHORITY_BRIDGE);
+
+  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth, "");
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate, "");
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst, "");
+  ENSURE_BANDWIDTH_PARAM(PerConnBWRate, "");
+  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst, "");
+
+  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWRate, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst, ENABLE_AUTHORITY_V3);
+
+  ENSURE_BANDWIDTH_PARAM(MaxAdvertisedBandwidth, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthRate, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(RelayBandwidthBurst, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWRate, ENABLE_AUTHORITY_BRIDGE);
+  ENSURE_BANDWIDTH_PARAM(PerConnBWBurst, ENABLE_AUTHORITY_BRIDGE);
+
+  ENSURE_BANDWIDTH_PARAM(AuthDirFastGuarantee, ENABLE_AUTHORITY_V3);
+  ENSURE_BANDWIDTH_PARAM(AuthDirGuardBWGuarantee, ENABLE_AUTHORITY_V3);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1000\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("RelayBandwidthRate 1000\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->RelayBandwidthBurst, OP_EQ, 1000);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthBurst 1001\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("RelayBandwidthBurst 1001\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->RelayBandwidthRate, OP_EQ, 1001);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1001\n"
+  tdata = get_options_test_data("RelayBandwidthRate 1001\n"
                                 "RelayBandwidthBurst 1000\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "RelayBandwidthBurst must be at least equal to "
             "RelayBandwidthRate.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "BandwidthRate 1001\n"
+  tdata = get_options_test_data("BandwidthRate 1001\n"
                                 "BandwidthBurst 1000\n");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "BandwidthBurst must be at least equal to BandwidthRate.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1001\n"
+  tdata = get_options_test_data("RelayBandwidthRate 1001\n"
                                 "BandwidthRate 1000\n"
                                 "BandwidthBurst 1000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->BandwidthRate, OP_EQ, 1001);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "RelayBandwidthRate 1001\n"
+  tdata = get_options_test_data("RelayBandwidthRate 1001\n"
                                 "BandwidthRate 1000\n"
                                 "RelayBandwidthBurst 1001\n"
                                 "BandwidthBurst 1000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_u64_op(tdata->opt->BandwidthBurst, OP_EQ, 1001);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "BandwidthRate is set to 1 bytes/second. For servers,"
             " it must be at least 76800.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 30000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "MaxAdvertisedBandwidth is set to 30000 bytes/second."
             " For servers, it must be at least 38400.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "RelayBandwidthRate 1\n"
                                 "MaxAdvertisedBandwidth 38400\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "RelayBandwidthRate is set to 1 bytes/second. For "
             "servers, it must be at least 76800.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ORPort 127.0.0.1:5555\n"
+  tdata = get_options_test_data("ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76800\n"
                                 "BandwidthBurst 76800\n"
                                 "RelayBandwidthRate 76800\n"
                                 "MaxAdvertisedBandwidth 38400\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
@@ -2388,9 +2269,8 @@ test_options_validate__circuits(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MaxCircuitDirtiness 2592001\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("MaxCircuitDirtiness 2592001\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("MaxCircuitDirtiness option is too "
             "high; setting to 30 days.\n");
   tt_int_op(tdata->opt->MaxCircuitDirtiness, OP_EQ, 2592000);
@@ -2398,9 +2278,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CircuitStreamTimeout 1\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("CircuitStreamTimeout 1\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("CircuitStreamTimeout option is too"
             " short; raising to 10 seconds.\n");
   tt_int_op(tdata->opt->CircuitStreamTimeout, OP_EQ, 10);
@@ -2408,9 +2287,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CircuitStreamTimeout 111\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("CircuitStreamTimeout 111\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_no_log_msg("CircuitStreamTimeout option is too"
             " short; raising to 10 seconds.\n");
   tt_int_op(tdata->opt->CircuitStreamTimeout, OP_EQ, 111);
@@ -2418,9 +2296,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HeartbeatPeriod 1\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HeartbeatPeriod 1\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("HeartbeatPeriod option is too short;"
             " raising to 1800 seconds.\n");
   tt_int_op(tdata->opt->HeartbeatPeriod, OP_EQ, 1800);
@@ -2428,9 +2305,8 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HeartbeatPeriod 1982\n");
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HeartbeatPeriod 1982\n");
+  options_validate(NULL, tdata->opt, &msg);
   expect_no_log_msg("HeartbeatPeriod option is too short;"
             " raising to 1800 seconds.\n");
   tt_int_op(tdata->opt->HeartbeatPeriod, OP_EQ, 1982);
@@ -2438,10 +2314,10 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data("LearnCircuitBuildTimeout 0\n"
                                 "CircuitBuildTimeout 1\n"
                                 );
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_log_msg("CircuitBuildTimeout is shorter (1"
             " seconds) than the recommended minimum (10 seconds), and "
             "LearnCircuitBuildTimeout is disabled.  If tor isn't working, "
@@ -2450,10 +2326,9 @@ test_options_validate__circuits(void *ignored)
 
   free_options_test_data(tdata);
   mock_clean_saved_logs();
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CircuitBuildTimeout 11\n"
+  tdata = get_options_test_data("CircuitBuildTimeout 11\n"
                                 );
-  options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  options_validate(NULL, tdata->opt, &msg);
   expect_no_log_msg("CircuitBuildTimeout is shorter (1 "
             "seconds) than the recommended minimum (10 seconds), and "
             "LearnCircuitBuildTimeout is disabled.  If tor isn't working, "
@@ -2477,51 +2352,46 @@ test_options_validate__rend(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                  "UseEntryGuards 0\n"
                  "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                  "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("UseEntryGuards is disabled, but you"
             " have configured one or more hidden services on this Tor "
             "instance.  Your hidden services will be very easy to locate using"
-            " a well-known attack -- see http://freehaven.net/anonbib/#hs-"
+            " a well-known attack -- see https://freehaven.net/anonbib/#hs-"
             "attack06 for details.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-            TEST_OPTIONS_DEFAULT_VALUES
             "UseEntryGuards 1\n"
             "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
             "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("UseEntryGuards is disabled, but you"
             " have configured one or more hidden services on this Tor "
             "instance.  Your hidden services will be very easy to locate using"
-            " a well-known attack -- see http://freehaven.net/anonbib/#hs-"
+            " a well-known attack -- see https://freehaven.net/anonbib/#hs-"
             "attack06 for details.\n");
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HiddenServicePort 80 127.0.0.1:8080\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HiddenServicePort 80 127.0.0.1:8080\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Failed to configure rendezvous options. See logs for details.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HidServAuth failed\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HidServAuth failed\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Failed to configure client authorization for hidden "
             "services. See logs for details.");
@@ -2545,11 +2415,10 @@ test_options_validate__single_onion(void *ignored)
 
   /* Test that HiddenServiceSingleHopMode must come with
    * HiddenServiceNonAnonymousMode */
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceSingleHopMode does not provide any "
             "server anonymity. It must be used with "
@@ -2557,12 +2426,11 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 0\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceSingleHopMode does not provide any "
             "server anonymity. It must be used with "
@@ -2570,23 +2438,21 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
   /* Test that SOCKSPort if HiddenServiceSingleHopMode is 1 */
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 5000\n"
+  tdata = get_options_test_data("SOCKSPort 5000\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode is incompatible with "
             "using Tor as an anonymous client. Please set "
@@ -2595,32 +2461,30 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 0\n"
+  tdata = get_options_test_data("SOCKSPort 0\n"
                                 "HiddenServiceSingleHopMode 1\n"
                                 "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "SOCKSPort 5000\n"
+  tdata = get_options_test_data("SOCKSPort 5000\n"
                                 "HiddenServiceSingleHopMode 0\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
   /* Test that a hidden service can't be run in non anonymous mode. */
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceNonAnonymousMode 1\n"
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                   "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode does not provide any "
             "server anonymity. It must be used with "
@@ -2628,10 +2492,10 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceNonAnonymousMode 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HiddenServiceNonAnonymousMode does not provide any "
             "server anonymity. It must be used with "
@@ -2639,23 +2503,23 @@ test_options_validate__single_onion(void *ignored)
   tor_free(msg);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                   "HiddenServicePort 80 127.0.0.1:8080\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(
                   "HiddenServiceNonAnonymousMode 1\n"
                   "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                   "HiddenServicePort 80 127.0.0.1:8080\n"
                   "HiddenServiceSingleHopMode 1\n"
                   "SOCKSPort 0\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_ptr_op(msg, OP_EQ, NULL);
 
@@ -2676,53 +2540,56 @@ test_options_validate__accounting(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingRule something_bad\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule something_bad\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "AccountingRule must be 'sum', 'max', 'in', or 'out'");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingRule sum\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule sum\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_SUM);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingRule max\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule max\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_MAX);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingStart fail\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingRule in\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_IN);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data("AccountingRule out\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_int_op(tdata->opt->AccountingRule, OP_EQ, ACCT_OUT);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data("AccountingStart fail\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Failed to parse accounting options. See logs for details.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccountingMax 10\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccountingMax 10\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-           TEST_OPTIONS_DEFAULT_VALUES
            "ORPort 127.0.0.1:5555\n"
            "BandwidthRate 76800\n"
            "BandwidthBurst 76800\n"
@@ -2732,7 +2599,7 @@ test_options_validate__accounting(void *ignored)
            "AccountingMax 10\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Using accounting with a hidden "
             "service and an ORPort is risky: your hidden service(s) and "
@@ -2743,13 +2610,12 @@ test_options_validate__accounting(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-                TEST_OPTIONS_DEFAULT_VALUES
                 "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
                 "HiddenServicePort 80 127.0.0.1:8080\n"
                 "AccountingMax 10\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("Using accounting with a hidden "
             "service and an ORPort is risky: your hidden service(s) and "
@@ -2760,7 +2626,6 @@ test_options_validate__accounting(void *ignored)
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-             TEST_OPTIONS_DEFAULT_VALUES
              "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service/\n"
              "HiddenServicePort 80 127.0.0.1:8080\n"
              "HiddenServiceDir /Library/Tor/var/lib/tor/hidden_service2/\n"
@@ -2768,7 +2633,7 @@ test_options_validate__accounting(void *ignored)
              "AccountingMax 10\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("Using accounting with multiple "
             "hidden services is risky: they will all turn off at the same"
@@ -2795,36 +2660,29 @@ test_options_validate__proxy(void *ignored)
   MOCK(tor_addr_lookup, mock_tor_addr_lookup__fail_on_bad_addrs);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPProxyPort, OP_EQ, 80);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
 
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPProxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxyAuthenticator "
+  tdata = get_options_test_data("HttpProxyAuthenticator "
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
                                 "othreonetwothreonetwothreonetwothreonetwothre"
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
@@ -2837,52 +2695,41 @@ test_options_validate__proxy(void *ignored)
                                 "othreonetwothreonetwothreonetwothreonetwothre"
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
                                 "othreonetwothreeonetwothreeonetwothree"
-
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPProxyAuthenticator is too long (>= 512 chars).");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxyAuthenticator validauth\n"
-
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpProxyAuthenticator validauth\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPSProxyPort, OP_EQ, 443);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->HTTPSProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPSProxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxyAuthenticator "
+  tdata = get_options_test_data("HttpsProxyAuthenticator "
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
                                 "othreonetwothreonetwothreonetwothreonetwothre"
                                 "onetwothreonetwothreonetwothreonetwothreonetw"
@@ -2897,103 +2744,86 @@ test_options_validate__proxy(void *ignored)
                                 "othreonetwothreeonetwothreeonetwothree"
 
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "HTTPSProxyAuthenticator is too long (>= 512 chars).");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpsProxyAuthenticator validauth\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("HttpsProxyAuthenticator validauth\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks4Proxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks4ProxyPort, OP_EQ, 1080);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks4Proxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks4ProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks4Proxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks4Proxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5Proxy 127.0.42.1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5Proxy 127.0.42.1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks5ProxyPort, OP_EQ, 1080);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5Proxy 127.0.42.1:444\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5Proxy 127.0.42.1:444\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_int_op(tdata->opt->Socks5ProxyPort, OP_EQ, 444);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5Proxy not_so_valid!\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5Proxy not_so_valid!\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks5Proxy failed to parse or resolve. Please fix.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks4Proxy 215.1.1.1\n"
+  tdata = get_options_test_data("Socks4Proxy 215.1.1.1\n"
                                 "Socks5Proxy 215.1.1.2\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "You have configured more than one proxy type. "
-            "(Socks4Proxy|Socks5Proxy|HTTPSProxy)");
+            "(Socks4Proxy|Socks5Proxy|HTTPSProxy|TCPProxy)");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
-                                );
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  expect_log_msg("HTTPProxy configured, but no SOCKS "
-            "proxy or HTTPS proxy configured. Watch out: this configuration "
-            "will proxy unencrypted directory connections only.\n");
+  expect_log_msg("HTTPProxy configured, but no SOCKS proxy, "
+            "HTTPS proxy, or any other TCP proxy configured. Watch out: "
+            "this configuration will proxy unencrypted directory "
+            "connections only.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n"
                                 "Socks4Proxy 215.1.1.1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("HTTPProxy configured, but no SOCKS "
             "proxy or HTTPS proxy configured. Watch out: this configuration "
@@ -3001,12 +2831,11 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n"
                                 "Socks5Proxy 215.1.1.1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("HTTPProxy configured, but no SOCKS "
             "proxy or HTTPS proxy configured. Watch out: this configuration "
@@ -3014,12 +2843,11 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HttpProxy 215.1.1.1\n"
+  tdata = get_options_test_data("HttpProxy 215.1.1.1\n"
                                 "HttpsProxy 215.1.1.1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "HTTPProxy configured, but no SOCKS proxy or HTTPS proxy "
@@ -3028,81 +2856,69 @@ test_options_validate__proxy(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                );
+  tdata = get_options_test_data("");
   tdata->opt->Socks5ProxyUsername = tor_strdup("");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyUsername must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                );
+  tdata = get_options_test_data("");
   tdata->opt->Socks5ProxyUsername =
     tor_strdup("ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789AB"
                "CDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCD"
                "EABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEA"
                "BCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABC"
                "DE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyUsername must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks5ProxyPassword must be included with "
             "Socks5ProxyUsername.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                );
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n");
   tdata->opt->Socks5ProxyPassword = tor_strdup("");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyPassword must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                );
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n");
   tdata->opt->Socks5ProxyPassword =
     tor_strdup("ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789AB"
                "CDEABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCD"
                "EABCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEA"
                "BCDE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789ABCDEABC"
                "DE0123456789ABCDEABCDE0123456789ABCDEABCDE0123456789");
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Socks5ProxyPassword must be between 1 and 255 characters.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyUsername hello_world\n"
-                                "Socks5ProxyPassword world_hello\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5ProxyUsername hello_world\n"
+                                "Socks5ProxyPassword world_hello\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "Socks5ProxyPassword hello_world\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("Socks5ProxyPassword hello_world\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Socks5ProxyPassword must be included with "
             "Socks5ProxyUsername.");
@@ -3127,69 +2943,62 @@ test_options_validate__control(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HashedControlPassword something_incorrect\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data(
+                         "HashedControlPassword something_incorrect\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Bad HashedControlPassword: wrong length or bad encoding");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "HashedControlPassword 16:872860B76453A77D60CA"
+  tdata = get_options_test_data("HashedControlPassword 16:872860B76453A77D60CA"
                                 "2BB8C1A7042072093276A3D701AD684053EC4C\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-                   TEST_OPTIONS_DEFAULT_VALUES
                    "__HashedControlSessionPassword something_incorrect\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bad HashedControlSessionPassword: wrong length or "
             "bad encoding");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "__HashedControlSessionPassword 16:872860B7645"
+  tdata = get_options_test_data("__HashedControlSessionPassword 16:872860B7645"
                                 "3A77D60CA2BB8C1A7042072093276A3D701AD684053EC"
                                 "4C\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
   tdata = get_options_test_data(
-                           TEST_OPTIONS_DEFAULT_VALUES
                            "__OwningControllerProcess something_incorrect\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Bad OwningControllerProcess: invalid PID");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "__OwningControllerProcess 123\n"
+  tdata = get_options_test_data("__OwningControllerProcess 123\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3199,13 +3008,12 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 "HashedControlPassword 16:872860B76453A77D60CA"
                                 "2BB8C1A7042072093276A3D701AD684053EC4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3215,14 +3023,13 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 "__HashedControlSessionPassword 16:872860B7645"
                                 "3A77D60CA2BB8C1A7042072093276A3D701AD684053EC"
                                 "4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3232,12 +3039,11 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlPort 127.0.0.1:1234\n"
+  tdata = get_options_test_data("ControlPort 127.0.0.1:1234\n"
                                 "CookieAuthentication 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlPort is open, but no authentication method has been "
@@ -3248,11 +3054,9 @@ test_options_validate__control(void *ignored)
 
 #ifdef HAVE_SYS_UN_H
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
-                                );
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3262,13 +3066,12 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n"
                                 "HashedControlPassword 16:872860B76453A77D60CA"
                                 "2BB8C1A7042072093276A3D701AD684053EC4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3278,14 +3081,13 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n"
                                 "__HashedControlSessionPassword 16:872860B7645"
                                 "3A77D60CA2BB8C1A7042072093276A3D701AD684053EC"
                                 "4C\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3295,12 +3097,11 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ControlSocket unix:/tmp WorldWritable\n"
+  tdata = get_options_test_data("ControlSocket unix:/tmp WorldWritable\n"
                                 "CookieAuthentication 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "ControlSocket is world writable, but no authentication method has"
@@ -3311,11 +3112,10 @@ test_options_validate__control(void *ignored)
 #endif /* defined(HAVE_SYS_UN_H) */
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CookieAuthFileGroupReadable 1\n"
+  tdata = get_options_test_data("CookieAuthFileGroupReadable 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "CookieAuthFileGroupReadable is set, but will have no effect: you "
@@ -3324,12 +3124,11 @@ test_options_validate__control(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "CookieAuthFileGroupReadable 1\n"
+  tdata = get_options_test_data("CookieAuthFileGroupReadable 1\n"
                                 "CookieAuthFile /tmp/somewhere\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "CookieAuthFileGroupReadable is set, but will have no effect: you "
@@ -3354,8 +3153,7 @@ test_options_validate__families(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MyFamily home\n"
+  tdata = get_options_test_data("MyFamily home\n"
                                 "BridgeRelay 1\n"
                                 "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 51300\n"
@@ -3364,7 +3162,7 @@ test_options_validate__families(void *ignored)
                                 "DirCache 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "Listing a family for a bridge relay is not supported: it can "
@@ -3374,11 +3172,9 @@ test_options_validate__families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MyFamily home\n"
-                                );
+  tdata = get_options_test_data("MyFamily home\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "Listing a family for a bridge relay is not supported: it can "
@@ -3388,22 +3184,18 @@ test_options_validate__families(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "MyFamily !\n"
-                                );
+  tdata = get_options_test_data("MyFamily !\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Invalid nickname '!' in MyFamily line");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "NodeFamily foo\n"
-                                "NodeFamily !\n"
-                                );
+  tdata = get_options_test_data("NodeFamily foo\n"
+                                "NodeFamily !\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_assert(!msg);
   tor_free(msg);
@@ -3424,11 +3216,10 @@ test_options_validate__addr_policies(void *ignored)
   options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ExitPolicy !!!\n"
+  tdata = get_options_test_data("ExitPolicy !!!\n"
                                 "ExitRelay 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Error in ExitPolicy entry.");
   tor_free(msg);
@@ -3449,12 +3240,11 @@ test_options_validate__dir_auth(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                VALID_DIR_AUTH
+  tdata = get_options_test_data(VALID_DIR_AUTH
                                 VALID_ALT_DIR_AUTH
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Directory authority/fallback line did not parse. See logs for "
@@ -3464,10 +3254,8 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingTorNetwork 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("TestingTorNetwork 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TestingTorNetwork may only be configured in combination with a "
@@ -3476,20 +3264,18 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                VALID_DIR_AUTH
+  tdata = get_options_test_data(VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingTorNetwork 1\n"
+  tdata = get_options_test_data("TestingTorNetwork 1\n"
                                 VALID_ALT_DIR_AUTH
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TestingTorNetwork may only be configured in combination with a "
@@ -3498,11 +3284,10 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingTorNetwork 1\n"
+  tdata = get_options_test_data("TestingTorNetwork 1\n"
                                 VALID_ALT_BRIDGE_AUTH
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingTorNetwork may only be configured in "
             "combination with a non-default set of DirAuthority or both of "
@@ -3510,12 +3295,11 @@ test_options_validate__dir_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                VALID_ALT_DIR_AUTH
+  tdata = get_options_test_data(VALID_ALT_DIR_AUTH
                                 VALID_ALT_BRIDGE_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
@@ -3536,11 +3320,10 @@ test_options_validate__transport(void *ignored)
   setup_capture_of_logs(LOG_NOTICE);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientTransportPlugin !!\n"
+  tdata = get_options_test_data("ClientTransportPlugin !!\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Invalid client transport line. See logs for details.");
@@ -3549,20 +3332,17 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ClientTransportPlugin foo exec bar\n"
+  tdata = get_options_test_data("ClientTransportPlugin foo exec bar\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportPlugin !!\n"
-                                );
+  tdata = get_options_test_data("ServerTransportPlugin !!\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Invalid server transport line. See logs for details.");
@@ -3571,11 +3351,9 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportPlugin foo exec bar\n"
-                                );
+  tdata = get_options_test_data("ServerTransportPlugin foo exec bar\n");
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "Tor is not configured as a relay but you specified a "
@@ -3584,15 +3362,14 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportPlugin foo exec bar\n"
+  tdata = get_options_test_data("ServerTransportPlugin foo exec bar\n"
                                 "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
                                 "BandwidthBurst 76900\n"
                                 "MaxAdvertisedBandwidth 38500\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "Tor is not configured as a relay but you specified a "
@@ -3601,22 +3378,19 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportListenAddr foo 127.0.0.42:55\n"
-                                "ServerTransportListenAddr !\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("ServerTransportListenAddr foo 127.0.0.42:55\n"
+                                "ServerTransportListenAddr !\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "ServerTransportListenAddr did not parse. See logs for details.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportListenAddr foo 127.0.0.42:55\n"
+  tdata = get_options_test_data("ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg(
             "You need at least a single managed-proxy to specify a transport "
@@ -3625,8 +3399,7 @@ test_options_validate__transport(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ServerTransportListenAddr foo 127.0.0.42:55\n"
+  tdata = get_options_test_data("ServerTransportListenAddr foo 127.0.0.42:55\n"
                                 "ServerTransportPlugin foo exec bar\n"
                                 "ORPort 127.0.0.1:5555\n"
                                 "BandwidthRate 76900\n"
@@ -3634,7 +3407,7 @@ test_options_validate__transport(void *ignored)
                                 "MaxAdvertisedBandwidth 38500\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "You need at least a single managed-proxy to specify a transport "
@@ -3659,50 +3432,46 @@ test_options_validate__constrained_sockets(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 0\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConstrainedSockSize is invalid.  Must be a value "
             "between 2048 and 262144 in 1024 byte increments.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 263168\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConstrainedSockSize is invalid.  Must be a value "
             "between 2048 and 262144 in 1024 byte increments.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 2047\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "ConstrainedSockSize is invalid.  Must be a value "
             "between 2048 and 262144 in 1024 byte increments.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 2048\n"
                                 "DirPort 999\n"
                                 "DirCache 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("You have requested constrained "
             "socket buffers while also serving directory entries via DirPort."
@@ -3711,12 +3480,11 @@ test_options_validate__constrained_sockets(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "ConstrainedSockets 1\n"
+  tdata = get_options_test_data("ConstrainedSockets 1\n"
                                 "ConstrainedSockSize 2048\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg(
             "You have requested constrained socket buffers while also serving"
@@ -3742,12 +3510,12 @@ test_options_validate__v3_auth(void *ignored)
   setup_capture_of_logs(LOG_WARN);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 1000\n"
                                 "V3AuthDistDelay 1000\n"
                                 "V3AuthVotingInterval 1000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
@@ -3755,20 +3523,18 @@ test_options_validate__v3_auth(void *ignored)
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "V3AuthVoteDelay 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3AuthVoteDelay 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVoteDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 1\n"
-                                "TestingTorNetwork 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+                                "TestingTorNetwork 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVoteDelay is way too low.");
   tor_free(msg);
@@ -3778,87 +3544,127 @@ test_options_validate__v3_auth(void *ignored)
   // since they are the same
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "V3AuthDistDelay 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3AuthDistDelay 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthDistDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthDistDelay 1\n"
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthDistDelay is way too low.");
   tor_free(msg);
 
-  // TODO: we can't reach the case of v3authdistdelay lower than
+  // We can't reach the case of v3authdistdelay lower than
   // MIN_DIST_SECONDS but not lower than MIN_DIST_SECONDS_TESTING,
   // since they are the same
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthNIntervalsValid 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthNIntervalsValid must be at least 2.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 200\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVotingInterval is insanely low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "V3AuthVoteDelay 49\n"
+                                "V3AuthDistDelay 49\n"
+                                "V3AuthVotingInterval 200\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "V3AuthVoteDelay 2\n"
+                                "V3AuthDistDelay 2\n"
+                                "V3AuthVotingInterval 9\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "V3AuthVoteDelay plus V3AuthDistDelay must be less than half "
+            "V3AuthVotingInterval");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "V3AuthVoteDelay 2\n"
+                                "V3AuthDistDelay 2\n"
+                                "V3AuthVotingInterval 10\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 200000\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "V3AuthVotingInterval is insanely high.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 1441\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("V3AuthVotingInterval does not divide"
             " evenly into 24 hours.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 1440\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_no_log_msg("V3AuthVotingInterval does not divide"
             " evenly into 24 hours.\n");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "V3AuthVoteDelay 49\n"
                                 "V3AuthDistDelay 49\n"
                                 "V3AuthVotingInterval 299\n"
@@ -3866,84 +3672,125 @@ test_options_validate__v3_auth(void *ignored)
                                 "TestingTorNetwork 1\n"
                                 );
   mock_clean_saved_logs();
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   expect_log_msg("V3AuthVotingInterval is very low. "
             "This may lead to failure to synchronise for a consensus.\n");
   tor_free(msg);
 
-  // TODO: It is impossible to reach the case of testingtor network, with
-  // v3authvotinginterval too low
-  /* free_options_test_data(tdata); */
-  /* tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES */
-  /*                               "V3AuthVoteDelay 1\n" */
-  /*                               "V3AuthDistDelay 1\n" */
-  /*                               "V3AuthVotingInterval 9\n" */
-                                   /* VALID_DIR_AUTH */
-  /*                               "TestingTorNetwork 1\n" */
-  /*                               ); */
-  /* ret = options_validate(tdata->old_opt, tdata->opt, */
-  /*                        tdata->def_opt, 0, &msg); */
-  /* tt_int_op(ret, OP_EQ, -1); */
-  /* tt_str_op(msg, OP_EQ, "V3AuthVotingInterval is insanely low."); */
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                "V3AuthVoteDelay 1\n"
+                                "V3AuthDistDelay 1\n"
+                                "V3AuthVotingInterval 9\n"
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                );
+  /* We have to call the dirauth-specific function to reach this case */
+  ret = options_validate_dirauth_schedule(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ, "V3AuthVoteDelay is way too low.");
+  tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "TestingV3AuthInitialVoteDelay 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthInitialVoteDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 "TestingV3AuthInitialDistDelay 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthInitialDistDelay is way too low.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
   tdata->opt->TestingV3AuthVotingStartOffset = 100000;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthVotingStartOffset is higher than the "
             "voting interval.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 );
   tdata->opt->TestingV3AuthVotingStartOffset = -1;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "TestingV3AuthVotingStartOffset must be non-negative.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "TestingV3AuthInitialVotingInterval 4\n"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingV3AuthInitialVotingInterval is insanely low.");
   tor_free(msg);
 
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "TestingV3AuthInitialVoteDelay 2\n"
+                                "TestingV3AuthInitialDistDelay 2\n"
+                                "TestingV3AuthInitialVotingInterval 5\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "TestingV3AuthInitialVotingInterval 7\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "TestingV3AuthInitialVotingInterval does not divide evenly into "
+            "30 minutes.");
+  tor_free(msg);
+
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(ENABLE_AUTHORITY_V3
+                                VALID_DIR_AUTH
+                                "TestingTorNetwork 1\n"
+                                "TestingV3AuthInitialVoteDelay 3\n"
+                                "TestingV3AuthInitialDistDelay 3\n"
+                                "TestingV3AuthInitialVotingInterval 5\n"
+                                );
+  ret = options_validate(NULL, tdata->opt, &msg);
+  tt_int_op(ret, OP_EQ, -1);
+  tt_str_op(msg, OP_EQ,
+            "TestingV3AuthInitialVoteDelay plus "
+            "TestingV3AuthInitialDistDelay must be less than "
+            "TestingV3AuthInitialVotingInterval");
+  tor_free(msg);
+
  done:
   policies_free_all();
   teardown_capture_of_logs();
@@ -3960,19 +3807,16 @@ test_options_validate__virtual_addr(void *ignored)
   options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "VirtualAddrNetworkIPv4 !!"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("VirtualAddrNetworkIPv4 !!");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Error parsing VirtualAddressNetwork !!");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "VirtualAddrNetworkIPv6 !!"
+  tdata = get_options_test_data("VirtualAddrNetworkIPv6 !!"
                                 );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "Error parsing VirtualAddressNetworkIPv6 !!");
   tor_free(msg);
@@ -3993,135 +3837,133 @@ test_options_validate__testing_options(void *ignored)
   options_test_data_t *tdata = NULL;
   setup_capture_of_logs(LOG_WARN);
 
-#define TEST_TESTING_OPTION(name, low_val, high_val, err_low)           \
+#define TEST_TESTING_OPTION(name, accessor, \
+                            low_val, high_val, err_low, EXTRA_OPT_STR)  \
   STMT_BEGIN                                                            \
     free_options_test_data(tdata);                                      \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES             \
+  tdata = get_options_test_data(EXTRA_OPT_STR                           \
                                 VALID_DIR_AUTH                          \
                                 "TestingTorNetwork 1\n"                 \
                                 );                                      \
-  tdata->opt-> name = low_val;                                       \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  accessor(tdata->opt)->name = low_val;                                 \
+  ret = options_validate(NULL, tdata->opt,  &msg);            \
   tt_int_op(ret, OP_EQ, -1);                                            \
   tt_str_op(msg, OP_EQ, #name " " err_low);                \
   tor_free(msg); \
                                                                         \
   free_options_test_data(tdata);                                        \
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES             \
+  tdata = get_options_test_data(EXTRA_OPT_STR                           \
                                 VALID_DIR_AUTH                          \
                                 "TestingTorNetwork 1\n"                 \
                                 );                                      \
-  tdata->opt->  name = high_val;                                      \
+  accessor(tdata->opt)->name = high_val;                                \
   mock_clean_saved_logs();                                              \
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);\
+  ret = options_validate(NULL, tdata->opt,  &msg);            \
   tt_int_op(ret, OP_EQ, 0);                                             \
+  tt_ptr_op(msg, OP_EQ, NULL);                                          \
   expect_log_msg( #name " is insanely high.\n"); \
   tor_free(msg); \
   STMT_END
 
-  TEST_TESTING_OPTION(TestingAuthDirTimeToLearnReachability, -1, 8000,
-                      "must be non-negative.");
-  TEST_TESTING_OPTION(TestingEstimatedDescriptorPropagationTime, -1, 3601,
-                      "must be non-negative.");
-  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, -1, 3601,
-                      "is way too low.");
-  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, 1, 3601,
-                      "is way too low.");
+  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, , -1, 3601,
+                      "is way too low.", "");
+  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, , 1, 3601,
+                      "is way too low.", "");
+
+  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, , -1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_V3);
+  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, , 1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_V3);
+
+  TEST_TESTING_OPTION(TestingClientMaxIntervalWithoutRequest, , -1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_BRIDGE);
+  TEST_TESTING_OPTION(TestingDirConnectionMaxStall, , 1, 3601,
+                      "is way too low.", ENABLE_AUTHORITY_BRIDGE);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableConnBwEvent 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("TestingEnableConnBwEvent 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingEnableConnBwEvent may only be changed in "
             "testing Tor networks!");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableConnBwEvent 1\n"
+  tdata = get_options_test_data("TestingEnableConnBwEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableConnBwEvent 1\n"
+  tdata = get_options_test_data("TestingEnableConnBwEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 0\n"
                                 "___UsingTestNetworkDefaults 1\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableCellStatsEvent 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("TestingEnableCellStatsEvent 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ, "TestingEnableCellStatsEvent may only be changed in "
             "testing Tor networks!");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableCellStatsEvent 1\n"
+  tdata = get_options_test_data("TestingEnableCellStatsEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableCellStatsEvent 1\n"
+  tdata = get_options_test_data("TestingEnableCellStatsEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 0\n"
                                 "___UsingTestNetworkDefaults 1\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableTbEmptyEvent 1\n"
+  tdata = get_options_test_data("TestingEnableTbEmptyEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 1\n"
                                 "___UsingTestNetworkDefaults 0\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "TestingEnableTbEmptyEvent 1\n"
+  tdata = get_options_test_data("TestingEnableTbEmptyEvent 1\n"
                                 VALID_DIR_AUTH
                                 "TestingTorNetwork 0\n"
                                 "___UsingTestNetworkDefaults 1\n"
                                 );
 
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tt_assert(!msg);
   tor_free(msg);
@@ -4142,40 +3984,32 @@ test_options_validate__accel(void *ignored)
   options_test_data_t *tdata = NULL;
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelName foo\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelName foo\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tdata->opt->HardwareAccel, OP_EQ, 1);
+  tt_int_op(get_crypto_options(tdata->opt)->HardwareAccel, OP_EQ, 0);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelName foo\n"
-                                );
-  tdata->opt->HardwareAccel = 2;
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelName foo\n");
+  get_crypto_options(tdata->opt)->HardwareAccel = 2;
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
-  tt_int_op(tdata->opt->HardwareAccel, OP_EQ, 2);
+  tt_int_op(get_crypto_options(tdata->opt)->HardwareAccel, OP_EQ, 2);
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelDir 1\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelDir 1\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, -1);
   tt_str_op(msg, OP_EQ,
             "Can't use hardware crypto accelerator dir without engine name.");
   tor_free(msg);
 
   free_options_test_data(tdata);
-  tdata = get_options_test_data(TEST_OPTIONS_DEFAULT_VALUES
-                                "AccelDir 1\n"
-                                "AccelName something\n"
-                                );
-  ret = options_validate(tdata->old_opt, tdata->opt, tdata->def_opt, 0, &msg);
+  tdata = get_options_test_data("AccelDir 1\n"
+                                "AccelName something\n");
+  ret = options_validate(NULL, tdata->opt, &msg);
   tt_int_op(ret, OP_EQ, 0);
   tor_free(msg);
 
@@ -4185,8 +4019,280 @@ test_options_validate__accel(void *ignored)
   tor_free(msg);
 }
 
+static int mocked_granularity;
+
+static void
+mock_set_log_time_granularity(int g)
+{
+  mocked_granularity = g;
+}
+
+static void
+test_options_init_logs_granularity(void *arg)
+{
+  options_test_data_t *tdata = get_options_test_data("");
+  int rv;
+  (void) arg;
+
+  MOCK(set_log_time_granularity, mock_set_log_time_granularity);
+
+  /* Reasonable value. */
+  tdata->opt->LogTimeGranularity = 100;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 100);
+
+  /* Doesn't divide 1000. */
+  tdata->opt->LogTimeGranularity = 249;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 250);
+
+  /* Doesn't divide 1000. */
+  tdata->opt->LogTimeGranularity = 3;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 4);
+
+  /* Not a multiple of 1000. */
+  tdata->opt->LogTimeGranularity = 1500;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 2000);
+
+  /* Reasonable value. */
+  tdata->opt->LogTimeGranularity = 3000;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 3000);
+
+  /* Negative. (Shouldn't be allowed by rest of config parsing.) */
+  tdata->opt->LogTimeGranularity = -1;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, -1);
+
+  /* Very big */
+  tdata->opt->LogTimeGranularity = 3600 * 1000;
+  mocked_granularity = -1;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(mocked_granularity, OP_EQ, 3600 * 1000);
+
+ done:
+  free_options_test_data(tdata);
+  UNMOCK(set_log_time_granularity);
+}
+
+typedef struct {
+  char *name;
+  log_severity_list_t sev;
+  int fd;
+  bool stream;
+} added_log_t;
+
+static smartlist_t *added_logs = NULL;
+
+static void
+mock_add_stream_log_impl(const log_severity_list_t *sev, const char *name,
+                         int fd)
+{
+  added_log_t *a = tor_malloc_zero(sizeof(added_log_t));
+  a->name = tor_strdup(name);
+  memcpy(&a->sev, sev, sizeof(log_severity_list_t));
+  a->fd = fd;
+  a->stream = true;
+  smartlist_add(added_logs, a);
+}
+
+static int
+mock_add_file_log(const log_severity_list_t *sev, const char *name, int fd)
+{
+  added_log_t *a = tor_malloc_zero(sizeof(added_log_t));
+  a->name = tor_strdup(name);
+  memcpy(&a->sev, sev, sizeof(log_severity_list_t));
+  a->fd = fd;
+  smartlist_add(added_logs, a);
+  return 0;
+}
+
+static void
+clear_added_logs(void)
+{
+  SMARTLIST_FOREACH(added_logs, added_log_t *, a,
+                    { tor_free(a->name); tor_free(a); });
+  smartlist_clear(added_logs);
+}
+
+static void
+test_options_init_logs_quiet(void *arg)
+{
+  (void)arg;
+  char *cfg = NULL;
+  options_test_data_t *tdata = get_options_test_data("");
+  char *fn1 = tor_strdup(get_fname_rnd("log"));
+  const added_log_t *a;
+  int rv;
+  tdata->opt->RunAsDaemon = 0;
+
+  added_logs = smartlist_new();
+  MOCK(add_stream_log_impl, mock_add_stream_log_impl);
+  MOCK(add_file_log, mock_add_file_log);
+
+  tt_ptr_op(tdata->opt->Logs, OP_EQ, NULL);
+
+  /* First, try with no configured logs, and make sure that our configured
+     logs match the quiet level. */
+  quiet_level = QUIET_SILENT;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 0);
+
+  quiet_level = QUIET_HUSH;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 1);
+  a = smartlist_get(added_logs, 0);
+  tt_assert(a);
+  tt_assert(a->stream);
+  tt_int_op(a->fd, OP_EQ, fileno(stdout));
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_INFO)], OP_EQ, 0);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_NOTICE)], OP_EQ, 0);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_WARN)], OP_EQ, LD_ALL_DOMAINS);
+  clear_added_logs();
+
+  quiet_level = QUIET_NONE;
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 1);
+  a = smartlist_get(added_logs, 0);
+  tt_assert(a);
+  tt_assert(a->stream);
+  tt_int_op(a->fd, OP_EQ, fileno(stdout));
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_INFO)], OP_EQ, 0);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_NOTICE)], OP_EQ,
+            LD_ALL_DOMAINS);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_WARN)], OP_EQ, LD_ALL_DOMAINS);
+  clear_added_logs();
+
+  /* Make sure that adding a configured log makes the default logs go away. */
+  tor_asprintf(&cfg, "Log info file %s\n", fn1);
+  free_options_test_data(tdata);
+  tdata = get_options_test_data(cfg);
+  rv = options_init_logs(NULL, tdata->opt, 0);
+  tt_int_op(rv, OP_EQ, 0);
+  tt_int_op(smartlist_len(added_logs), OP_EQ, 1);
+  a = smartlist_get(added_logs, 0);
+  tt_assert(a);
+  tt_assert(! a->stream);
+  tt_int_op(a->fd, OP_NE, fileno(stdout));
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_INFO)], OP_EQ, LD_ALL_DOMAINS);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_NOTICE)], OP_EQ,
+            LD_ALL_DOMAINS);
+  tt_u64_op(a->sev.masks[SEVERITY_MASK_IDX(LOG_WARN)], OP_EQ, LD_ALL_DOMAINS);
+
+ done:
+  free_options_test_data(tdata);
+  tor_free(fn1);
+  tor_free(cfg);
+  clear_added_logs();
+  smartlist_free(added_logs);
+  UNMOCK(add_stream_log_impl);
+  UNMOCK(add_file_log);
+}
+
+static int mock_options_act_status = 0;
+static int
+mock_options_act(const or_options_t *old_options)
+{
+  (void)old_options;
+  return mock_options_act_status;
+}
+static int
+mock_options_act_reversible(const or_options_t *old_options, char **msg_out)
+{
+  (void)old_options;
+  (void)msg_out;
+  return 0;
+}
+
+static void
+test_options_trial_assign(void *arg)
+{
+  (void)arg;
+  setopt_err_t v;
+  config_line_t *lines = NULL;
+  char *msg = NULL;
+  int r;
+
+  // replace options_act*() so that we don't actually launch tor here.
+  MOCK(options_act, mock_options_act);
+  MOCK(options_act_reversible, mock_options_act_reversible);
+
+  // Try assigning nothing; that should work.
+  v = options_trial_assign(lines, 0, &msg);
+  if (msg)
+    puts(msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(v, OP_EQ, SETOPT_OK);
+
+  // Assigning a nickname is okay
+  r = config_get_lines("Nickname Hemiramphinae", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(v, OP_EQ, SETOPT_OK);
+  tt_str_op(get_options()->Nickname, OP_EQ, "Hemiramphinae");
+  config_free_lines(lines);
+
+  // We can't change the User; that's a transition error.
+  r = config_get_lines("User Heraclitus", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_int_op(v, OP_EQ, SETOPT_ERR_TRANSITION);
+  tt_str_op(msg, OP_EQ,  "While Tor is running, changing User is not allowed");
+  tor_free(msg);
+  config_free_lines(lines);
+
+  // We can't set the ORPort to nonsense: that's a validation error.
+  r = config_get_lines("ORPort fractabling planished", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_int_op(v, OP_EQ, SETOPT_ERR_PARSE); // (same error code for now)
+  tt_str_op(msg, OP_EQ, "Invalid ORPort configuration");
+  tor_free(msg);
+  config_free_lines(lines);
+
+  // We can't set UseBridges to a non-boolean: that's a parse error.
+  r = config_get_lines("UseBridges ambidextrous", &lines, 0);
+  tt_int_op(r, OP_EQ, 0);
+  v = options_trial_assign(lines, 0, &msg);
+  tt_int_op(v, OP_EQ, SETOPT_ERR_PARSE);
+  tt_str_op(msg, OP_EQ,
+            "Could not parse UseBridges: Unrecognized value ambidextrous. "
+            "Allowed values are 0 and 1.");
+  tor_free(msg);
+  config_free_lines(lines);
+
+  // this didn't change.
+  tt_str_op(get_options()->Nickname, OP_EQ, "Hemiramphinae");
+
+ done:
+  config_free_lines(lines);
+  tor_free(msg);
+  UNMOCK(options_act);
+  UNMOCK(options_act_reversible);
+}
+
+#ifndef COCCI
 #define LOCAL_VALIDATE_TEST(name) \
   { "validate__" #name, test_options_validate__ ## name, TT_FORK, NULL, NULL }
+#endif
 
 struct testcase_t options_tests[] = {
   { "validate", test_options_validate, TT_FORK, NULL, NULL },
@@ -4199,11 +4305,11 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(logs),
   LOCAL_VALIDATE_TEST(authdir),
   LOCAL_VALIDATE_TEST(relay_with_hidden_services),
+  LOCAL_VALIDATE_TEST(listen_ports),
   LOCAL_VALIDATE_TEST(transproxy),
   LOCAL_VALIDATE_TEST(exclude_nodes),
   LOCAL_VALIDATE_TEST(node_families),
   LOCAL_VALIDATE_TEST(token_bucket),
-  LOCAL_VALIDATE_TEST(recommended_packages),
   LOCAL_VALIDATE_TEST(fetch_dir),
   LOCAL_VALIDATE_TEST(conn_limit),
   LOCAL_VALIDATE_TEST(paths_needed),
@@ -4233,5 +4339,10 @@ struct testcase_t options_tests[] = {
   LOCAL_VALIDATE_TEST(virtual_addr),
   LOCAL_VALIDATE_TEST(testing_options),
   LOCAL_VALIDATE_TEST(accel),
+  { "init_logs/granularity", test_options_init_logs_granularity, TT_FORK,
+    NULL, NULL },
+  { "init_logs/quiet", test_options_init_logs_quiet, TT_FORK,
+    NULL, NULL },
+  { "trial_assign", test_options_trial_assign, TT_FORK, NULL, NULL },
   END_OF_TESTCASES              /*  */
 };
diff --git a/src/test/test_options_act.c b/src/test/test_options_act.c
new file mode 100644
index 0000000000..942584bffd
--- /dev/null
+++ b/src/test/test_options_act.c
@@ -0,0 +1,272 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define CONFIG_PRIVATE
+#include "core/or/or.h"
+#include "app/config/config.h"
+#include "lib/encoding/confline.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+#include "test/test_helpers.h"
+
+#ifndef _WIN32
+#include <sys/stat.h>
+
+/**
+ * Check whether fname is readable. On success set
+ * *<b>is_group_readable_out</b> to as appropriate and return 0. On failure
+ * return -1.
+ */
+static int
+get_file_mode(const char *fname, unsigned *permissions_out)
+{
+  struct stat st;
+  int r = stat(fname, &st);
+  if (r < 0)
+    return -1;
+  *permissions_out = (unsigned) st.st_mode;
+  return 0;
+}
+#define assert_mode(fn,mask,expected) STMT_BEGIN                 \
+  unsigned mode_;                                                \
+  int tmp_ = get_file_mode((fn), &mode_);                        \
+  if (tmp_ < 0) {                                                \
+    TT_DIE(("Couldn't stat %s: %s", (fn), strerror(errno)));     \
+  }                                                              \
+  if ((mode_ & (mask)) != (expected)) {                          \
+    TT_DIE(("Bad mode %o on %s", mode_, (fn)));                  \
+  }                                                              \
+  STMT_END
+#else /* defined(_WIN32) */
+/* "group-readable" isn't meaningful on windows */
+#define assert_mode(fn,mask,expected) STMT_NIL
+#endif /* !defined(_WIN32) */
+
+static or_options_t *mock_opts;
+static const or_options_t *
+mock_get_options(void)
+{
+  return mock_opts;
+}
+
+static void
+test_options_act_create_dirs(void *arg)
+{
+  (void)arg;
+  MOCK(get_options, mock_get_options);
+  char *msg = NULL;
+  or_options_t *opts = mock_opts = options_new();
+
+  /* We're testing options_create_directories(), which assumes that
+     validate_data_directories() has already been called, and all of
+     KeyDirectory, DataDirectory, and CacheDirectory are set. */
+
+  /* Success case 1: all directories are the default */
+  char *fn;
+  fn = tor_strdup(get_fname_rnd("ddir"));
+  opts->DataDirectory = tor_strdup(fn);
+  opts->CacheDirectory = tor_strdup(fn);
+  tor_asprintf(&opts->KeyDirectory, "%s/keys", fn);
+  opts->DataDirectoryGroupReadable = 1;
+  opts->CacheDirectoryGroupReadable = -1; /* default. */
+  int r = options_create_directories(&msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->DataDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->CacheDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->KeyDirectory));
+  assert_mode(opts->DataDirectory, 0777, 0750);
+  assert_mode(opts->KeyDirectory, 0777, 0700);
+  tor_free(fn);
+  tor_free(opts->KeyDirectory);
+  or_options_free(opts);
+
+  /* Success case 2: all directories are different. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  opts->CacheDirectoryGroupReadable = 1; // cache directory group readable
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->DataDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->CacheDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->KeyDirectory));
+  assert_mode(opts->DataDirectory, 0777, 0700);
+  assert_mode(opts->KeyDirectory, 0777, 0700);
+  assert_mode(opts->CacheDirectory, 0777, 0750);
+  tor_free(fn);
+  or_options_free(opts);
+
+  /* Success case 3: all directories are the same. */
+  opts = mock_opts = options_new();
+  fn = tor_strdup(get_fname_rnd("ddir"));
+  opts->DataDirectory = tor_strdup(fn);
+  opts->CacheDirectory = tor_strdup(fn);
+  opts->KeyDirectory = tor_strdup(fn);
+  opts->DataDirectoryGroupReadable = 1;
+  opts->CacheDirectoryGroupReadable = -1; /* default. */
+  opts->KeyDirectoryGroupReadable = -1; /* default */
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_EQ, 0);
+  tt_ptr_op(msg, OP_EQ, NULL);
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->DataDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->CacheDirectory));
+  tt_int_op(FN_DIR, OP_EQ, file_status(opts->KeyDirectory));
+  assert_mode(opts->DataDirectory, 0777, 0750);
+  assert_mode(opts->KeyDirectory, 0777, 0750);
+  assert_mode(opts->CacheDirectory, 0777, 0750);
+  tor_free(fn);
+  or_options_free(opts);
+
+  /* Failure case 1: Can't make datadir. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  write_str_to_file(opts->DataDirectory, "foo", 0);
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_assert(!strcmpstart(msg, "Couldn't create private data directory"));
+  or_options_free(opts);
+  tor_free(msg);
+
+  /* Failure case 2: Can't make keydir. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  write_str_to_file(opts->KeyDirectory, "foo", 0);
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_assert(!strcmpstart(msg, "Couldn't create private data directory"));
+  or_options_free(opts);
+  tor_free(msg);
+
+  /* Failure case 3: Can't make cachedir. */
+  opts = mock_opts = options_new();
+  opts->DataDirectory = tor_strdup(get_fname_rnd("ddir"));
+  opts->CacheDirectory = tor_strdup(get_fname_rnd("cdir"));
+  opts->KeyDirectory = tor_strdup(get_fname_rnd("kdir"));
+  write_str_to_file(opts->CacheDirectory, "foo", 0);
+  r = options_create_directories(&msg);
+  tt_int_op(r, OP_LT, 0);
+  tt_assert(!strcmpstart(msg, "Couldn't create private data directory"));
+  tor_free(fn);
+  or_options_free(opts);
+  tor_free(msg);
+
+ done:
+  UNMOCK(get_options);
+  or_options_free(opts);
+  mock_opts = NULL;
+  tor_free(fn);
+  tor_free(msg);
+}
+
+static void
+test_options_act_log_transition(void *arg)
+{
+  (void)arg;
+  or_options_t *opts = mock_opts = options_new();
+  or_options_t *old_opts = NULL;
+  opts->LogTimeGranularity = 1000;
+  opts->SafeLogging_ = SAFELOG_SCRUB_ALL;
+  struct log_transaction_t *lt = NULL;
+  char *msg = NULL;
+  MOCK(get_options, mock_get_options);
+
+  tt_ptr_op(opts->Logs, OP_EQ, NULL);
+  config_line_append(&opts->Logs, "Log", "notice stdout");
+  lt = options_start_log_transaction(NULL, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+
+  // commit, see that there is a change.
+  options_commit_log_transaction(lt);
+  lt=NULL;
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_NOTICE);
+
+  // Now drop to debug.
+  old_opts = opts;
+  opts = mock_opts = options_new();
+  opts->LogTimeGranularity = 1000;
+  opts->SafeLogging_ = SAFELOG_SCRUB_ALL;
+  config_line_append(&opts->Logs, "Log", "debug stdout");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+
+  setup_full_capture_of_logs(LOG_NOTICE);
+  options_commit_log_transaction(lt);
+  lt=NULL;
+  expect_single_log_msg_containing("may contain sensitive information");
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+  // Turn off SafeLogging
+  or_options_free(old_opts);
+  mock_clean_saved_logs();
+  old_opts = opts;
+  opts = mock_opts = options_new();
+  opts->SafeLogging_ = SAFELOG_SCRUB_NONE;
+  opts->LogTimeGranularity = 1000;
+  config_line_append(&opts->Logs, "Log", "debug stdout");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+  options_commit_log_transaction(lt);
+  lt=NULL;
+  expect_single_log_msg_containing("may contain sensitive information");
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+  // Try rolling back.
+  or_options_free(old_opts);
+  mock_clean_saved_logs();
+  old_opts = opts;
+  opts = mock_opts = options_new();
+  opts->SafeLogging_ = SAFELOG_SCRUB_NONE;
+  opts->LogTimeGranularity = 1000;
+  config_line_append(&opts->Logs, "Log", "notice stdout");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(lt);
+  tt_assert(!msg);
+  options_rollback_log_transaction(lt);
+  expect_no_log_entry();
+  lt = NULL;
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+  // Now try some bad options.
+  or_options_free(opts);
+  mock_clean_saved_logs();
+  opts = mock_opts = options_new();
+  opts->LogTimeGranularity = 1000;
+  config_line_append(&opts->Logs, "Log", "warn blaznert");
+  lt = options_start_log_transaction(old_opts, &msg);
+  tt_assert(!lt);
+  tt_str_op(msg, OP_EQ, "Failed to init Log options. See logs for details.");
+  expect_single_log_msg_containing("Couldn't parse");
+  tt_int_op(get_min_log_level(), OP_EQ, LOG_DEBUG);
+
+ done:
+  UNMOCK(get_options);
+  or_options_free(opts);
+  or_options_free(old_opts);
+  tor_free(msg);
+  if (lt)
+    options_rollback_log_transaction(lt);
+  teardown_capture_of_logs();
+}
+
+#ifndef COCCI
+#define T(name) { #name, test_options_act_##name, TT_FORK, NULL, NULL }
+#endif
+
+struct testcase_t options_act_tests[] = {
+  T(create_dirs),
+  T(log_transition),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_parsecommon.c b/src/test/test_parsecommon.c
new file mode 100644
index 0000000000..9c22266da1
--- /dev/null
+++ b/src/test/test_parsecommon.c
@@ -0,0 +1,594 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/memarea/memarea.h"
+#include "lib/encoding/binascii.h"
+#include "feature/dirparse/parsecommon.h"
+#include "test/log_test_helpers.h"
+
+static void
+test_parsecommon_tokenize_string_null(void *arg)
+{
+
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  const char *str_with_null = "a\0bccccccccc";
+
+  int retval =
+  tokenize_string(area, str_with_null,
+                  str_with_null + 3,
+                  tokens, NULL, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_multiple_lines(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, GE(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          T1( "published", K_PUBLISHED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup(
+          "hibernating 0\nuptime 1024\n"
+          "published 2018-10-15 10:00:00\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(smartlist_len(tokens), OP_EQ, 3);
+  directory_token_t *token = smartlist_get(tokens, 0);
+
+  tt_int_op(token->tp, OP_EQ, K_HIBERNATING);
+
+  token = smartlist_get(tokens, 1);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+
+  token = smartlist_get(tokens, 2);
+
+  tt_int_op(token->tp, OP_EQ, K_PUBLISHED);
+
+  tt_int_op(retval, OP_EQ, 0);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_min_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(2), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // Missing "uptime"
+  char *str = tor_strdup("uptime 1024\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_max_cnt(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          T01("hibernating", K_HIBERNATING, GE(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "uptime" expected once, but occurs twice in input.
+  char *str = tor_strdup(
+                  "uptime 1024\nuptime 2048\nhibernating 0\n");
+
+  int retval =
+  tokenize_string(area, str, NULL,
+                  tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_start(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_START("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the first line.
+  char *str = tor_strdup(
+                  "uptime 1024\nclient-name Alice\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_at_end(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          T1_END("client-name", C_CLIENT_NAME, CONCAT_ARGS, NO_OBJ),
+          T01("uptime", K_UPTIME, EQ(1), NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  // "client-name" is not the last line.
+  char *str = tor_strdup(
+                  "client-name Alice\nuptime 1024\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_tokenize_string_no_annotations(void *arg)
+{
+  memarea_t *area = memarea_new();
+  smartlist_t *tokens = smartlist_new();
+
+  (void)arg;
+
+  token_rule_t table[] = {
+          A01("@last-listed", A_LAST_LISTED, CONCAT_ARGS, NO_OBJ),
+          END_OF_TABLE,
+  };
+
+  char *str = tor_strdup("@last-listed 2018-09-21 15:30:03\n");
+
+  int retval =
+  tokenize_string(area, str, NULL, tokens, table, 0);
+
+  tt_int_op(retval, OP_EQ, -1);
+
+ done:
+  tor_free(str);
+  memarea_drop_all(area);
+  smartlist_free(tokens);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_success(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, GE(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, K_UPTIME);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "1024");
+  tt_assert(!token->object_type);
+  tt_int_op(token->object_size, OP_EQ, 0);
+  tt_assert(!token->object_body);
+
+  tt_ptr_op(*s, OP_EQ, end);
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_concat_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "proto A=1 B=2";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T01("proto", K_PROTO, CONCAT_ARGS, NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_PROTO);
+  tt_int_op(token->n_args, OP_EQ, 1);
+  tt_str_op(*(token->args), OP_EQ, "A=1 B=2");
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_parse_keys(void *arg)
+{
+  (void)arg;
+
+  memarea_t *area = memarea_new();
+  const char *str =
+    "onion-key\n"
+    "-----BEGIN RSA PUBLIC KEY-----\n"
+    "MIGJAoGBAMDdIya33BfNlHOkzoTKSTT8EjD64waMfUr372syVHiFjHhObwKwGA5u\n"
+    "sHaMIe9r+Ij/4C1dKyuXkcz3DOl6gWNhTD7dZ89I+Okoh1jWe30jxCiAcywC22p5\n"
+    "XLhrDkX1A63Z7XCH9ltwU2WMqWsVM98N2GR6MTujP7wtqdLExYN1AgMBAAE=\n"
+    "-----END RSA PUBLIC KEY-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = (const char **)&str;
+  directory_token_t *token = NULL;
+  directory_token_t *token2 = NULL;
+
+  token_rule_t rule = T1("onion-key", R_IPO_ONION_KEY, NO_ARGS, NEED_KEY_1024);
+
+  token = get_next_token(area, s, end, &rule);
+  tt_assert(token);
+
+  tt_int_op(token->tp, OP_EQ, R_IPO_ONION_KEY);
+  tt_int_op(token->n_args, OP_EQ, 0);
+  tt_str_op(token->object_type, OP_EQ, "RSA PUBLIC KEY");
+  tt_int_op(token->object_size, OP_EQ, 140);
+  tt_assert(token->object_body);
+  tt_assert(token->key);
+  tt_assert(!token->error);
+
+  const char *str2 =
+    "client-key\n"
+    "-----BEGIN RSA PRIVATE KEY-----\n"
+    "MIICXAIBAAKBgQCwS810a2auH2PQchOBz9smNgjlDu31aq0IYlUohSYbhcv5AJ+d\n"
+    "DY0nfZWzS+mZPwzL3UiEnTt6PVv7AgoZ5V9ZJWJTKIURjJpkK0mstfJKHKIZhf84\n"
+    "pmFfRej9GQViB6NLtp1obOXJgJixSlMfw9doDI4NoAnEISCyH/tD77Qs2wIDAQAB\n"
+    "AoGAbDg8CKkdQOnX9c7xFpCnsE8fKqz9eddgHHNwXw1NFTwOt+2gDWKSMZmv2X5S\n"
+    "CVZg3owZxf5W0nT0D6Ny2+6nliak7foYAvkD0BsCiBhgftwC0zAo6k5rIbUKB3PJ\n"
+    "QLFXgpJhqWuXkODyt/hS/GTernR437WVSEGp1bnALqiFabECQQDaqHOxzoWY/nvH\n"
+    "KrfUi8EhqCnqERlRHwrW0MQZ1RPvF16OPPma+xa+ht/amfh3vYN5tZY82Zm43gGl\n"
+    "XWL5cZhNAkEAzmdSootYVnqLLLRMfHKXnO1XbaEcA/08MDNKGlSclBJixFenE8jX\n"
+    "iQsUbHwMJuGONvzWpRGPBP2f8xBd28ZtxwJARY+LZshtpfNniz/ixYJESaHG28je\n"
+    "xfjbKOW3TQSFV+2WTifFvHEeljQwKMoMyoMGvYRwLCGJjs9JtMLVxsdFjQJBAKwD\n"
+    "3BBvBQ39TuPQ1zWX4tb7zjMlY83HTFP3Sriq71tP/1QWoL2SUl56B2lp8E6vB/C3\n"
+    "wsMK4SCNprHRYAd7VZ0CQDKn6Zhd11P94PLs0msybFEh1VXr6CEW/BrxBgbL4ls6\n"
+    "dbX5XO0z4Ra8gYXgObgimhyMDYO98Idt5+Z3HIdyrSc=\n"
+    "-----END RSA PRIVATE KEY-----\n";
+
+  const char *end2 = str2 + strlen(str2);
+  const char **s2 = (const char **)&str2;
+
+  token_rule_t rule2 = T01("client-key", C_CLIENT_KEY, NO_ARGS,
+                           NEED_SKEY_1024);
+
+  token2 = get_next_token(area, s2, end2, &rule2);
+  tt_assert(token2);
+
+  tt_int_op(token2->tp, OP_EQ, C_CLIENT_KEY);
+  tt_int_op(token2->n_args, OP_EQ, 0);
+  tt_str_op(token2->object_type, OP_EQ, "RSA PRIVATE KEY");
+  tt_int_op(token2->object_size, OP_EQ, 608);
+  tt_assert(token2->object_body);
+  tt_assert(token2->key);
+  tt_assert(!token->error);
+
+ done:
+  if (token) token_clear(token);
+  if (token2) token_clear(token2);
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_object(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, K_DIRECTORY_SIGNATURE);
+  tt_int_op(token->n_args, OP_EQ, 2);
+  tt_str_op(token->args[0], OP_EQ,
+                  "0232AF901C31A04EE9848595AF9BB7620D4C5B2E");
+  tt_str_op(token->args[1], OP_EQ,
+                  "CD1FD971855430880D3C31E0331C5C55800C2F79");
+
+  tt_assert(!token->error);
+
+  char decoded[256];
+  const char *signature =
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+  tt_assert(signature);
+  size_t signature_len = strlen(signature);
+  base64_decode(decoded, sizeof(decoded), signature, signature_len);
+
+  tt_str_op(token->object_type, OP_EQ, "SIGNATURE");
+  tt_int_op(token->object_size, OP_EQ, 256);
+  tt_mem_op(token->object_body, OP_EQ, decoded, 256);
+
+  tt_assert(!token->key);
+
+ done:
+  memarea_drop_all(area);
+}
+
+static void
+test_parsecommon_get_next_token_err_too_many_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime 1024 1024 1024";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too many arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_too_few_args(void *arg)
+{
+  memarea_t *area = memarea_new();
+  const char *str = "uptime";
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t table = T01("uptime", K_UPTIME, EQ(1), NO_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &table);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Too few arguments to uptime");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_obj_missing_endline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: missing object end line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_beginline(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-Z---\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad begin line");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_tag_mismatch(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "dLTbc1Lad/OWKBJhA/dERzDHumswTAzBFAWAz2vnQhLsebs1SOm0W/vceEsiEkiF\n"
+    "A+JJSzIyfywJc6Mnk7aKMEIFjOO/MaxuAp4zv+q+JonJkF0ExjMqvKR0D6pSFmfN\n"
+    "cnemnxGHxNuPDnKl0imbWKmWDsHtwgi4zWeTq3MekfMOXKi6gIh+bDFzCs9/Vquh\n"
+    "uNKJI1jW/A2DEKeaSAODEv9VoCsYSvbVVEuHCBWjeNAurd5aL26BrAolW6m7pkD6\n"
+    "I+cQ8dQG6Wa/Zt6gLXtBbOP2o/iDI7ahDP9diNkBI/rm4nfp9j4piTwsqpi7xz9J\n"
+    "Ua9DEZB9KbJHVX1rGShrLA==\n"
+    "-----END SOMETHINGELSE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ,
+            "Malformed object: mismatched end tag SIGNATURE");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+static void
+test_parsecommon_get_next_token_err_bad_base64(void *arg)
+{
+  memarea_t *area = memarea_new();
+
+  const char *str =
+    "directory-signature 0232AF901C31A04EE9848595AF9BB7620D4C5B2E "
+    "CD1FD971855430880D3C31E0331C5C55800C2F79\n"
+    "-----BEGIN SIGNATURE-----\n"
+    "%%@%%%%%%%!!!'\n"
+    "-----END SIGNATURE-----\n";
+
+  const char *end = str + strlen(str);
+  const char **s = &str;
+  token_rule_t rule = T("directory-signature", K_DIRECTORY_SIGNATURE,
+                        GE(2), NEED_OBJ);
+  (void)arg;
+
+  directory_token_t *token = get_next_token(area, s, end, &rule);
+
+  tt_int_op(token->tp, OP_EQ, ERR_);
+  tt_str_op(token->error, OP_EQ, "Malformed object: bad base64-encoded data");
+
+ done:
+  memarea_drop_all(area);
+  return;
+}
+
+#define PARSECOMMON_TEST(name) \
+  { #name, test_parsecommon_ ## name, 0, NULL, NULL }
+
+struct testcase_t parsecommon_tests[] = {
+  PARSECOMMON_TEST(tokenize_string_null),
+  PARSECOMMON_TEST(tokenize_string_multiple_lines),
+  PARSECOMMON_TEST(tokenize_string_min_cnt),
+  PARSECOMMON_TEST(tokenize_string_max_cnt),
+  PARSECOMMON_TEST(tokenize_string_at_start),
+  PARSECOMMON_TEST(tokenize_string_at_end),
+  PARSECOMMON_TEST(tokenize_string_no_annotations),
+  PARSECOMMON_TEST(get_next_token_success),
+  PARSECOMMON_TEST(get_next_token_concat_args),
+  PARSECOMMON_TEST(get_next_token_parse_keys),
+  PARSECOMMON_TEST(get_next_token_object),
+  PARSECOMMON_TEST(get_next_token_err_too_many_args),
+  PARSECOMMON_TEST(get_next_token_err_too_few_args),
+  PARSECOMMON_TEST(get_next_token_err_obj_missing_endline),
+  PARSECOMMON_TEST(get_next_token_err_bad_beginline),
+  PARSECOMMON_TEST(get_next_token_err_tag_mismatch),
+  PARSECOMMON_TEST(get_next_token_err_bad_base64),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_parseconf.sh b/src/test/test_parseconf.sh
new file mode 100755
index 0000000000..4fe27d9f5d
--- /dev/null
+++ b/src/test/test_parseconf.sh
@@ -0,0 +1,655 @@
+#!/bin/sh
+# Copyright 2019, The Tor Project, Inc.
+# See LICENSE for licensing information
+
+# Integration test script for verifying that Tor configurations are parsed as
+# we expect.
+#
+# Valid configurations are tested with --dump-config, which parses and
+# validates the configuration before writing it out.  We then make sure that
+# the result is what we expect, before parsing and dumping it again to make
+# sure that there is no change. Optionally, we can also test the log messages
+# with --verify-config.
+#
+# Invalid configurations are tested with --verify-config, which parses
+# and validates the configuration. We capture its output and make sure that
+# it contains the error message we expect.
+#
+# When tor is compiled with different libraries or modules, some
+# configurations may have different results. We can specify these result
+# variants using additional result files.
+
+# This script looks for its test cases as individual directories in
+# src/test/conf_examples/.  Each test may have these files:
+#
+# Configuration Files
+#
+# torrc -- Usually needed. This file is passed to Tor on the command line
+#      with the "-f" flag. (If you omit it, you'll test Tor's behavior when
+#      it receives a nonexistent configuration file.)
+#
+# torrc.defaults -- Optional. If present, it is passed to Tor on the command
+#      line with the --defaults-torrc option. If this file is absent, an empty
+#      file is passed instead to prevent Tor from reading the system defaults.
+#
+# cmdline -- Optional. If present, it contains command-line arguments that
+#      will be passed to Tor.
+#
+# (included torrc files or directories) -- Optional. Additional files can be
+#      included in configuration, using the "%include" directive. Files or
+#      directories can be included in any of the config files listed above.
+#      Include paths should be specified relative to the test case directory.
+#
+# Result Files
+#
+# expected -- If this file is present, then it should be the expected result
+#      of "--dump-config short" for this test case.  Exactly one of
+#      "expected" or "error" must be present, or the test will fail.
+#
+# expected_log -- Optional. If this file is present, then it contains a regex
+#      that must be matched by some line in the output of "--verify-config",
+#      which must succeed. Only used if "expected" is also present.
+#
+# error -- If this file is present, then it contains a regex that must be
+#      matched by some line in the output of "--verify-config", which must
+#      fail. Exactly one of "expected" or "error" must be present, or the
+#      test will fail.
+#
+# {expected,expected_log,error}_${TOR_LIBS_ENABLED}* -- If this file is
+#      present, then the outcome is different when some optional libraries are
+#      enabled. If there is no result file matching the exact list of enabled
+#      libraries, the script searches for result files with one or more of
+#      those libraries disabled. The search terminates at the standard result
+#      file. If expected* is present, the script also searches for
+#      expected_log*.
+#
+#      For example:
+#      A test that succeeds, regardless of any enabled libraries:
+#       - expected
+#      A test that has a different result if the nss library is enabled
+#      (but the same result if any other library is enabled). We also check
+#      the log output in this test:
+#       - expected
+#       - expected_log
+#       - expected_nss
+#       - expected_log_nss
+#      A test that fails if the lzma and zstd modules are *not* enabled:
+#       - error
+#       - expected_lzma_zstd
+#
+# {expected,expected_log,error}*_no_${TOR_MODULES_DISABLED} -- If this file is
+#      present, then the outcome is different when some modules are disabled.
+#      If there is no result file matching the exact list of disabled modules,
+#      the standard result file is used. If expected* is present, the script
+#      also searches for expected_log*.
+#
+#      For example:
+#      A test that succeeds, regardless of any disabled modules:
+#       - expected
+#      A test that has a different result if the relay module is disabled
+#      (but the same result if just the dirauth module is disabled):
+#       - expected
+#       - expected_no_relay_dirauth
+#      A test that fails if the dirauth module is disabled:
+#       - expected
+#       - error_no_dirauth
+#       - error_no_relay_dirauth
+#      (Disabling the relay module also disables dirauth module. But we don't
+#      want to encode that knowledge in this test script, so we supply a
+#      separate result file for every combination of disabled modules that
+#      has a different result.)
+
+umask 077
+set -e
+
+MYNAME="$0"
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if test -d "$f"; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if test $# -ge 1; then
+  TOR_BINARY="$1"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "Using Tor binary '$TOR_BINARY'."
+
+# make a safe space for temporary files
+DATA_DIR=$(mktemp -d -t tor_parseconf_tests.XXXXXX)
+trap 'rm -rf "$DATA_DIR"' 0
+
+# This is where we look for examples
+EXAMPLEDIR="$(dirname "$0")"/conf_examples
+
+case "$(uname -s)" in
+    CYGWIN*) WINDOWS=1;;
+    MINGW*) WINDOWS=1;;
+    MSYS*) WINDOWS=1;;
+    *) WINDOWS=0;;
+esac
+
+####
+# BUG WORKAROUND FOR 31757:
+#  On Appveyor, it seems that Tor sometimes randomly fails to produce
+#  output with --dump-config.  Whil we are figuring this out, do not treat
+#  windows errors as hard failures.
+####
+if test "$WINDOWS" = 1; then
+    EXITCODE=0
+else
+    EXITCODE=1
+fi
+
+FINAL_EXIT=0
+NEXT_TEST=
+
+# Log a failure message to stderr, using $@ as a printf string and arguments
+# Set NEXT_TEST to "yes" and FINAL_EXIT to $EXITCODE.
+fail_printf()
+{
+    printf "FAIL: " >&2
+    # The first argument is a printf string, so this warning is spurious
+    # shellcheck disable=SC2059
+    printf "$@" >&2
+    NEXT_TEST="yes"
+    FINAL_EXIT=$EXITCODE
+}
+
+# Log a failure message to stderr, using $@ as a printf string and arguments
+# Exit with status $EXITCODE.
+die_printf()
+{
+    printf "FAIL: CRITICAL error in '%s':" "$MYNAME" >&2
+    # The first argument is a printf string, so this warning is spurious
+    # shellcheck disable=SC2059
+    printf "$@" >&2
+    exit $EXITCODE
+}
+
+if test "$WINDOWS" = 1; then
+    FILTER="dos2unix"
+else
+    FILTER="cat"
+fi
+
+EMPTY="${DATA_DIR}/EMPTY"
+touch "$EMPTY" || die_printf "Couldn't create empty file '%s'.\\n" \
+                             "$EMPTY"
+NON_EMPTY="${DATA_DIR}/NON_EMPTY"
+echo "This pattern should not match any log messages" \
+     > "$NON_EMPTY" || die_printf "Couldn't create non-empty file '%s'.\\n" \
+                                  "$NON_EMPTY"
+
+STANDARD_LIBS="libevent\\|openssl\\|zlib"
+# Lib names are restricted to [a-z0-9]* at the moment
+# We don't actually want to support foreign accents here
+# shellcheck disable=SC2018,SC2019
+TOR_LIBS_ENABLED="$("$TOR_BINARY" --verify-config \
+                      -f "$EMPTY" --defaults-torrc "$EMPTY" \
+                    | sed -n 's/.* Tor .* running on .* with\(.*\)\./\1/p' \
+                    | tr 'A-Z' 'a-z' | tr ',' '\n' \
+                    | grep -v "$STANDARD_LIBS" | grep -v "n/a" \
+                    | sed 's/\( and\)* \(lib\)*\([a-z0-9]*\) .*/\3/' \
+                    | sort | tr '\n' '_')"
+# Remove the last underscore, if there is one
+TOR_LIBS_ENABLED=${TOR_LIBS_ENABLED%_}
+
+# If we ever have more than 3 optional libraries, we'll need more code here
+TOR_LIBS_ENABLED_COUNT="$(echo "$TOR_LIBS_ENABLED_SEARCH" \
+                          | tr ' ' '\n' | wc -l)"
+if test "$TOR_LIBS_ENABLED_COUNT" -gt 3; then
+    die_printf "Can not handle more than 3 optional libraries.\\n"
+fi
+# Brute-force the combinations of libraries
+TOR_LIBS_ENABLED_SEARCH_3="$(echo "$TOR_LIBS_ENABLED" \
+    | sed -n \
+      's/^\([^_]*\)_\([^_]*\)_\([^_]*\)$/_\1_\2 _\1_\3 _\2_\3 _\1 _\2 _\3/p')"
+TOR_LIBS_ENABLED_SEARCH_2="$(echo "$TOR_LIBS_ENABLED" \
+    | sed -n 's/^\([^_]*\)_\([^_]*\)$/_\1 _\2/p')"
+TOR_LIBS_ENABLED_SEARCH="_$TOR_LIBS_ENABLED \
+                           $TOR_LIBS_ENABLED_SEARCH_3 \
+                           $TOR_LIBS_ENABLED_SEARCH_2"
+TOR_LIBS_ENABLED_SEARCH="$(echo "$TOR_LIBS_ENABLED_SEARCH" | tr ' ' '\n' \
+                           | grep -v '^_*$' | tr '\n' ' ')"
+
+TOR_MODULES_DISABLED="$("$TOR_BINARY" --list-modules | grep ': no' \
+                        | cut -d ':' -f1 | sort | tr '\n' '_')"
+# Remove the last underscore, if there is one
+TOR_MODULES_DISABLED=${TOR_MODULES_DISABLED%_}
+
+echo "Tor is configured with:"
+echo "Optional Libraries: ${TOR_LIBS_ENABLED:-(None)}"
+if test "$TOR_LIBS_ENABLED"; then
+    echo "Optional Library Search List: $TOR_LIBS_ENABLED_SEARCH"
+fi
+echo "Disabled Modules: ${TOR_MODULES_DISABLED:-(None)}"
+
+# Yes, unix uses "0" for a successful command
+TRUE=0
+FALSE=1
+
+# Run tor --verify-config on the torrc $1, and defaults torrc $2, which may
+# be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted.
+# Send tor's standard output to stderr.
+log_verify_config()
+{
+    # show the command we're about to execute
+    # log_verify_config() is only called when we've failed
+    printf "Tor --verify-config said:\\n" >&2
+    printf "$ %s %s %s %s %s %s %s\\n" \
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  "$3" \
+                  >&2
+    # We need cmdline unquoted
+    # shellcheck disable=SC2086
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  $3 \
+                  >&2 \
+        || true
+}
+
+# Run "tor --dump-config short" on the torrc $1, and defaults torrc $2, which
+# may be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4.
+#
+# Set $FULL_TOR_CMD to the tor command line that was executed.
+#
+# If tor fails, fail_printf() using the file name $5, and context $6,
+# which may be an empty string. Then run log_verify_config().
+dump_config()
+{
+    if test "$6"; then
+        CONTEXT=" $6"
+    else
+        CONTEXT=""
+    fi
+
+    # keep the command we're about to execute, and show if it we fail
+    FULL_TOR_CMD=$(printf "$ %s %s %s %s %s %s %s %s" \
+         "$TOR_BINARY" --dump-config short \
+                       -f "$1" \
+                       --defaults-torrc "$2" \
+                       "$3"
+             )
+    # We need cmdline unquoted
+    # shellcheck disable=SC2086
+    if ! "$TOR_BINARY" --dump-config short \
+                       -f "$1" \
+                       --defaults-torrc "$2" \
+                       $3 \
+                       > "$4"; then
+        fail_printf "'%s': Tor --dump-config reported an error%s:\\n%s\\n" \
+                    "$5" \
+                    "$CONTEXT" \
+                    "$FULL_TOR_CMD"
+        log_verify_config "$1" \
+                          "$2" \
+                          "$3"
+    fi
+}
+
+# Run "$FILTER" on the input $1.
+# Send the standard output to $2.
+# If tor fails, log a failure message using the file name $3, and context $4,
+# which may be an empty string.
+filter()
+{
+    if test "$4"; then
+        CONTEXT=" $4"
+    else
+        CONTEXT=""
+    fi
+
+    "$FILTER" "$1" \
+              > "$2" \
+        || fail_printf "'%s': Filter '%s' reported an error%s.\\n" \
+                       "$3" \
+                       "$FILTER" \
+                       "$CONTEXT"
+}
+
+# Compare the expected file $1, and output file $2.
+#
+# If they are different, fail. Log the differences between the files.
+# Run log_verify_config() with torrc $3, defaults torrc $4, and command
+# line $5, to log Tor's error messages.
+#
+# If the file contents are identical, returns true. Otherwise, return false.
+#
+# Log failure messages using fail_printf(), with the expected file name,
+# context $6, which may be an empty string, and the tor command line $7.
+check_diff()
+{
+    if test "$6"; then
+        CONTEXT=" $6"
+    else
+        CONTEXT=""
+    fi
+
+    if cmp "$1" "$2" > /dev/null; then
+        return "$TRUE"
+    else
+        fail_printf "'%s': Tor --dump-config said%s:\\n%s\\n" \
+                    "$1" \
+                    "$CONTEXT" \
+                    "$7"
+        diff -u "$1" "$2" >&2 \
+            || true
+        log_verify_config "$3" \
+                          "$4" \
+                          "$5"
+        return "$FALSE"
+    fi
+}
+
+# Run "tor --dump-config short" on the torrc $1, and defaults torrc $2, which
+# may be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4, after running $FILTER
+# on it.
+#
+# If tor fails, run log_verify_config().
+#
+# Compare the expected file $5, and output file. If they are different, fail.
+# If this is the first step that failed in this test, run log_verify_config().
+#
+# If the file contents are identical, returns true. Otherwise, return false,
+# and log the differences between the files.
+#
+# Log failure messages using fail_printf(), with the expected file name, and
+# context $6, which may be an empty string.
+check_dump_config()
+{
+    OUTPUT="$4"
+    OUTPUT_RAW="${OUTPUT}_raw"
+
+    FULL_TOR_CMD=
+    dump_config "$1" \
+                "$2" \
+                "$3" \
+                "$OUTPUT_RAW" \
+                "$5" \
+                "$6"
+
+    filter "$OUTPUT_RAW" \
+           "$OUTPUT" \
+           "$5" \
+           "$6"
+
+    if check_diff "$5" \
+                  "$OUTPUT" \
+                  "$1" \
+                  "$2" \
+                  "$3" \
+                  "$6" \
+                  "$FULL_TOR_CMD"; then
+        return "$TRUE"
+    else
+        return "$FALSE"
+    fi
+}
+
+# Check if $1 is an empty file.
+# If it is, fail_printf() using $2 as the type of file.
+# Returns true if the file is empty, false otherwise.
+check_empty_pattern()
+{
+    if ! test -s "$1"; then
+        fail_printf "%s file '%s' is empty, and will match any output.\\n" \
+                    "$2" \
+                    "$1"
+        return "$TRUE"
+    else
+        return "$FALSE"
+    fi
+}
+
+# Run tor --verify-config on the torrc $1, and defaults torrc $2, which may
+# be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4.
+#
+# Set $FULL_TOR_CMD to the tor command line that was executed.
+#
+# If tor's exit status does not match the boolean $5, fail_printf()
+# using the file name $6, and context $7, which is required.
+verify_config()
+{
+    RESULT=$TRUE
+
+    # keep the command we're about to execute, and show if it we fail
+    FULL_TOR_CMD=$(printf "$ %s %s %s %s %s %s %s" \
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  "$3"
+             )
+    # We need cmdline unquoted
+    # shellcheck disable=SC2086
+    "$TOR_BINARY" --verify-config \
+                  -f "$1" \
+                  --defaults-torrc "$2" \
+                  $3 \
+                  > "$4" || RESULT=$FALSE
+
+    # Convert the actual and expected results to boolean, and compare
+    if test $((! (! RESULT))) -ne $((! (! $5))); then
+        fail_printf "'%s': Tor --verify-config did not %s:\\n%s\\n" \
+                    "$6" \
+                    "$7" \
+                    "$FULL_TOR_CMD"
+        cat "$4" >&2
+    fi
+}
+
+# Check for the patterns in the match file $1, in the output file $2.
+# Uses grep with the entire contents of the match file as the pattern.
+# (Not "grep -f".)
+#
+# If the pattern does not match any lines in the output file, fail.
+# Log the pattern, and the entire contents of the output file.
+#
+# Log failure messages using fail_printf(), with the match file name,
+# context $3, and tor command line $4, which are required.
+check_pattern()
+{
+    expect_log="$(cat "$1")"
+    if ! grep "$expect_log" "$2" > /dev/null; then
+        fail_printf "Expected %s '%s':\\n%s\\n" \
+                    "$3" \
+                    "$1" \
+                    "$expect_log"
+        printf "Tor --verify-config said:\\n%s\\n" \
+               "$4" >&2
+        cat "$2" >&2
+    fi
+}
+
+# Run tor --verify-config on the torrc $1, and defaults torrc $2, which may
+# be $EMPTY. Pass tor the extra command line arguments $3, which will be
+# passed unquoted. Send tor's standard output to $4.
+#
+# If tor's exit status does not match the boolean $5, fail.
+#
+# Check for the patterns in the match file $6, in the output file.
+# Uses grep with the entire contents of the match file as the pattern.
+# (Not "grep -f".) The match file must not be empty.
+#
+# If the pattern does not match any lines in the output file, fail.
+# Log the pattern, and the entire contents of the output file.
+#
+# Log failure messages using fail_printf(), with the match file name,
+# and context $7, which is required.
+check_verify_config()
+{
+    if check_empty_pattern "$6" "$7"; then
+        return
+    fi
+
+    FULL_TOR_CMD=
+    verify_config "$1" \
+                  "$2" \
+                  "$3" \
+                  "$4" \
+                  "$5" \
+                  "$6" \
+                  "$7"
+
+    check_pattern "$6" \
+                  "$4" \
+                  "$7" \
+                  "$FULL_TOR_CMD"
+}
+
+for dir in "${EXAMPLEDIR}"/*; do
+    NEXT_TEST=
+
+    if ! test -d "$dir"; then
+       # Only count directories.
+       continue
+    fi
+
+    testname="$(basename "${dir}")"
+    # We use printf since "echo -n" is not standard
+    printf "%s: " \
+           "$testname"
+
+    PREV_DIR="$(pwd)"
+    cd "$dir"
+
+    if test -f "./torrc.defaults"; then
+        DEFAULTS="./torrc.defaults"
+    else
+        DEFAULTS="${DATA_DIR}/EMPTY"
+    fi
+
+    if test -f "./cmdline"; then
+        CMDLINE="$(cat ./cmdline)"
+    else
+        CMDLINE=""
+    fi
+
+    EXPECTED=
+    EXPECTED_LOG=
+    ERROR=
+    # Search for a custom result file for any combination of enabled optional
+    # libraries
+    # The libs in the list are [A-Za-z0-9_]* and space-separated.
+    # shellcheck disable=SC2086
+    for lib_suffix in $TOR_LIBS_ENABLED_SEARCH ""; do
+        # Search for a custom result file for any disabled modules
+        for mod_suffix in "_no_${TOR_MODULES_DISABLED}" ""; do
+            suffix="${lib_suffix}${mod_suffix}"
+
+            if test -f "./expected${suffix}"; then
+
+                # Check for broken configs
+                if test -f "./error${suffix}"; then
+                    fail_printf "Found both '%s' and '%s'.%s\\n" \
+                                "${dir}/expected${suffix}" \
+                                "${dir}/error${suffix}" \
+                                "(Only one of these files should exist.)"
+                    break
+                fi
+
+                EXPECTED="./expected${suffix}"
+                if test -f "./expected_log${suffix}"; then
+                    EXPECTED_LOG="./expected_log${suffix}"
+                fi
+                break
+
+            elif test -f "./error${suffix}"; then
+                ERROR="./error${suffix}"
+                break
+            fi
+        done
+
+        # Exit as soon as the inner loop finds a file, or fails
+        if test -f "$EXPECTED" || test -f "$ERROR" || test "$NEXT_TEST"; then
+            break
+        fi
+    done
+
+    if test "$NEXT_TEST"; then
+        # The test failed inside the file search loop: go to the next test
+        continue
+    elif test -f "$EXPECTED"; then
+        # This case should succeed: run dump-config and see if it does.
+
+        if check_dump_config "./torrc" \
+                             "$DEFAULTS" \
+                             "$CMDLINE" \
+                             "${DATA_DIR}/output.${testname}" \
+                             "$EXPECTED" \
+                             ""; then
+            # Check round-trip.
+            check_dump_config "${DATA_DIR}/output.${testname}" \
+                              "$EMPTY" \
+                              "" \
+                              "${DATA_DIR}/output_2.${testname}" \
+                              "$EXPECTED" \
+                              "on round-trip" || true
+        fi
+
+        if test -f "$EXPECTED_LOG"; then
+            # This case should succeed: run verify-config and see if it does.
+
+            check_verify_config "./torrc" \
+                                "$DEFAULTS" \
+                                "$CMDLINE" \
+                                "${DATA_DIR}/output_log.${testname}" \
+                                "$TRUE" \
+                                "$EXPECTED_LOG" \
+                                "log success"
+        else
+            printf "\\nNOTICE: Missing '%s_log' file:\\n" \
+                   "$EXPECTED" >&2
+            log_verify_config "./torrc" \
+                              "$DEFAULTS" \
+                              "$CMDLINE"
+        fi
+
+   elif test -f "$ERROR"; then
+        # This case should fail: run verify-config and see if it does.
+
+        check_verify_config "./torrc" \
+                            "$DEFAULTS" \
+                            "$CMDLINE" \
+                            "${DATA_DIR}/output.${testname}" \
+                            "$FALSE" \
+                            "$ERROR" \
+                            "log error"
+    else
+        # This case is not actually configured with a success or a failure.
+        # call that an error.
+        fail_printf "Did not find ${dir}/*expected or ${dir}/*error.\\n"
+    fi
+
+    if test -z "$NEXT_TEST"; then
+        echo "OK"
+    fi
+
+    cd "$PREV_DIR"
+
+done
+
+exit "$FINAL_EXIT"
diff --git a/src/test/test_pem.c b/src/test/test_pem.c
index 9eb99181e2..9772be124b 100644
--- a/src/test/test_pem.c
+++ b/src/test/test_pem.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
diff --git a/src/test/test_periodic_event.c b/src/test/test_periodic_event.c
index 4a53639dad..b7f1785805 100644
--- a/src/test/test_periodic_event.c
+++ b/src/test/test_periodic_event.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -19,6 +19,7 @@
 #include "feature/hibernate/hibernate.h"
 #include "feature/hs/hs_service.h"
 #include "core/mainloop/mainloop.h"
+#include "core/mainloop/netstatus.h"
 #include "core/mainloop/periodic.h"
 
 /** Helper function: This is replaced in some tests for the event callbacks so
@@ -50,16 +51,21 @@ test_pe_initialize(void *arg)
    * need to run the main loop and then wait for a second delaying the unit
    * tests. Instead, we'll test the callback work indepedently elsewhere. */
   initialize_periodic_events();
+  periodic_events_connect_all();
+  set_network_participation(false);
+  rescan_periodic_events(get_options());
 
   /* Validate that all events have been set up. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     tt_assert(item->ev);
     tt_assert(item->fn);
     tt_u64_op(item->last_action_time, OP_EQ, 0);
     /* Every event must have role(s) assign to it. This is done statically. */
     tt_u64_op(item->roles, OP_NE, 0);
-    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, 0);
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_uint_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
  done:
@@ -79,17 +85,20 @@ test_pe_launch(void *arg)
    * network gets enabled. */
   consider_hibernation(time(NULL));
 
+  set_network_participation(true);
+
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
   options = get_options_mutable();
   options->SocksPort_set = 1;
   periodic_events_on_new_options(options);
+
 #if 0
   /* Lets make sure that before intialization, we can't scan the periodic
    * events list and launch them. Lets try by being a Client. */
@@ -99,20 +108,20 @@ test_pe_launch(void *arg)
     periodic_event_item_t *item = &periodic_events[i];
     tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
   }
-#endif
+#endif /* 0 */
 
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Now that we've initialized, rescan the list to launch. */
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    if (item->roles & PERIODIC_EVENT_ROLE_CLIENT) {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
-    } else {
-      tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
-    }
+  int mask = PERIODIC_EVENT_ROLE_CLIENT|PERIODIC_EVENT_ROLE_ALL|
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = !!(item->roles & mask);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
     // enabled or not, the event has not yet been run.
     tt_u64_op(item->last_action_time, OP_EQ, 0);
   }
@@ -124,10 +133,11 @@ test_pe_launch(void *arg)
 
   unsigned roles = get_my_roles(options);
   tt_uint_op(roles, OP_EQ,
-             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER);
+             PERIODIC_EVENT_ROLE_RELAY|PERIODIC_EVENT_ROLE_DIRSERVER|
+             PERIODIC_EVENT_ROLE_ALL|PERIODIC_EVENT_ROLE_NET_PARTICIPANT);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     /* Only Client role should be disabled. */
     if (item->roles == PERIODIC_EVENT_ROLE_CLIENT) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
@@ -144,17 +154,23 @@ test_pe_launch(void *arg)
   /* Disable everything and we'll enable them ALL. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
+  options->DisableNetwork = 1;
+  set_network_participation(false);
   periodic_events_on_new_options(options);
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    int should_be_enabled = (item->roles & PERIODIC_EVENT_ROLE_ALL) &&
+      !(item->flags & PERIODIC_EVENT_FLAG_NEED_NET);
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ, should_be_enabled);
   }
 
   /* Enable everything. */
   options->SocksPort_set = 1; options->ORPort_set = 1;
   options->BridgeRelay = 1; options->AuthoritativeDir = 1;
   options->V3AuthoritativeDir = 1; options->BridgeAuthoritativeDir = 1;
+  options->DisableNetwork = 0;
+  set_network_participation(true);
   register_dummy_hidden_service(&service);
   periodic_events_on_new_options(options);
   /* Note down the reference because we need to remove this service from the
@@ -163,9 +179,10 @@ test_pe_launch(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
-    tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
+    tt_int_op(periodic_event_is_enabled(item), OP_EQ,
+              (item->roles != PERIODIC_EVENT_ROLE_CONTROLEV));
   }
 
  done:
@@ -187,42 +204,49 @@ test_pe_get_roles(void *arg)
 
   or_options_t *options = get_options_mutable();
   tt_assert(options);
+  set_network_participation(true);
+
+  const int ALL = PERIODIC_EVENT_ROLE_ALL |
+    PERIODIC_EVENT_ROLE_NET_PARTICIPANT;
 
   /* Nothing configured, should be no roles. */
+  tt_assert(net_is_disabled());
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Indicate we have a SocksPort, roles should be come Client. */
   options->SocksPort_set = 1;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT);
+  tt_int_op(roles, OP_EQ, PERIODIC_EVENT_ROLE_CLIENT|ALL);
 
   /* Now, we'll add a ORPort so should now be a Relay + Client. */
   options->ORPort_set = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_DIRSERVER | ALL));
 
   /* Now add a Bridge. */
   options->BridgeRelay = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_CLIENT | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_BRIDGE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_ROUTER);
   /* Unset client so we can solely test Router role. */
   options->SocksPort_set = 0;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_ROUTER | PERIODIC_EVENT_ROLE_DIRSERVER |
+            ALL);
 
   /* Reset options so we can test authorities. */
   options->SocksPort_set = 0;
   options->ORPort_set = 0;
   options->BridgeRelay = 0;
   roles = get_my_roles(options);
-  tt_int_op(roles, OP_EQ, 0);
+  tt_int_op(roles, OP_EQ, ALL);
 
   /* Now upgrade to Dirauth. */
   options->DirPort_set = 1;
@@ -230,7 +254,7 @@ test_pe_get_roles(void *arg)
   options->V3AuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_DIRAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Now Bridge Authority. */
@@ -238,7 +262,7 @@ test_pe_get_roles(void *arg)
   options->BridgeAuthoritativeDir = 1;
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
-            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER);
+            PERIODIC_EVENT_ROLE_BRIDGEAUTH|PERIODIC_EVENT_ROLE_DIRSERVER|ALL);
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* Move that bridge auth to become a relay. */
@@ -246,7 +270,7 @@ test_pe_get_roles(void *arg)
   roles = get_my_roles(options);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY
-             | PERIODIC_EVENT_ROLE_DIRSERVER));
+             | PERIODIC_EVENT_ROLE_DIRSERVER|ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
   /* And now an Hidden service. */
@@ -257,7 +281,8 @@ test_pe_get_roles(void *arg)
   remove_service(get_hs_service_map(), &service);
   tt_int_op(roles, OP_EQ,
             (PERIODIC_EVENT_ROLE_BRIDGEAUTH | PERIODIC_EVENT_ROLE_RELAY |
-             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER));
+             PERIODIC_EVENT_ROLE_HS_SERVICE | PERIODIC_EVENT_ROLE_DIRSERVER |
+             ALL));
   tt_assert(roles & PERIODIC_EVENT_ROLE_AUTHORITIES);
 
  done:
@@ -277,12 +302,13 @@ test_pe_hs_service(void *arg)
   consider_hibernation(time(NULL));
   /* Initialize the events so we can enable them */
   initialize_periodic_events();
+  periodic_events_connect_all();
 
   /* Hack: We'll set a dumb fn() of each events so they don't get called when
    * dispatching them. We just want to test the state of the callbacks, not
    * the whole code path. */
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     item->fn = dumb_event_fn;
   }
 
@@ -295,8 +321,8 @@ test_pe_hs_service(void *arg)
    * trigger a rescan of the event disabling the HS service event. */
   to_remove = &service;
 
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 1);
     }
@@ -306,8 +332,8 @@ test_pe_hs_service(void *arg)
   /* Remove the service from the global map, it should trigger a rescan and
    * disable the HS service events. */
   remove_service(get_hs_service_map(), &service);
-  for (int i = 0; periodic_events[i].name; ++i) {
-    periodic_event_item_t *item = &periodic_events[i];
+  for (int i = 0; mainloop_periodic_events[i].name; ++i) {
+    periodic_event_item_t *item = &mainloop_periodic_events[i];
     if (item->roles & PERIODIC_EVENT_ROLE_HS_SERVICE) {
       tt_int_op(periodic_event_is_enabled(item), OP_EQ, 0);
     }
diff --git a/src/test/test_policy.c b/src/test/test_policy.c
index 9c001c294a..7949e90e9e 100644
--- a/src/test/test_policy.c
+++ b/src/test/test_policy.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2013-2019, The Tor Project, Inc. */
+/* Copyright (c) 2013-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CONFIG_PRIVATE
@@ -6,13 +6,18 @@
 
 #include "core/or/or.h"
 #include "app/config/config.h"
+#include "core/or/circuitbuild.h"
 #include "core/or/policies.h"
 #include "feature/dirparse/policy_parse.h"
+#include "feature/hs/hs_common.h"
+#include "feature/hs/hs_descriptor.h"
 #include "feature/relay/router.h"
 #include "lib/encoding/confline.h"
 #include "test/test.h"
+#include "test/log_test_helpers.h"
 
 #include "core/or/addr_policy_st.h"
+#include "core/or/extend_info_st.h"
 #include "core/or/port_cfg_st.h"
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerinfo_st.h"
@@ -57,8 +62,8 @@ test_policy_summary_helper_family_flags(const char *policy_str,
   short_policy_t *short_policy = NULL;
   int success = 0;
 
-  line.key = (char*)"foo";
-  line.value = (char *)policy_str;
+  line.key = (char *) "foo";
+  line.value = (char *) policy_str;
   line.next = NULL;
 
   r = policies_parse_exit_policy(&line, &policy,
@@ -2024,6 +2029,101 @@ test_policies_fascist_firewall_allows_address(void *arg)
                            expect_ap); \
   STMT_END
 
+/* Check that fascist_firewall_choose_address_ls() returns the expected
+ * results. */
+#define CHECK_CHOSEN_ADDR_NULL_LS() \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(NULL, 1, &chosen_ls_ap); \
+    expect_single_log_msg("Unknown or missing link specifiers"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_CHOSEN_ADDR_LS(fake_ls, pref_only, expect_rv, expect_ap) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, pref_only, &chosen_ls_ap); \
+    if (smartlist_len(fake_ls) == 0) { \
+      expect_single_log_msg("Link specifiers are empty"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &chosen_ls_ap.addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, chosen_ls_ap.port); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_LS_LEGACY_ONLY(fake_ls) \
+  STMT_BEGIN \
+    tor_addr_port_t chosen_ls_ap; \
+    tor_addr_make_null(&chosen_ls_ap.addr, AF_UNSPEC); \
+    chosen_ls_ap.port = 0; \
+    setup_full_capture_of_logs(LOG_WARN); \
+    fascist_firewall_choose_address_ls(fake_ls, 0, &chosen_ls_ap); \
+    expect_single_log_msg("None of our link specifiers have IPv4 or IPv6"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS(fake_ls, direct_conn, expect_ap) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    if (fake_ls == NULL) { \
+      tt_ptr_op(ei, OP_EQ, NULL); \
+      expect_single_log_msg("Specified link specifiers is null"); \
+    } else { \
+      expect_no_log_entry(); \
+      tt_assert(tor_addr_eq(&(expect_ap).addr, &ei->addr)); \
+      tt_int_op((expect_ap).port, OP_EQ, ei->port); \
+      extend_info_free(ei); \
+    } \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(fake_ls) \
+  STMT_BEGIN \
+    setup_full_capture_of_logs(LOG_WARN); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, NULL, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg("Specified onion key is null"); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(fake_ls, direct_conn) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, \
+                                                       direct_conn); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+  STMT_END
+
+#define CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(fake_ls, msg_level, msg) \
+  STMT_BEGIN \
+    curve25519_secret_key_t seckey; \
+    curve25519_secret_key_generate(&seckey, 0); \
+    curve25519_public_key_t pubkey; \
+    curve25519_public_key_generate(&pubkey, &seckey); \
+    setup_full_capture_of_logs(msg_level); \
+    extend_info_t *ei = hs_get_extend_info_from_lspecs(fake_ls, &pubkey, 0); \
+    tt_ptr_op(ei, OP_EQ, NULL); \
+    expect_single_log_msg(msg); \
+    teardown_capture_of_logs(); \
+  STMT_END
+
 /** Run unit tests for fascist_firewall_choose_address */
 static void
 test_policies_fascist_firewall_choose_address(void *arg)
@@ -2422,6 +2522,141 @@ test_policies_fascist_firewall_choose_address(void *arg)
   CHECK_CHOSEN_ADDR_RN(fake_rs, fake_node, FIREWALL_DIR_CONNECTION, 1, 1,
                        ipv4_dir_ap);
 
+  /* Test firewall_choose_address_ls(). To do this, we make a fake link
+   * specifier. */
+  smartlist_t *lspecs = smartlist_new(),
+              *lspecs_blank = smartlist_new(),
+              *lspecs_v4 = smartlist_new(),
+              *lspecs_v6 = smartlist_new(),
+              *lspecs_no_legacy = smartlist_new(),
+              *lspecs_legacy_only = smartlist_new();
+  link_specifier_t *fake_ls;
+
+  /* IPv4 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV4);
+  link_specifier_set_un_ipv4_addr(fake_ls,
+                                  tor_addr_to_ipv4h(&ipv4_or_ap.addr));
+  link_specifier_set_un_ipv4_port(fake_ls, ipv4_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, sizeof(ipv4_or_ap.addr.addr.in_addr) +
+                            sizeof(ipv4_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_no_legacy, fake_ls);
+
+  /* IPv6 link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_IPV6);
+  size_t addr_len = link_specifier_getlen_un_ipv6_addr(fake_ls);
+  const uint8_t *in6_addr = tor_addr_to_in6_addr8(&ipv6_or_ap.addr);
+  uint8_t *ipv6_array = link_specifier_getarray_un_ipv6_addr(fake_ls);
+  memcpy(ipv6_array, in6_addr, addr_len);
+  link_specifier_set_un_ipv6_port(fake_ls, ipv6_or_ap.port);
+  link_specifier_set_ls_len(fake_ls, addr_len + sizeof(ipv6_or_ap.port));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Legacy ID link specifier */
+  fake_ls = link_specifier_new();
+  link_specifier_set_ls_type(fake_ls, LS_LEGACY_ID);
+  uint8_t *legacy_id = link_specifier_getarray_un_legacy_id(fake_ls);
+  memset(legacy_id, 'A', sizeof(*legacy_id));
+  link_specifier_set_ls_len(fake_ls,
+                            link_specifier_getlen_un_legacy_id(fake_ls));
+  smartlist_add(lspecs, fake_ls);
+  smartlist_add(lspecs_legacy_only, fake_ls);
+  smartlist_add(lspecs_v4, fake_ls);
+  smartlist_add(lspecs_v6, fake_ls);
+
+  /* Check with bogus requests. */
+  tor_addr_port_t null_ap; \
+  tor_addr_make_null(&null_ap.addr, AF_UNSPEC); \
+  null_ap.port = 0; \
+
+  /* Check for a null link state. */
+  CHECK_CHOSEN_ADDR_NULL_LS();
+  CHECK_HS_EXTEND_INFO_ADDR_LS(NULL, 1, null_ap);
+
+  /* Check for a blank link state. */
+  CHECK_CHOSEN_ADDR_LS(lspecs_blank, 0, 0, null_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_blank, 0);
+
+  /* Check for a link state with only a Legacy ID. */
+  CHECK_LS_LEGACY_ONLY(lspecs_legacy_only);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_legacy_only, 0);
+  smartlist_free(lspecs_legacy_only);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_NULL_KEY(lspecs_blank);
+  smartlist_free(lspecs_blank);
+
+  /* Check with a null onion_key. */
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_MSG(lspecs_no_legacy, LOG_WARN,
+                                          "Missing Legacy ID in link state");
+  smartlist_free(lspecs_no_legacy);
+
+  /* Enable both IPv4 and IPv6. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 1;
+
+  /* Prefer IPv4, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* Prefer IPv6, enable both IPv4 and IPv6. */
+  mock_options.ClientPreferIPv6ORPort = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  /* IPv4-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 1;
+  mock_options.ClientUseIPv6 = 0;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv4_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv4_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v6, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv4_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 1);
+
+  /* IPv6-only. */
+  memset(&mock_options, 0, sizeof(or_options_t));
+  mock_options.ClientUseIPv4 = 0;
+  mock_options.ClientUseIPv6 = 1;
+
+  CHECK_CHOSEN_ADDR_LS(lspecs, 0, 1, ipv6_or_ap);
+  CHECK_CHOSEN_ADDR_LS(lspecs, 1, 1, ipv6_or_ap);
+
+  CHECK_CHOSEN_ADDR_LS(lspecs_v4, 0, 0, null_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 1, ipv6_or_ap);
+  CHECK_HS_EXTEND_INFO_ADDR_LS(lspecs, 0, ipv4_or_ap);
+
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v4, 1);
+  CHECK_HS_EXTEND_INFO_ADDR_LS_EXPECT_NULL(lspecs_v6, 0);
+
+  smartlist_free(lspecs_v4);
+  smartlist_free(lspecs_v6);
+
+  SMARTLIST_FOREACH(lspecs, link_specifier_t *, lspec, \
+                    link_specifier_free(lspec)); \
+  smartlist_free(lspecs);
+
  done:
   UNMOCK(get_options);
 }
diff --git a/src/test/test_prob_distr.c b/src/test/test_prob_distr.c
new file mode 100644
index 0000000000..541a81df3a
--- /dev/null
+++ b/src/test/test_prob_distr.c
@@ -0,0 +1,1406 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_prob_distr.c
+ * \brief Test probability distributions.
+ * \detail
+ *
+ * For each probability distribution we do two kinds of tests:
+ *
+ * a) We do numerical deterministic testing of their cdf/icdf/sf/isf functions
+ *    and the various relationships between them for each distribution. We also
+ *    do deterministic tests on their sampling functions. Test vectors for
+ *    these tests were computed from alternative implementations and were
+ *    eyeballed to make sure they make sense
+ *    (e.g. src/test/prob_distr_mpfr_ref.c computes logit(p) using GNU mpfr
+ *    with 200-bit precision and is then tested in test_logit_logistic()).
+ *
+ * b) We do stochastic hypothesis testing (G-test) to ensure that sampling from
+ *    the given distributions is distributed properly. The stochastic tests are
+ *    slow and their false positive rate is not well suited for CI, so they are
+ *    currently disabled-by-default and put into 'tests-slow'.
+ */
+
+#define PROB_DISTR_PRIVATE
+
+#include "orconfig.h"
+
+#include "test/test.h"
+
+#include "core/or/or.h"
+
+#include "lib/math/prob_distr.h"
+#include "lib/math/fp.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "test/rng_test_helpers.h"
+
+#include <float.h>
+#include <math.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/**
+ * Return floor(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)floor(d).
+ */
+static size_t
+floor_to_size_t(double d)
+{
+  double integral_d = floor(d);
+  return (size_t)integral_d;
+}
+
+/**
+ * Return ceil(d) converted to size_t, as a workaround for complaints
+ * under -Wbad-function-cast for (size_t)ceil(d).
+ */
+static size_t
+ceil_to_size_t(double d)
+{
+  double integral_d = ceil(d);
+  return (size_t)integral_d;
+}
+
+/*
+ * Geometric(p) distribution, supported on {1, 2, 3, ...}.
+ *
+ * Compute the probability mass function Geom(n; p) of the number of
+ * trials before the first success when success has probability p.
+ */
+static double
+logpmf_geometric(unsigned n, double p)
+{
+  /* This is actually a check against 1, but we do >= so that the compiler
+     does not raise a -Wfloat-equal */
+  if (p >= 1) {
+    if (n == 1)
+      return 0;
+    else
+      return -HUGE_VAL;
+  }
+  return (n - 1)*log1p(-p) + log(p);
+}
+
+/**
+ * Compute the logistic function, translated in output by 1/2:
+ * logistichalf(x) = logistic(x) - 1/2.  Well-conditioned on the entire
+ * real plane, with maximum condition number 1 at 0.
+ *
+ * This implementation gives relative error bounded by 5 eps.
+ */
+static double
+logistichalf(double x)
+{
+  /*
+   * Rewrite this with the identity
+   *
+   *  1/(1 + e^{-x}) - 1/2
+   *  = (1 - 1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1/2 - e^{-x}/2)/(1 + e^{-x})
+   *  = (1 - e^{-x})/[2 (1 + e^{-x})]
+   *  = -(e^{-x} - 1)/[2 (1 + e^{-x})],
+   *
+   * which we can evaluate by -expm1(-x)/[2 (1 + exp(-x))].
+   *
+   * Suppose exp has error d0, + has error d1, expm1 has error
+   * d2, and / has error d3, so we evaluate
+   *
+   *  -(1 + d2) (1 + d3) (e^{-x} - 1)
+   *    / [2 (1 + d1) (1 + (1 + d0) e^{-x})].
+   *
+   * In the denominator,
+   *
+   *  1 + (1 + d0) e^{-x}
+   *  = 1 + e^{-x} + d0 e^{-x}
+   *  = (1 + e^{-x}) (1 + d0 e^{-x}/(1 + e^{-x})),
+   *
+   * so the relative error of the numerator is
+   *
+   *  d' = d2 + d3 + d2 d3,
+   * and of the denominator,
+   *  d'' = d1 + d0 e^{-x}/(1 + e^{-x}) + d0 d1 e^{-x}/(1 + e^{-x})
+   *      = d1 + d0 L(-x) + d0 d1 L(-x),
+   *
+   * where L(-x) is logistic(-x).  By Lemma 1 the relative error
+   * of the quotient is bounded by
+   *
+   *  2|d2 + d3 + d2 d3 - d1 - d0 L(x) + d0 d1 L(x)|,
+   *
+   * Since 0 < L(x) < 1, this is bounded by
+   *
+   *  2|d2| + 2|d3| + 2|d2 d3| + 2|d1| + 2|d0| + 2|d0 d1|
+   *  <= 4 eps + 2 eps^2.
+   */
+  if (x < log(DBL_EPSILON/8)) {
+    /*
+     * Avoid overflow in e^{-x}.  When x < log(eps/4), we
+     * we further have x < logit(eps/4), so that
+     * logistic(x) < eps/4.  Hence the relative error of
+     * logistic(x) - 1/2 from -1/2 is bounded by eps/2, and
+     * so the relative error of -1/2 from logistic(x) - 1/2
+     * is bounded by eps.
+     */
+    return -0.5;
+  } else {
+    return -expm1(-x)/(2*(1 + exp(-x)));
+  }
+}
+
+/**
+ * Compute the log of the sum of the exps.  Caller should arrange the
+ * array in descending order to minimize error because I don't want to
+ * deal with using temporary space and the one caller in this file
+ * arranges that anyway.
+ *
+ * Warning: This implementation does not handle infinite or NaN inputs
+ * sensibly, because I don't need that here at the moment.  (NaN, or
+ * -inf and +inf together, should yield NaN; +inf and finite should
+ * yield +inf; otherwise all -inf should be ignored because exp(-inf) =
+ * 0.)
+ */
+static double
+logsumexp(double *A, size_t n)
+{
+  double maximum, sum;
+  size_t i;
+
+  if (n == 0)
+    return log(0);
+
+  maximum = A[0];
+  for (i = 1; i < n; i++) {
+    if (A[i] > maximum)
+      maximum = A[i];
+  }
+
+  sum = 0;
+  for (i = n; i --> 0;)
+    sum += exp(A[i] - maximum);
+
+  return log(sum) + maximum;
+}
+
+/**
+ * Compute log(1 - e^x).  Defined only for negative x so that e^x < 1.
+ * This is the complement of a probability in log space.
+ */
+static double
+log1mexp(double x)
+{
+
+  /*
+   * We want to compute log on [0, 1/2) but log1p on [1/2, +inf),
+   * so partition x at -log(2) = log(1/2).
+   */
+  if (-log(2) < x)
+    return log(-expm1(x));
+  else
+    return log1p(-exp(x));
+}
+
+/*
+ * Tests of numerical errors in computing logit, logistic, and the
+ * various cdfs, sfs, icdfs, and isfs.
+ */
+
+#define arraycount(A) (sizeof(A)/sizeof(A[0]))
+
+/** Return relative error between <b>actual</b> and <b>expected</b>.
+ *  Special cases: If <b>expected</b> is zero or infinite, return 1 if
+ *  <b>actual</b> is equal to <b>expected</b> and 0 if not, since the
+ *  usual notion of relative error is undefined but we only use this
+ *  for testing relerr(e, a) <= bound.  If either is NaN, return NaN,
+ *  which has the property that NaN <= bound is false no matter what
+ *  bound is.
+ *
+ *  Beware: if you test !(relerr(e, a) > bound), then then the result
+ *  is true when a is NaN because NaN > bound is false too.  See
+ *  CHECK_RELERR for correct use to decide when to report failure.
+ */
+static double
+relerr(double expected, double actual)
+{
+  /*
+   * To silence -Wfloat-equal, we have to test for equality using
+   * inequalities: we have (fabs(expected) <= 0) iff (expected == 0),
+   * and (actual <= expected && actual >= expected) iff actual ==
+   * expected whether expected is zero or infinite.
+   */
+  if (fabs(expected) <= 0 || tor_isinf(expected)) {
+    if (actual <= expected && actual >= expected)
+      return 0;
+    else
+      return 1;
+  } else {
+    return fabs((expected - actual)/expected);
+  }
+}
+
+/** Check that relative error of <b>expected</b> and <b>actual</b> is within
+ *  <b>relerr_bound</b>.  Caller must arrange to have i and relerr_bound in
+ *  scope.  */
+#define CHECK_RELERR(expected, actual) do {                                   \
+  double check_expected = (expected);                                         \
+  double check_actual = (actual);                                             \
+  const char *str_expected = #expected;                                       \
+  const char *str_actual = #actual;                                           \
+  double check_relerr = relerr(expected, actual);                             \
+  if (!(relerr(check_expected, check_actual) <= relerr_bound)) {              \
+    log_warn(LD_GENERAL, "%s:%d: case %u: relerr(%s=%.17e, %s=%.17e)"        \
+             " = %.17e > %.17e\n",                                            \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_expected, check_expected,                                    \
+             str_actual, check_actual,                                        \
+             check_relerr, relerr_bound);                                     \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/* Check that a <= b.
+ * Caller must arrange to have i in scope.  */
+#define CHECK_LE(a, b) do {                                                   \
+  double check_a = (a);                                                       \
+  double check_b = (b);                                                       \
+  const char *str_a = #a;                                                     \
+  const char *str_b = #b;                                                     \
+  if (!(check_a <= check_b)) {                                                \
+    log_warn(LD_GENERAL, "%s:%d: case %u: %s=%.17e > %s=%.17e\n",             \
+             __func__, __LINE__, (unsigned) i,                                \
+             str_a, check_a, str_b, check_b);                                 \
+    ok = false;                                                               \
+  }                                                                           \
+} while (0)
+
+/**
+ * Test the logit and logistic functions.  Confirm that they agree with
+ * the cdf, sf, icdf, and isf of the standard Logistic distribution.
+ * Confirm that the sampler for the standard logistic distribution maps
+ * [0, 1] into the right subinterval for the inverse transform, for
+ * this implementation.
+ */
+static void
+test_logit_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    double x;                   /* x = logit(p) */
+    double p;                   /* p = logistic(x) */
+    double phalf;               /* p - 1/2 = logistic(x) - 1/2 */
+  } cases[] = {
+    { -HUGE_VAL, 0, -0.5 },
+    { -1000, 0, -0.5 },
+    { -710, 4.47628622567513e-309, -0.5 },
+    { -708, 3.307553003638408e-308, -0.5 },
+    { -2, .11920292202211755, -.3807970779778824 },
+    { -1.0000001, .2689414017088022, -.23105859829119776 },
+    { -1, .2689414213699951, -.23105857863000487 },
+    { -0.9999999, .26894144103118883, -.2310585589688111 },
+    /* see src/test/prob_distr_mpfr_ref.c for computation */
+    { -4.000000000537333e-5, .49999, -1.0000000000010001e-5 },
+    { -4.000000000533334e-5, .49999, -.00001 },
+    { -4.000000108916878e-9, .499999999, -1.0000000272292198e-9 },
+    { -4e-9, .499999999, -1e-9 },
+    { -4e-16, .5, -1e-16 },
+    { -4e-300, .5, -1e-300 },
+    { 0, .5, 0 },
+    { 4e-300, .5, 1e-300 },
+    { 4e-16, .5, 1e-16 },
+    { 3.999999886872274e-9, .500000001, 9.999999717180685e-10 },
+    { 4e-9, .500000001, 1e-9 },
+    { 4.0000000005333336e-5, .50001, .00001 },
+    { 8.000042667076272e-3, .502, .002 },
+    { 0.9999999, .7310585589688111, .2310585589688111 },
+    { 1, .7310585786300049, .23105857863000487 },
+    { 1.0000001, .7310585982911977, .23105859829119774 },
+    { 2, .8807970779778823, .3807970779778824 },
+    { 708, 1, .5 },
+    { 710, 1, .5 },
+    { 1000, 1, .5 },
+    { HUGE_VAL, 1, .5 },
+  };
+  double relerr_bound = 3e-15; /* >10eps */
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double phalf = cases[i].phalf;
+
+    /*
+     * cdf is logistic, icdf is logit, and symmetry for
+     * sf/isf.
+     */
+    CHECK_RELERR(logistic(x), cdf_logistic(x, 0, 1));
+    CHECK_RELERR(logistic(-x), sf_logistic(x, 0, 1));
+    CHECK_RELERR(logit(p), icdf_logistic(p, 0, 1));
+    CHECK_RELERR(-logit(p), isf_logistic(p, 0, 1));
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2, 0, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2, 0, 2));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0, 2)/2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, 2)/2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x/2, 0, .5));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x/2, 0, .5));
+    CHECK_RELERR(icdf_logistic(p, 0, 1), icdf_logistic(p, 0,.5)*2);
+    CHECK_RELERR(isf_logistic(p, 0, 1), isf_logistic(p, 0, .5)*2);
+
+    CHECK_RELERR(cdf_logistic(x, 0, 1), cdf_logistic(x*2 + 1, 1, 2));
+    CHECK_RELERR(sf_logistic(x, 0, 1), sf_logistic(x*2 + 1, 1, 2));
+
+    /*
+     * For p near 0 and p near 1/2, the arithmetic of
+     * translating by 1 loses precision.
+     */
+    if (fabs(p) > DBL_EPSILON && fabs(p) < 0.4) {
+      CHECK_RELERR(icdf_logistic(p, 0, 1),
+          (icdf_logistic(p, 1, 2) - 1)/2);
+      CHECK_RELERR(isf_logistic(p, 0, 1),
+          (isf_logistic(p, 1, 2) - 1)/2);
+    }
+
+    CHECK_RELERR(p, logistic(x));
+    CHECK_RELERR(phalf, logistichalf(x));
+
+    /*
+     * On the interior floating-point numbers, either logit or
+     * logithalf had better give the correct answer.
+     *
+     * For probabilities near 0, we can get much finer resolution with
+     * logit, and for probabilities near 1/2, we can get much finer
+     * resolution with logithalf by representing them using p - 1/2.
+     *
+     * E.g., we can write -.00001 for phalf, and .49999 for p, but the
+     * difference 1/2 - .00001 gives 1.0000000000010001e-5 in binary64
+     * arithmetic.  So test logit(.49999) which should give the same
+     * answer as logithalf(-1.0000000000010001e-5), namely
+     * -4.000000000537333e-5, and also test logithalf(-.00001) which
+     * gives -4.000000000533334e-5 instead -- but don't expect
+     * logit(.49999) to give -4.000000000533334e-5 even though it looks
+     * like 1/2 - .00001.
+     *
+     * A naive implementation of logit will just use log(p/(1 - p)) and
+     * give the answer -4.000000000551673e-05 for .49999, which is
+     * wrong in a lot of digits, which happens because log is
+     * ill-conditioned near 1 and thus amplifies whatever relative
+     * error we made in computing p/(1 - p).
+     */
+    if ((0 < p && p < 1) || tor_isinf(x)) {
+      if (phalf >= p - 0.5 && phalf <= p - 0.5)
+        CHECK_RELERR(x, logit(p));
+      if (p >= 0.5 + phalf && p <= 0.5 + phalf)
+        CHECK_RELERR(x, logithalf(phalf));
+    }
+
+    CHECK_RELERR(-phalf, logistichalf(-x));
+    if (fabs(phalf) < 0.5 || tor_isinf(x))
+      CHECK_RELERR(-x, logithalf(-phalf));
+    if (p < 1 || tor_isinf(x)) {
+      CHECK_RELERR(1 - p, logistic(-x));
+      if (p > .75 || tor_isinf(x))
+        CHECK_RELERR(-x, logit(1 - p));
+    } else {
+      CHECK_LE(logistic(-x), 1e-300);
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(logit(p0/(1 + M_E)), sample_logistic(0, 0, p0));
+    CHECK_RELERR(-logit(p0/(1 + M_E)), sample_logistic(1, 0, p0));
+    CHECK_RELERR(logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(0, 1, p0));
+    CHECK_RELERR(-logithalf(p0*(0.5 - 1/(1 + M_E))),
+        sample_logistic(1, 1, p0));
+  }
+
+  if (!ok)
+    printf("fail logit/logistic / logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the LogLogistic distribution.
+ */
+static void
+test_log_logistic(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the LogLogistic distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given LogLogistic
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given LogLogistic
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { 1e-15, 1e-15, .999999999999999 },
+    { .1, .09090909090909091, .90909090909090909 },
+    { .25, .2, .8 },
+    { .5, .33333333333333333, .66666666666666667 },
+    { .75, .42857142857142855, .5714285714285714 },
+    { .9999, .49997499874993756, .5000250012500626 },
+    { .99999999, .49999999749999996, .5000000025 },
+    { .999999999999999, .49999999999999994, .5000000000000002 },
+    { 1, .5, .5 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_log_logistic(x, 1, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(p, cdf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, sf_log_logistic(x, 1, 1));
+    CHECK_RELERR(np, sf_log_logistic(x/2, .5, 1));
+    CHECK_RELERR(np, sf_log_logistic(x*2, 2, 1));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x), 1, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(np, sf_log_logistic(sqrt(x)*2, 2, 2));
+    if (2*sqrt(DBL_MIN) < x) {
+      CHECK_RELERR(np, sf_log_logistic(x*x, 1, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x/2, .5, .5));
+      CHECK_RELERR(np, sf_log_logistic(x*x*2, 2, .5));
+    }
+
+    CHECK_RELERR(np, cdf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(np, cdf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(np, cdf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(np, cdf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(np, cdf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(np, cdf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(np, cdf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(np, cdf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(p, sf_log_logistic(1/x, 1, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*x), .5, 1));
+    CHECK_RELERR(p, sf_log_logistic(2/x, 2, 1));
+    CHECK_RELERR(p, sf_log_logistic(1/sqrt(x), 1, 2));
+    CHECK_RELERR(p, sf_log_logistic(1/(2*sqrt(x)), .5, 2));
+    CHECK_RELERR(p, sf_log_logistic(2/sqrt(x), 2, 2));
+    if (2*sqrt(DBL_MIN) < x && x < 1/(2*sqrt(DBL_MIN))) {
+      CHECK_RELERR(p, sf_log_logistic(1/(x*x), 1, .5));
+      CHECK_RELERR(p, sf_log_logistic(1/(2*x*x), .5, .5));
+      CHECK_RELERR(p, sf_log_logistic(2/(x*x), 2, .5));
+    }
+
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(x/2, icdf_log_logistic(p, .5, 1));
+    CHECK_RELERR(x*2, icdf_log_logistic(p, 2, 1));
+    CHECK_RELERR(x, icdf_log_logistic(p, 1, 1));
+    CHECK_RELERR(sqrt(x)/2, icdf_log_logistic(p, .5, 2));
+    CHECK_RELERR(sqrt(x)*2, icdf_log_logistic(p, 2, 2));
+    CHECK_RELERR(sqrt(x), icdf_log_logistic(p, 1, 2));
+    CHECK_RELERR(x*x/2, icdf_log_logistic(p, .5, .5));
+    CHECK_RELERR(x*x*2, icdf_log_logistic(p, 2, .5));
+
+    if (np < .9) {
+      CHECK_RELERR(x, isf_log_logistic(np, 1, 1));
+      CHECK_RELERR(x/2, isf_log_logistic(np, .5, 1));
+      CHECK_RELERR(x*2, isf_log_logistic(np, 2, 1));
+      CHECK_RELERR(sqrt(x), isf_log_logistic(np, 1, 2));
+      CHECK_RELERR(sqrt(x)/2, isf_log_logistic(np, .5, 2));
+      CHECK_RELERR(sqrt(x)*2, isf_log_logistic(np, 2, 2));
+      CHECK_RELERR(x*x, isf_log_logistic(np, 1, .5));
+      CHECK_RELERR(x*x/2, isf_log_logistic(np, .5, .5));
+      CHECK_RELERR(x*x*2, isf_log_logistic(np, 2, .5));
+
+      CHECK_RELERR(1/x, icdf_log_logistic(np, 1, 1));
+      CHECK_RELERR(1/(2*x), icdf_log_logistic(np, .5, 1));
+      CHECK_RELERR(2/x, icdf_log_logistic(np, 2, 1));
+      CHECK_RELERR(1/sqrt(x), icdf_log_logistic(np, 1, 2));
+      CHECK_RELERR(1/(2*sqrt(x)),
+          icdf_log_logistic(np, .5, 2));
+      CHECK_RELERR(2/sqrt(x), icdf_log_logistic(np, 2, 2));
+      CHECK_RELERR(1/(x*x), icdf_log_logistic(np, 1, .5));
+      CHECK_RELERR(1/(2*x*x), icdf_log_logistic(np, .5, .5));
+      CHECK_RELERR(2/(x*x), icdf_log_logistic(np, 2, .5));
+    }
+
+    CHECK_RELERR(1/x, isf_log_logistic(p, 1, 1));
+    CHECK_RELERR(1/(2*x), isf_log_logistic(p, .5, 1));
+    CHECK_RELERR(2/x, isf_log_logistic(p, 2, 1));
+    CHECK_RELERR(1/sqrt(x), isf_log_logistic(p, 1, 2));
+    CHECK_RELERR(1/(2*sqrt(x)), isf_log_logistic(p, .5, 2));
+    CHECK_RELERR(2/sqrt(x), isf_log_logistic(p, 2, 2));
+    CHECK_RELERR(1/(x*x), isf_log_logistic(p, 1, .5));
+    CHECK_RELERR(1/(2*x*x), isf_log_logistic(p, .5, .5));
+    CHECK_RELERR(2/(x*x), isf_log_logistic(p, 2, .5));
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(0.5*p0/(1 - 0.5*p0), sample_log_logistic(0, p0));
+    CHECK_RELERR((1 - 0.5*p0)/(0.5*p0),
+        sample_log_logistic(1, p0));
+  }
+
+  if (!ok)
+    printf("fail log logistic cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, isf of the Weibull distribution.
+ */
+static void
+test_weibull(void *arg)
+{
+  (void) arg;
+
+  static const struct {
+    /* x is a point in the support of the Weibull distribution */
+    double x;
+    /* 'p' is the probability that a random variable X for a given Weibull
+     * probability ditribution will take value less-or-equal to x */
+    double p;
+    /* 'np' is the probability that a random variable X for a given Weibull
+     * probability distribution will take value greater-or-equal to x. */
+    double np;
+  } cases[] = {
+    { 0, 0, 1 },
+    { 1e-300, 1e-300, 1 },
+    { 1e-17, 1e-17, 1 },
+    { .1, .09516258196404043, .9048374180359595 },
+    { .5, .3934693402873666, .6065306597126334 },
+    { .6931471805599453, .5, .5 },
+    { 1, .6321205588285577, .36787944117144233 },
+    { 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 36, .9999999999999998, 2.319522830243569e-16 },
+    { 37, .9999999999999999, 8.533047625744066e-17 },
+    { 38, 1, 3.1391327920480296e-17 },
+    { 100, 1, 3.720075976020836e-44 },
+    { 708, 1, 3.307553003638408e-308 },
+    { 710, 1, 4.47628622567513e-309 },
+    { 1000, 1, 0 },
+    { HUGE_VAL, 1, 0 },
+  };
+  double relerr_bound = 3e-15;
+  size_t i;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_weibull(x, 1, 1));
+    CHECK_RELERR(p, cdf_weibull(x/2, .5, 1));
+    CHECK_RELERR(p, cdf_weibull(x*2, 2, 1));
+    /* For 0 < x < sqrt(DBL_MIN), x^2 loses lots of bits.  */
+    if (x <= 0 ||
+        sqrt(DBL_MIN) <= x) {
+      CHECK_RELERR(p, cdf_weibull(x*x, 1, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x/2, .5, .5));
+      CHECK_RELERR(p, cdf_weibull(x*x*2, 2, .5));
+    }
+    CHECK_RELERR(p, cdf_weibull(sqrt(x), 1, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)/2, .5, 2));
+    CHECK_RELERR(p, cdf_weibull(sqrt(x)*2, 2, 2));
+    CHECK_RELERR(np, sf_weibull(x, 1, 1));
+    CHECK_RELERR(np, sf_weibull(x/2, .5, 1));
+    CHECK_RELERR(np, sf_weibull(x*2, 2, 1));
+    CHECK_RELERR(np, sf_weibull(x*x, 1, .5));
+    CHECK_RELERR(np, sf_weibull(x*x/2, .5, .5));
+    CHECK_RELERR(np, sf_weibull(x*x*2, 2, .5));
+    if (x >= 10) {
+      /*
+       * exp amplifies the error of sqrt(x)^2
+       * proportionally to exp(x); for large inputs
+       * this is significant.
+       */
+      double t = -expm1(-x*(2*DBL_EPSILON + DBL_EPSILON));
+      relerr_bound = t + DBL_EPSILON + t*DBL_EPSILON;
+      if (relerr_bound < 3e-15)
+        /*
+         * The tests are written only to 16
+         * decimal places anyway even if your
+         * `double' is, say, i387 binary80, for
+         * whatever reason.
+         */
+        relerr_bound = 3e-15;
+      CHECK_RELERR(np, sf_weibull(sqrt(x), 1, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)/2, .5, 2));
+      CHECK_RELERR(np, sf_weibull(sqrt(x)*2, 2, 2));
+    }
+
+    if (p <= 0.75) {
+      /*
+       * For p near 1, not enough precision near 1 to
+       * recover x.
+       */
+      CHECK_RELERR(x, icdf_weibull(p, 1, 1));
+      CHECK_RELERR(x/2, icdf_weibull(p, .5, 1));
+      CHECK_RELERR(x*2, icdf_weibull(p, 2, 1));
+    }
+    if (p >= 0.25 && !tor_isinf(x) && np > 0) {
+      /*
+       * For p near 0, not enough precision in np
+       * near 1 to recover x.  For 0, isf gives inf,
+       * even if p is precise enough for the icdf to
+       * work.
+       */
+      CHECK_RELERR(x, isf_weibull(np, 1, 1));
+      CHECK_RELERR(x/2, isf_weibull(np, .5, 1));
+      CHECK_RELERR(x*2, isf_weibull(np, 2, 1));
+    }
+  }
+
+  for (i = 0; i <= 100; i++) {
+    double p0 = (double)i/100;
+
+    CHECK_RELERR(3*sqrt(-log(p0/2)), sample_weibull(0, p0, 3, 2));
+    CHECK_RELERR(3*sqrt(-log1p(-p0/2)),
+        sample_weibull(1, p0, 3, 2));
+  }
+
+  if (!ok)
+    printf("fail Weibull cdf/sf\n");
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the cdf, sf, icdf, and isf of the generalized Pareto
+ * distribution.
+ */
+static void
+test_genpareto(void *arg)
+{
+  (void) arg;
+
+  struct {
+    /* xi is the 'xi' parameter of the generalized Pareto distribution, and the
+     * rest are the same as in the above tests */
+    double xi, x, p, np;
+  } cases[] = {
+    { 0, 0, 0, 1 },
+    { 1e-300, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-300, .1, .09516258196404043, .9048374180359595 },
+    { 1e-300, 1, .6321205588285577, .36787944117144233 },
+    { 1e-300, 10, .9999546000702375, 4.5399929762484854e-5 },
+    { 1e-200, 1e-16, 9.999999999999999e-17, .9999999999999999 },
+    { 1e-16, 1e-200, 9.999999999999998e-201, 1 },
+    { 1e-16, 1e-16, 1e-16, 1 },
+    { 1e-16, .004, 3.992010656008528e-3, .9960079893439915 },
+    { 1e-16, .1, .09516258196404043, .9048374180359595 },
+    { 1e-16, 1, .6321205588285577, .36787944117144233 },
+    { 1e-16, 10, .9999546000702375, 4.539992976248509e-5 },
+    { 1e-10, 1e-6, 9.999995000001667e-7, .9999990000005 },
+    { 1e-8, 1e-8, 9.999999950000001e-9, .9999999900000001 },
+    { 1, 1e-300, 1e-300, 1 },
+    { 1, 1e-16, 1e-16, .9999999999999999 },
+    { 1, .1, .09090909090909091, .9090909090909091 },
+    { 1, 1, .5, .5 },
+    { 1, 10, .9090909090909091, .0909090909090909 },
+    { 1, 100, .9900990099009901, .0099009900990099 },
+    { 1, 1000, .999000999000999, 9.990009990009992e-4 },
+    { 10, 1e-300, 1e-300, 1 },
+    { 10, 1e-16, 9.999999999999995e-17, .9999999999999999 },
+    { 10, .1, .06696700846319258, .9330329915368074 },
+    { 10, 1, .21320655780322778, .7867934421967723 },
+    { 10, 10, .3696701667040189, .6303298332959811 },
+    { 10, 100, .49886285755007337, .5011371424499267 },
+    { 10, 1000, .6018968102992647, .3981031897007353 },
+  };
+  double xi_array[] = { -1.5, -1, -1e-30, 0, 1e-30, 1, 1.5 };
+  size_t i, j;
+  double relerr_bound = 3e-15;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double xi = cases[i].xi;
+    double x = cases[i].x;
+    double p = cases[i].p;
+    double np = cases[i].np;
+
+    CHECK_RELERR(p, cdf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(p, cdf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(p, cdf_genpareto(x/2, 0, .5, xi));
+    CHECK_RELERR(np, sf_genpareto(x, 0, 1, xi));
+    CHECK_RELERR(np, sf_genpareto(x*2, 0, 2, xi));
+    CHECK_RELERR(np, sf_genpareto(x/2, 0, .5, xi));
+
+    if (p < .5) {
+      CHECK_RELERR(x, icdf_genpareto(p, 0, 1, xi));
+      CHECK_RELERR(x*2, icdf_genpareto(p, 0, 2, xi));
+      CHECK_RELERR(x/2, icdf_genpareto(p, 0, .5, xi));
+    }
+    if (np < .5) {
+      CHECK_RELERR(x, isf_genpareto(np, 0, 1, xi));
+      CHECK_RELERR(x*2, isf_genpareto(np, 0, 2, xi));
+      CHECK_RELERR(x/2, isf_genpareto(np, 0, .5, xi));
+    }
+  }
+
+  for (i = 0; i < arraycount(xi_array); i++) {
+    for (j = 0; j <= 100; j++) {
+      double p0 = (j == 0 ? 2*DBL_MIN : (double)j/100);
+
+      /* This is actually a check against 0, but we do <= so that the compiler
+         does not raise a -Wfloat-equal */
+      if (fabs(xi_array[i]) <= 0) {
+        /*
+         * When xi == 0, the generalized Pareto
+         * distribution reduces to an
+         * exponential distribution.
+         */
+        CHECK_RELERR(-log(p0/2),
+            sample_genpareto(0, p0, 0));
+        CHECK_RELERR(-log1p(-p0/2),
+            sample_genpareto(1, p0, 0));
+      } else {
+        CHECK_RELERR(expm1(-xi_array[i]*log(p0/2))/xi_array[i],
+            sample_genpareto(0, p0, xi_array[i]));
+        CHECK_RELERR((j == 0 ? DBL_MIN :
+                expm1(-xi_array[i]*log1p(-p0/2))/xi_array[i]),
+            sample_genpareto(1, p0, xi_array[i]));
+      }
+
+      CHECK_RELERR(isf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(0, p0, xi_array[i]));
+      CHECK_RELERR(icdf_genpareto(p0/2, 0, 1, xi_array[i]),
+          sample_genpareto(1, p0, xi_array[i]));
+    }
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/**
+ * Test the deterministic sampler for uniform distribution on [a, b].
+ *
+ * This currently only tests whether the outcome lies within [a, b].
+ */
+static void
+test_uniform_interval(void *arg)
+{
+  (void) arg;
+  struct {
+    /* Sample from a uniform distribution with parameters 'a' and 'b', using
+     * 't' as the sampling index. */
+    double t, a, b;
+  } cases[] = {
+    { 0, 0, 0 },
+    { 0, 0, 1 },
+    { 0, 1.0000000000000007, 3.999999999999995 },
+    { 0, 4000, 4000 },
+    { 0.42475836677491291, 4000, 4000 },
+    { 0, -DBL_MAX, DBL_MAX },
+    { 0.25, -DBL_MAX, DBL_MAX },
+    { 0.5, -DBL_MAX, DBL_MAX },
+  };
+  size_t i = 0;
+  bool ok = true;
+
+  for (i = 0; i < arraycount(cases); i++) {
+    double t = cases[i].t;
+    double a = cases[i].a;
+    double b = cases[i].b;
+
+    CHECK_LE(a, sample_uniform_interval(t, a, b));
+    CHECK_LE(sample_uniform_interval(t, a, b), b);
+
+    CHECK_LE(a, sample_uniform_interval(1 - t, a, b));
+    CHECK_LE(sample_uniform_interval(1 - t, a, b), b);
+
+    CHECK_LE(sample_uniform_interval(t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(t, -b, -a));
+
+    CHECK_LE(sample_uniform_interval(1 - t, -b, -a), -a);
+    CHECK_LE(-b, sample_uniform_interval(1 - t, -b, -a));
+  }
+
+  tt_assert(ok);
+
+ done:
+  ;
+}
+
+/********************** Stochastic tests ****************************/
+
+/*
+ * Psi test, sometimes also called G-test.  The psi test statistic,
+ * suitably scaled, has chi^2 distribution, but the psi test tends to
+ * have better statistical power in practice to detect deviations than
+ * the chi^2 test does.  (The chi^2 test statistic is the first term of
+ * the Taylor expansion of the psi test statistic.)  The psi test is
+ * generic, for any CDF; particular distributions might have higher-
+ * power tests to distinguish them from predictable deviations or bugs.
+ *
+ * We choose the psi critical value so that a single psi test has
+ * probability below alpha = 1% of spuriously failing even if all the
+ * code is correct.  But the false positive rate for a suite of n tests
+ * is higher: 1 - Binom(0; n, alpha) = 1 - (1 - alpha)^n.  For n = 10,
+ * this is about 10%, and for n = 100 it is well over 50%.
+ *
+ * Given that these tests will run with every CI job, we want to drive down the
+ * false positive rate. We can drive it down by running each test four times,
+ * and accepting it if it passes at least once; in that case, it is as if we
+ * used Binom(4; 2, alpha) = alpha^4 as the false positive rate for each test,
+ * and for n = 10 tests, it would be 9.99999959506e-08. If each CI build has 14
+ * jobs, then the chance of a CI build failing is 1.39999903326e-06, which
+ * means that a CI build will break with probability 50% after about 495106
+ * builds.
+ *
+ * The critical value for a chi^2 distribution with 100 degrees of
+ * freedom and false positive rate alpha = 1% was taken from:
+ *
+ *  NIST/SEMATECH e-Handbook of Statistical Methods, Section
+ *  1.3.6.7.4 `Critical Values of the Chi-Square Distribution',
+ *  <https://www.itl.nist.gov/div898/handbook/eda/section3/eda3674.htm>,
+ *  retrieved 2018-10-28.
+ */
+
+static const size_t NSAMPLES = 100000;
+/* Number of chances we give to the test to succeed. */
+static const unsigned NTRIALS = 4;
+/* Number of times we want the test to pass per NTRIALS. */
+static const unsigned NPASSES_MIN = 1;
+
+#define PSI_DF 100                          /* degrees of freedom */
+static const double PSI_CRITICAL = 135.807; /* critical value, alpha = .01 */
+
+/**
+ * Perform a psi test on an array of sample counts, C, adding up to N
+ * samples, and an array of log expected probabilities, logP,
+ * representing the null hypothesis for the distribution of samples
+ * counted.  Return false if the psi test rejects the null hypothesis,
+ * true if otherwise.
+ */
+static bool
+psi_test(const size_t C[PSI_DF], const double logP[PSI_DF], size_t N)
+{
+  double psi = 0;
+  double c = 0;                 /* Kahan compensation */
+  double t, u;
+  size_t i;
+
+  for (i = 0; i < PSI_DF; i++) {
+    /*
+     * c*log(c/(n*p)) = (1/n) * f*log(f/p) where f = c/n is
+     * the frequency, and f*log(f/p) ---> 0 as f ---> 0, so
+     * this is a reasonable choice.  Further, any mass that
+     * _fails_ to turn up in this bin will inflate another
+     * bin instead, so we don't really lose anything by
+     * ignoring empty bins even if they have high
+     * probability.
+     */
+    if (C[i] == 0)
+      continue;
+    t = C[i]*(log((double)C[i]/N) - logP[i]) - c;
+    u = psi + t;
+    c = (u - psi) - t;
+    psi = u;
+  }
+  psi *= 2;
+
+  return psi <= PSI_CRITICAL;
+}
+
+static bool
+test_stochastic_geometric_impl(double p)
+{
+  const struct geometric_t geometric = {
+    .base = GEOMETRIC(geometric),
+    .p = p,
+  };
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  unsigned i;
+  size_t j;
+
+  /* Compute logP[i] = Geom(i + 1; p).  */
+  for (i = 0; i < PSI_DF - 1; i++)
+    logP[i] = logpmf_geometric(i + 1, p);
+
+  /* Compute logP[n-1] = log (1 - (P[0] + P[1] + ... + P[n-2])).  */
+  logP[PSI_DF - 1] = log1mexp(logsumexp(logP, PSI_DF - 1));
+
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+
+    for (j = 0; j < NSAMPLES; j++) {
+      double n_tmp = dist_sample(&geometric.base);
+
+      /* Must be an integer.  (XXX -Wfloat-equal)  */
+      tor_assert(ceil(n_tmp) <= n_tmp && ceil(n_tmp) >= n_tmp);
+
+      /* Must be a positive integer.  */
+      tor_assert(n_tmp >= 1);
+
+      /* Probability of getting a value in the billions is negligible.  */
+      tor_assert(n_tmp <= (double)UINT_MAX);
+
+      unsigned n = (unsigned) n_tmp;
+
+      if (n > PSI_DF)
+        n = PSI_DF;
+      C[n - 1]++;
+    }
+
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", "geometric"); */
+    return true;
+  } else {
+    printf("fail %s sampler\n", "geometric");
+    return false;
+  }
+}
+
+/**
+ * Divide the support of <b>dist</b> into histogram bins in <b>logP</b>. Start
+ * at the 1st percentile and ending at the 99th percentile. Pick the bin
+ * boundaries using linear interpolation so that they are uniformly spaced.
+ *
+ * In each bin logP[i] we insert the expected log-probability that a sampled
+ * value will fall into that bin. We will use this as the null hypothesis of
+ * the psi test.
+ *
+ * Set logP[i] = log(CDF(x_i) - CDF(x_{i-1})), where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_cdfs(const struct dist_t *dist, double lo, double hi, double *logP,
+         size_t n)
+{
+#define CDF(x)  dist_cdf(dist, x)
+#define SF(x)   dist_sf(dist, x)
+  const double w = (hi - lo)/(n - 2);
+  double halfway = dist_icdf(dist, 0.5);
+  double x_0, x_1;
+  size_t i;
+  size_t n2 = ceil_to_size_t((halfway - lo)/w);
+
+  tor_assert(lo <= halfway);
+  tor_assert(halfway <= hi);
+  tor_assert(n2 <= n);
+
+  x_1 = lo;
+  logP[0] = log(CDF(x_1) - 0); /* 0 = CDF(-inf) */
+  for (i = 1; i < n2; i++) {
+    x_0 = x_1;
+    /* do the linear interpolation */
+    x_1 = (i <= n/2 ? lo + i*w : hi - (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[i] = log(CDF(x_1) - CDF(x_0));
+  }
+  x_0 = hi;
+  logP[n - 1] = log(SF(x_0) - 0); /* 0 = SF(+inf) = 1 - CDF(+inf) */
+
+  /* In this loop we are filling out the high part of the array. We are using
+   * SF because in these cases the CDF is near 1 where precision is lower. So
+   * instead we are using SF near 0 where the precision is higher. We have
+   * SF(t) = 1 - CDF(t).  */
+  for (i = 1; i < n - n2; i++) {
+    x_1 = x_0;
+    /* do the linear interpolation */
+    x_0 = (i <= n/2 ? hi - i*w : lo + (n - 2 - i)*w);
+    /* set the expected log-probability */
+    logP[n - i - 1] = log(SF(x_0) - SF(x_1));
+  }
+#undef SF
+#undef CDF
+}
+
+/**
+ * Draw NSAMPLES samples from dist, counting the number of samples x in
+ * the ith bin C[i] if x_{i-1} <= x < x_i, where x_-1 = -inf, x_n =
+ * +inf, and x_i = i*(hi - lo)/(n - 2).
+ */
+static void
+bin_samples(const struct dist_t *dist, double lo, double hi, size_t *C,
+            size_t n)
+{
+  const double w = (hi - lo)/(n - 2);
+  size_t i;
+
+  for (i = 0; i < NSAMPLES; i++) {
+    double x = dist_sample(dist);
+    size_t bin;
+
+    if (x < lo)
+      bin = 0;
+    else if (x < hi)
+      bin = 1 + floor_to_size_t((x - lo)/w);
+    else
+      bin = n - 1;
+    tor_assert(bin < n);
+    C[bin]++;
+  }
+}
+
+/**
+ * Carry out a Psi test on <b>dist</b>.
+ *
+ * Sample NSAMPLES from dist, putting them in bins from -inf to lo to
+ * hi to +inf, and apply up to two psi tests.  True if at least one psi
+ * test passes; false if not.  False positive rate should be bounded by
+ * 0.01^2 = 0.0001.
+ */
+static bool
+test_psi_dist_sample(const struct dist_t *dist)
+{
+  double logP[PSI_DF] = {0};
+  unsigned ntry = NTRIALS, npass = 0;
+  double lo = dist_icdf(dist, 1/(double)(PSI_DF + 2));
+  double hi = dist_isf(dist, 1/(double)(PSI_DF + 2));
+
+  /* Create the null hypothesis in logP */
+  bin_cdfs(dist, lo, hi, logP, PSI_DF);
+
+  /* Now run the test */
+  while (ntry --> 0) {
+    size_t C[PSI_DF] = {0};
+    bin_samples(dist, lo, hi, C, PSI_DF);
+    if (psi_test(C, logP, NSAMPLES)) {
+      if (++npass >= NPASSES_MIN)
+        break;
+    }
+  }
+
+  /* Did we fail or succeed? */
+  if (npass >= NPASSES_MIN) {
+    /* printf("pass %s sampler\n", dist_name(dist));*/
+    return true;
+  } else {
+    printf("fail %s sampler\n", dist_name(dist));
+    return false;
+  }
+}
+
+static void
+write_stochastic_warning(void)
+{
+  if (tinytest_cur_test_has_failed()) {
+    printf("\n"
+         "NOTE: This is a stochastic test, and we expect it to fail from\n"
+         "time to time, with some low probability. If you see it fail more\n"
+         "than one trial in 100, though, please tell us.\n\n");
+  }
+}
+
+static void
+test_stochastic_uniform(void *arg)
+{
+  (void) arg;
+
+  const struct uniform_t uniform01 = {
+    .base = UNIFORM(uniform01),
+    .a = 0,
+    .b = 1,
+  };
+  const struct uniform_t uniform_pos = {
+    .base = UNIFORM(uniform_pos),
+    .a = 1.23,
+    .b = 4.56,
+  };
+  const struct uniform_t uniform_neg = {
+    .base = UNIFORM(uniform_neg),
+    .a = -10,
+    .b = -1,
+  };
+  const struct uniform_t uniform_cross = {
+    .base = UNIFORM(uniform_cross),
+    .a = -1.23,
+    .b = 4.56,
+  };
+  const struct uniform_t uniform_subnormal = {
+    .base = UNIFORM(uniform_subnormal),
+    .a = 4e-324,
+    .b = 4e-310,
+  };
+  const struct uniform_t uniform_subnormal_cross = {
+    .base = UNIFORM(uniform_subnormal_cross),
+    .a = -4e-324,
+    .b = 4e-310,
+  };
+  bool ok = true, tests_failed = true;
+
+  testing_enable_reproducible_rng();
+
+  ok &= test_psi_dist_sample(&uniform01.base);
+  ok &= test_psi_dist_sample(&uniform_pos.base);
+  ok &= test_psi_dist_sample(&uniform_neg.base);
+  ok &= test_psi_dist_sample(&uniform_cross.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal.base);
+  ok &= test_psi_dist_sample(&uniform_subnormal_cross.base);
+
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static bool
+test_stochastic_logistic_impl(double mu, double sigma)
+{
+  const struct logistic_t dist = {
+    .base = LOGISTIC(dist),
+    .mu = mu,
+    .sigma = sigma,
+  };
+
+  /* XXX Consider some fancier logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_log_logistic_impl(double alpha, double beta)
+{
+  const struct log_logistic_t dist = {
+    .base = LOG_LOGISTIC(dist),
+    .alpha = alpha,
+    .beta = beta,
+  };
+
+  /* XXX Consider some fancier log logistic test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_weibull_impl(double lambda, double k)
+{
+  const struct weibull_t dist = {
+    .base = WEIBULL(dist),
+    .lambda = lambda,
+    .k = k,
+  };
+
+// clang-format off
+/*
+ * XXX Consider applying a Tiku-Singh test:
+ *
+ *    M.L. Tiku and M. Singh, `Testing the two-parameter
+ *    Weibull distribution', Communications in Statistics --
+ *    Theory and Methods A10(9), 1981, 907--918.
+https://www.tandfonline.com/doi/pdf/10.1080/03610928108828082?needAccess=true
+ */
+// clang-format on
+  return test_psi_dist_sample(&dist.base);
+}
+
+static bool
+test_stochastic_genpareto_impl(double mu, double sigma, double xi)
+{
+  const struct genpareto_t dist = {
+    .base = GENPARETO(dist),
+    .mu = mu,
+    .sigma = sigma,
+    .xi = xi,
+  };
+
+  /* XXX Consider some fancier GPD test.  */
+  return test_psi_dist_sample(&dist.base);
+}
+
+static void
+test_stochastic_genpareto(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_genpareto_impl(0, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, -1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 1e-30);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(0, 1, 0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(-1, 1, -0.25);
+  tt_assert(ok);
+  ok = test_stochastic_genpareto_impl(1, 2, 0.25);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_geometric(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_geometric_impl(0.1);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.5);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(0.9);
+  tt_assert(ok);
+  ok = test_stochastic_geometric_impl(1);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_logistic(void *arg)
+{
+  bool ok = 0;
+  bool tests_failed = true;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_logistic_impl(0, 1);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(0, 1e-16);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_logistic_impl(-10, 100);
+  tt_assert(ok);
+
+  tests_failed = false;
+
+ done:
+  if (tests_failed) {
+    write_stochastic_warning();
+  }
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_log_logistic(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_log_logistic_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(1, 10);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(M_E, 1e-1);
+  tt_assert(ok);
+  ok = test_stochastic_log_logistic_impl(exp(-10), 1e-2);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+}
+
+static void
+test_stochastic_weibull(void *arg)
+{
+  bool ok = 0;
+  (void) arg;
+
+  testing_enable_reproducible_rng();
+
+  ok = test_stochastic_weibull_impl(1, 0.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 1.5);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(1, 2);
+  tt_assert(ok);
+  ok = test_stochastic_weibull_impl(10, 1);
+  tt_assert(ok);
+
+ done:
+  write_stochastic_warning();
+  testing_disable_reproducible_rng();
+  UNMOCK(crypto_rand);
+}
+
+struct testcase_t prob_distr_tests[] = {
+  { "logit_logistics", test_logit_logistic, TT_FORK, NULL, NULL },
+  { "log_logistic", test_log_logistic, TT_FORK, NULL, NULL },
+  { "weibull", test_weibull, TT_FORK, NULL, NULL },
+  { "genpareto", test_genpareto, TT_FORK, NULL, NULL },
+  { "uniform_interval", test_uniform_interval, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
+
+struct testcase_t slow_stochastic_prob_distr_tests[] = {
+  { "stochastic_genpareto", test_stochastic_genpareto, TT_FORK, NULL, NULL },
+  { "stochastic_geometric", test_stochastic_geometric, TT_FORK, NULL, NULL },
+  { "stochastic_uniform", test_stochastic_uniform, TT_FORK, NULL, NULL },
+  { "stochastic_logistic", test_stochastic_logistic, TT_FORK, NULL, NULL },
+  { "stochastic_log_logistic", test_stochastic_log_logistic, TT_FORK, NULL,
+    NULL },
+  { "stochastic_weibull", test_stochastic_weibull, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process.c b/src/test/test_process.c
new file mode 100644
index 0000000000..c1da6db278
--- /dev/null
+++ b/src/test/test_process.c
@@ -0,0 +1,669 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process.c
+ * \brief Test cases for the Process API.
+ */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "lib/process/env.h"
+
+#define PROCESS_PRIVATE
+#include "lib/process/process.h"
+#define PROCESS_UNIX_PRIVATE
+#include "lib/process/process_unix.h"
+#define PROCESS_WIN32_PRIVATE
+#include "lib/process/process_win32.h"
+
+static const char *stdout_read_buffer;
+static const char *stderr_read_buffer;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static int
+process_mocked_read_stdout(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stdout_read_buffer != NULL) {
+    buf_add_string(buffer, stdout_read_buffer);
+    stdout_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static int
+process_mocked_read_stderr(process_t *process, buf_t *buffer)
+{
+  (void)process;
+
+  if (stderr_read_buffer != NULL) {
+    buf_add_string(buffer, stderr_read_buffer);
+    stderr_read_buffer = NULL;
+  }
+
+  return (int)buf_datalen(buffer);
+}
+
+static void
+process_mocked_write_stdin(process_t *process, buf_t *buffer)
+{
+  const size_t size = buf_datalen(buffer);
+
+  if (size == 0)
+    return;
+
+  char *data = tor_malloc_zero(size + 1);
+  process_data_t *process_data = process_get_data(process);
+
+  buf_get_bytes(buffer, data, size);
+  smartlist_add(process_data->stdin_data, data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+static void
+test_default_values(void *arg)
+{
+  (void)arg;
+  process_t *process = process_new("/path/to/nothing");
+
+  /* We are not running by default. */
+  tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process));
+
+  /* We use the line protocol by default. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  /* We don't set any custom data by default. */
+  tt_ptr_op(NULL, OP_EQ, process_get_data(process));
+
+  /* Our command was given to the process_t's constructor in process_new(). */
+  tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process));
+
+  /* Make sure we are listed in the list of proccesses. */
+  tt_assert(smartlist_contains(process_get_all_processes(),
+                               process));
+
+  /* Default PID is 0. */
+  tt_u64_op(0, OP_EQ, process_get_pid(process));
+
+  /* Our arguments should be empty. */
+  tt_int_op(0, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+ done:
+  process_free(process);
+}
+
+static void
+test_environment(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("");
+  process_environment_t *env = NULL;
+
+  process_set_environment(process, "E", "F");
+  process_set_environment(process, "C", "D");
+  process_set_environment(process, "A", "B");
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "A=B\0C=D\0E=F\0", 12);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "A=B");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "C=D");
+  tt_str_op(env->unixoid_environment_block[2], OP_EQ,
+            "E=F");
+  tt_ptr_op(env->unixoid_environment_block[3], OP_EQ,
+            NULL);
+  process_environment_free(env);
+
+  /* Reset our environment. */
+  smartlist_t *new_env = smartlist_new();
+  smartlist_add(new_env, (char *)"FOO=bar");
+  smartlist_add(new_env, (char *)"HELLO=world");
+
+  process_reset_environment(process, new_env);
+  smartlist_free(new_env);
+
+  env = process_get_environment(process);
+  tt_mem_op(env->windows_environment_block, OP_EQ,
+            "FOO=bar\0HELLO=world\0", 20);
+  tt_str_op(env->unixoid_environment_block[0], OP_EQ,
+            "FOO=bar");
+  tt_str_op(env->unixoid_environment_block[1], OP_EQ,
+            "HELLO=world");
+  tt_ptr_op(env->unixoid_environment_block[2], OP_EQ,
+            NULL);
+
+ done:
+  process_environment_free(env);
+  process_free(process);
+}
+
+static void
+test_stringified_types(void *arg)
+{
+  (void)arg;
+
+  /* process_protocol_t values. */
+  tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW));
+  tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE));
+
+  /* process_status_t values. */
+  tt_str_op("not running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_NOT_RUNNING));
+  tt_str_op("running", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_RUNNING));
+  tt_str_op("error", OP_EQ,
+            process_status_to_string(PROCESS_STATUS_ERROR));
+
+ done:
+  return;
+}
+
+static void
+test_line_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\r\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_multi(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Onion Onion Onion");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "A B C D");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Foo bar baz");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Onion Onion Onion");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_line_protocol_partial(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the line protocol. */
+  tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout this is a partial line ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr this is a partial line ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should NOT be ready. */
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nAnother partial string goes here ...";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = " the end\nFoo bar baz\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Some data should be ready. */
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+                          "Hello stdout this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+                          "Hello stderr this is a partial line ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+                          "Another partial string goes here ... the end");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+                          "Foo bar baz");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_raw_protocol_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_protocol(process, PROCESS_PROTOCOL_RAW);
+
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+
+  MOCK(process_read_stdout, process_mocked_read_stdout);
+  MOCK(process_read_stderr, process_mocked_read_stderr);
+
+  /* Make sure we are running with the raw protocol. */
+  tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process));
+
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello stdout\n";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello stderr\n";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines";
+  process_notify_event_stdout(process);
+  tt_ptr_op(NULL, OP_EQ, stdout_read_buffer);
+
+  stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines";
+  process_notify_event_stderr(process);
+  tt_ptr_op(NULL, OP_EQ, stderr_read_buffer);
+
+  /* Data should be ready. */
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data));
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data));
+
+  /* Check if the data is correct. */
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            "Hello stdout\n");
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "Hello, again, stdout\nThis contains multiple lines");
+
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Hello stderr\n");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "Hello, again, stderr\nThis contains multiple lines");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_read_stdout);
+  UNMOCK(process_read_stderr);
+}
+
+static void
+test_write_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+
+  MOCK(process_write_stdin, process_mocked_write_stdin);
+
+  process_write(process, (uint8_t *)"Hello world\n", 12);
+  process_notify_event_stdin(process);
+  tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data));
+
+  process_printf(process, "Hello %s !\n", "moon");
+  process_notify_event_stdin(process);
+  tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data));
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+
+  UNMOCK(process_write_stdin);
+}
+
+static void
+test_exit_simple(void *arg)
+{
+  (void)arg;
+
+  process_data_t *process_data = process_data_new();
+
+  process_t *process = process_new("");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Our default is 0. */
+  tt_u64_op(0, OP_EQ, process_data->exit_code);
+
+  /* Fake that we are a running process. */
+  process_set_status(process, PROCESS_STATUS_RUNNING);
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Fake an exit. */
+  process_notify_event_exit(process, 1337);
+
+  /* Check if our state changed and if our callback fired. */
+  tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+  tt_u64_op(1337, OP_EQ, process_data->exit_code);
+
+ done:
+  process_set_data(process, process_data);
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_argv_simple(void *arg)
+{
+  (void)arg;
+
+  process_t *process = process_new("/bin/cat");
+  char **argv = NULL;
+
+  /* Setup some arguments. */
+  process_append_argument(process, "foo");
+  process_append_argument(process, "bar");
+  process_append_argument(process, "baz");
+
+  /* Check the number of elements. */
+  tt_int_op(3, OP_EQ,
+            smartlist_len(process_get_arguments(process)));
+
+  /* Let's try to convert it into a Unix style char **argv. */
+  argv = process_get_argv(process);
+
+  /* Check our values. */
+  tt_str_op(argv[0], OP_EQ, "/bin/cat");
+  tt_str_op(argv[1], OP_EQ, "foo");
+  tt_str_op(argv[2], OP_EQ, "bar");
+  tt_str_op(argv[3], OP_EQ, "baz");
+  tt_ptr_op(argv[4], OP_EQ, NULL);
+
+ done:
+  tor_free(argv);
+  process_free(process);
+}
+
+static void
+test_unix(void *arg)
+{
+  (void)arg;
+#ifndef _WIN32
+  process_t *process = process_new("");
+
+  /* On Unix all processes should have a Unix process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_unix_process(process));
+
+ done:
+  process_free(process);
+#endif /* !defined(_WIN32) */
+}
+
+static void
+test_win32(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  process_t *process = process_new("");
+  char *joined_argv = NULL;
+
+  /* On Win32 all processes should have a Win32 process handle. */
+  tt_ptr_op(NULL, OP_NE, process_get_win32_process(process));
+
+  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
+   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
+   * will try to only generate simple cases for the child process to parse;
+   * i.e. we never embed quoted strings in arguments. */
+
+  const char *argvs[][4] = {
+    {"a", "bb", "CCC", NULL}, // Normal
+    {NULL, NULL, NULL, NULL}, // Empty argument list
+    {"", NULL, NULL, NULL}, // Empty argument
+    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
+    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
+    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
+    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
+    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
+    { NULL } // Terminator
+  };
+
+  const char *cmdlines[] = {
+    "a bb CCC",
+    "",
+    "\"\"",
+    "\\\"a b\\\"b CCC\\\"",
+    "\"a\tbc\" \"dd  dd\" E",
+    "a\\\\\\b \"de fg\" H",
+    "a\\\\\\\"b \\c D\\",
+    "\"a\\\\b c\" d E",
+    NULL // Terminator
+  };
+
+  int i;
+
+  for (i=0; cmdlines[i]!=NULL; i++) {
+    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
+    joined_argv = tor_join_win_cmdline(argvs[i]);
+    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
+    tor_free(joined_argv);
+  }
+
+ done:
+  tor_free(joined_argv);
+  process_free(process);
+#endif /* defined(_WIN32) */
+}
+
+struct testcase_t process_tests[] = {
+  { "default_values", test_default_values, TT_FORK, NULL, NULL },
+  { "environment", test_environment, TT_FORK, NULL, NULL },
+  { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL },
+  { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL },
+  { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL },
+  { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL },
+  { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL },
+  { "write_simple", test_write_simple, TT_FORK, NULL, NULL },
+  { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL },
+  { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL },
+  { "unix", test_unix, TT_FORK, NULL, NULL },
+  { "win32", test_win32, TT_FORK, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_descs.c b/src/test/test_process_descs.c
new file mode 100644
index 0000000000..14865cff13
--- /dev/null
+++ b/src/test/test_process_descs.c
@@ -0,0 +1,70 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#include "core/or/or.h"
+#include "feature/dirauth/process_descs.h"
+
+#include "test/test.h"
+
+static void
+test_process_descs_versions(void *arg)
+{
+  (void)arg;
+  struct {
+    const char *version;
+    bool should_reject;
+  } cases[] = {
+    // a very old version: reject.
+    { "Tor 0.1.2.3-alpha", true },
+    // a non-tor program: don't reject.
+    { "Wombat 0.1.2.3-alpha", false },
+    // some unsupported versions: reject.
+    { "Tor 0.2.9.4-alpha", true },
+    { "Tor 0.2.9.5-alpha", true },
+    { "Tor 0.2.9.100", true },
+    { "Tor 0.3.0.0-alpha-dev", true },
+    { "Tor 0.3.0.2-alpha", true },
+    { "Tor 0.3.0.5", true },
+    { "Tor 0.3.1.4", true },
+    { "Tor 0.3.2.4", true },
+    { "Tor 0.3.3.4", true },
+    { "Tor 0.3.4.1-alpha", true },
+    { "Tor 0.3.4.100", true },
+    { "Tor 0.3.5.1-alpha", true },
+    { "Tor 0.3.5.6-rc", true},
+    { "Tor 0.4.0.1-alpha", true },
+    { "Tor 0.4.0.5", true },
+    { "Tor 0.4.1.1-alpha", true },
+    { "Tor 0.4.1.4-rc", true },
+    // new enough to be supported
+    { "Tor 0.3.5.7", false },
+    { "Tor 0.3.5.8", false },
+    { "Tor 0.4.1.5", false },
+    { "Tor 0.4.2.1-alpha", false },
+    { "Tor 0.4.2.4-rc", false },
+    { "Tor 0.4.3.0-alpha-dev", false },
+    // Very far in the future
+    { "Tor 100.100.1.5", false },
+  };
+  size_t n_cases = ARRAY_LENGTH(cases);
+
+  for (unsigned i = 0; i < n_cases; ++i) {
+    const char *msg = NULL;
+    bool rejected = dirserv_rejects_tor_version(cases[i].version, &msg);
+    tt_int_op(rejected, OP_EQ, cases[i].should_reject);
+    tt_int_op(msg == NULL, OP_EQ, rejected == false);
+  }
+
+ done:
+  ;
+}
+
+#define T(name,flags)                                   \
+  { #name, test_process_descs_##name, (flags), NULL, NULL }
+
+struct testcase_t process_descs_tests[] = {
+  T(versions,0),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c
new file mode 100644
index 0000000000..f74d4adc9a
--- /dev/null
+++ b/src/test/test_process_slow.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_process_slow.c
+ * \brief Slow test cases for the Process API.
+ */
+
+#define MAINLOOP_PRIVATE
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/evloop/compat_libevent.h"
+#include "lib/process/process.h"
+#include "lib/process/waitpid.h"
+#include "test/test.h"
+
+#ifndef BUILDDIR
+#define BUILDDIR "."
+#endif
+
+#ifdef _WIN32
+#define TEST_PROCESS "test-process.exe"
+#else
+#define TEST_PROCESS BUILDDIR "/src/test/test-process"
+#endif /* defined(_WIN32) */
+
+/** Timer that ticks once a second and stop the event loop after 5 ticks. */
+static periodic_timer_t *main_loop_timeout_timer;
+
+/** How many times have our timer ticked? */
+static int timer_tick_count;
+
+struct process_data_t {
+  smartlist_t *stdout_data;
+  smartlist_t *stderr_data;
+  smartlist_t *stdin_data;
+  process_exit_code_t exit_code;
+  bool did_exit;
+};
+
+typedef struct process_data_t process_data_t;
+
+static process_data_t *
+process_data_new(void)
+{
+  process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t));
+  process_data->stdout_data = smartlist_new();
+  process_data->stderr_data = smartlist_new();
+  process_data->stdin_data = smartlist_new();
+  return process_data;
+}
+
+static void
+process_data_free(process_data_t *process_data)
+{
+  if (process_data == NULL)
+    return;
+
+  SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x));
+  SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x));
+
+  smartlist_free(process_data->stdout_data);
+  smartlist_free(process_data->stderr_data);
+  smartlist_free(process_data->stdin_data);
+  tor_free(process_data);
+}
+
+static void
+process_stdout_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stdout_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static void
+process_stderr_callback(process_t *process, const char *data, size_t size)
+{
+  tt_ptr_op(process, OP_NE, NULL);
+  tt_ptr_op(data, OP_NE, NULL);
+  tt_int_op(strlen(data), OP_EQ, size);
+
+  process_data_t *process_data = process_get_data(process);
+  smartlist_add(process_data->stderr_data, tor_strdup(data));
+
+ done:
+  return;
+}
+
+static bool
+process_exit_callback(process_t *process, process_exit_code_t exit_code)
+{
+  process_status_t status;
+
+  tt_ptr_op(process, OP_NE, NULL);
+
+  process_data_t *process_data = process_get_data(process);
+  process_data->exit_code = exit_code;
+  process_data->did_exit = true;
+
+  /* Check if our process is still running? */
+  status = process_get_status(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING);
+
+ done:
+  /* Do not free up our process_t. */
+  return false;
+}
+
+#ifdef _WIN32
+static const char *
+get_win32_test_binary_path(void)
+{
+  static char buffer[MAX_PATH];
+
+  /* Get the absolute path of our binary: \path\to\test-slow.exe. */
+  GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer));
+
+  /* Find our process name. */
+  char *offset = strstr(buffer, "test-slow.exe");
+  tt_ptr_op(offset, OP_NE, NULL);
+
+  /* Change test-slow.exe to test-process.exe. */
+  memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS));
+
+  return buffer;
+ done:
+  return NULL;
+}
+#endif /* defined(_WIN32) */
+
+static void
+main_loop_timeout_cb(periodic_timer_t *timer, void *data)
+{
+  /* Sanity check. */
+  tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer);
+  tt_ptr_op(data, OP_NE, NULL);
+
+  /* Our process data. */
+  process_data_t *process_data = data;
+
+  /* Our process did exit. */
+  if (process_data->did_exit)
+    tor_shutdown_event_loop_and_exit(0);
+
+  /* Have we been called 10 times we exit the main loop. */
+  timer_tick_count++;
+
+  tt_int_op(timer_tick_count, OP_LT, 10);
+
+#ifndef _WIN32
+  /* Call waitpid callbacks. */
+  notify_pending_waitpid_callbacks();
+#endif
+
+  return;
+ done:
+  /* Exit with an error. */
+  tor_shutdown_event_loop_and_exit(-1);
+}
+
+static void
+run_main_loop(process_data_t *process_data)
+{
+  int ret;
+
+  /* Wake up after 1 seconds. */
+  static const struct timeval interval = {1, 0};
+
+  timer_tick_count = 0;
+  main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(),
+                                               &interval,
+                                               main_loop_timeout_cb,
+                                               process_data);
+
+  /* Run our main loop. */
+  ret = run_main_loop_until_done();
+
+  /* Clean up our main loop timeout timer. */
+  tt_int_op(ret, OP_EQ, 0);
+
+ done:
+  periodic_timer_free(main_loop_timeout_timer);
+}
+
+static void
+test_callbacks(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_stdout_read_callback(process, process_stdout_callback);
+  process_set_stderr_read_callback(process, process_stderr_callback);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Set environment variable. */
+  process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!");
+
+  /* Add some arguments. */
+  process_append_argument(process, "This is the first one");
+  process_append_argument(process, "Second one");
+  process_append_argument(process, "Third: Foo bar baz");
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Write some lines to stdin. */
+  process_printf(process, "Hi process!\r\n");
+  process_printf(process, "Can you read more than one line?\n");
+  process_printf(process, "Can you read partial ...");
+  process_printf(process, " lines?\r\n");
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* We returned. Let's see what our event loop said. */
+  tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12);
+  tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3);
+  tt_assert(process_data->did_exit);
+  tt_u64_op(process_data->exit_code, OP_EQ, 0);
+
+  /* Check stdout output. */
+  char argv0_expected[256];
+  tor_snprintf(argv0_expected, sizeof(argv0_expected),
+               "argv[0] = '%s'", filename);
+
+  tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ,
+            argv0_expected);
+  tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ,
+            "argv[1] = 'This is the first one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ,
+            "argv[2] = 'Second one'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ,
+            "argv[3] = 'Third: Foo bar baz'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ,
+            "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ,
+            "Output on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ,
+            "Partial line on stdout ...end of partial line on stdout");
+  tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ,
+            "Read line from stdin: 'Hi process!'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ,
+            "Read line from stdin: 'Can you read more than one line?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ,
+            "Read line from stdin: 'Can you read partial ... lines?'");
+  tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ,
+            "We are done for here, thank you!");
+
+  /* Check stderr output. */
+  tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ,
+            "Output on stderr");
+  tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ,
+            "This is a new line");
+  tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ,
+            "Partial line on stderr ...end of partial line on stderr");
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_callbacks_terminate(void *arg)
+{
+  (void)arg;
+  const char *filename = NULL;
+
+#ifdef _WIN32
+  filename = get_win32_test_binary_path();
+#else
+  filename = TEST_PROCESS;
+#endif
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new(filename);
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_status_t status;
+
+  status = process_exec(process);
+  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
+
+  /* Zap our process. */
+  bool success;
+
+  success = process_terminate(process);
+  tt_assert(success);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Check if we did exit. */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+static void
+test_nonexistent_executable(void *arg)
+{
+  (void)arg;
+
+  /* Process callback data. */
+  process_data_t *process_data = process_data_new();
+
+  /* Setup our process. */
+  process_t *process = process_new("binary-does-not-exist");
+  process_set_data(process, process_data);
+  process_set_exit_callback(process, process_exit_callback);
+
+  /* Run our process. */
+  process_exec(process);
+
+  /* Start our main loop. */
+  run_main_loop(process_data);
+
+  /* Ensure that the exit callback was actually called even though the binary
+   * did not exist.
+   */
+  tt_assert(process_data->did_exit);
+
+ done:
+  process_data_free(process_data);
+  process_free(process);
+}
+
+struct testcase_t slow_process_tests[] = {
+  { "callbacks", test_callbacks, 0, NULL, NULL },
+  { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL },
+  { "nonexistent_executable", test_nonexistent_executable, 0, NULL, NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_procmon.c b/src/test/test_procmon.c
index e23578f4fd..1752008f63 100644
--- a/src/test/test_procmon.c
+++ b/src/test/test_procmon.c
@@ -1,7 +1,6 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define PROCMON_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "test/test.h"
@@ -10,8 +9,6 @@
 
 #include "test/log_test_helpers.h"
 
-#define NS_MODULE procmon
-
 struct event_base;
 
 static void
diff --git a/src/test/test_proto_haproxy.c b/src/test/test_proto_haproxy.c
new file mode 100644
index 0000000000..040354ec1f
--- /dev/null
+++ b/src/test/test_proto_haproxy.c
@@ -0,0 +1,66 @@
+/* Copyright (c) 2019-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_proto_haproxy.c
+ * \brief Tests for our HAProxy protocol parser code
+ */
+
+#define PROTO_HAPROXY_PRIVATE
+
+#include "test/test.h"
+#include "core/proto/proto_haproxy.h"
+#include "test/log_test_helpers.h"
+
+static void
+test_format_proxy_header_line(void *arg)
+{
+  tor_addr_t addr;
+  tor_addr_port_t *addr_port = NULL;
+  char *output = NULL;
+
+  (void) arg;
+
+  /* IPv4 address. */
+  tor_addr_parse(&addr, "192.168.1.2");
+  addr_port = tor_addr_port_new(&addr, 8000);
+  output = haproxy_format_proxy_header_line(addr_port);
+
+  tt_str_op(output, OP_EQ, "PROXY TCP4 0.0.0.0 192.168.1.2 0 8000\r\n");
+
+  tor_free(addr_port);
+  tor_free(output);
+
+  /* IPv6 address. */
+  tor_addr_parse(&addr, "123:45:6789::5005:11");
+  addr_port = tor_addr_port_new(&addr, 8000);
+  output = haproxy_format_proxy_header_line(addr_port);
+
+  tt_str_op(output, OP_EQ, "PROXY TCP6 :: 123:45:6789::5005:11 0 8000\r\n");
+
+  tor_free(addr_port);
+  tor_free(output);
+
+  /* UNIX socket address. */
+  memset(&addr, 0, sizeof(addr));
+  addr.family = AF_UNIX;
+  addr_port = tor_addr_port_new(&addr, 8000);
+  output = haproxy_format_proxy_header_line(addr_port);
+
+  /* If it's not an IPv4 or IPv6 address, haproxy_format_proxy_header_line
+   * must return NULL. */
+  tt_ptr_op(output, OP_EQ, NULL);
+
+  tor_free(addr_port);
+  tor_free(output);
+
+ done:
+  tor_free(addr_port);
+  tor_free(output);
+}
+
+struct testcase_t proto_haproxy_tests[] = {
+  { "format_proxy_header_line", test_format_proxy_header_line, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_proto_http.c b/src/test/test_proto_http.c
index 08990c0b6a..481d78b2c1 100644
--- a/src/test/test_proto_http.c
+++ b/src/test/test_proto_http.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 #include "test/log_test_helpers.h"
 
diff --git a/src/test/test_proto_misc.c b/src/test/test_proto_misc.c
index af9cf7eee2..64bf5c4993 100644
--- a/src/test/test_proto_misc.c
+++ b/src/test/test_proto_misc.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_cell.h"
diff --git a/src/test/test_protover.c b/src/test/test_protover.c
index b4689045cf..71f984a0ac 100644
--- a/src/test/test_protover.c
+++ b/src/test/test_protover.c
@@ -1,7 +1,8 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define PROTOVER_PRIVATE
+#define DIRVOTE_PRIVATE
 
 #include "orconfig.h"
 #include "test/test.h"
@@ -12,6 +13,8 @@
 #include "core/or/connection_or.h"
 #include "lib/tls/tortls.h"
 
+#include "feature/dirauth/dirvote.h"
+
 static void
 test_protover_parse(void *arg)
 {
@@ -22,7 +25,7 @@ test_protover_parse(void *arg)
   tt_skip();
  done:
   ;
-#else
+#else /* !defined(HAVE_RUST) */
   char *re_encoded = NULL;
 
   const char *orig = "Foo=1,3 Bar=3 Baz= Quux=9-12,14,15-16";
@@ -85,7 +88,7 @@ test_protover_parse(void *arg)
     SMARTLIST_FOREACH(elts, proto_entry_t *, ent, proto_entry_free(ent));
   smartlist_free(elts);
   tor_free(re_encoded);
-#endif
+#endif /* defined(HAVE_RUST) */
 }
 
 static void
@@ -129,7 +132,7 @@ test_protover_parse_fail(void *arg)
                            "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
   tt_ptr_op(elts, OP_EQ, NULL);
 
-#endif
+#endif /* defined(HAVE_RUST) */
  done:
   ;
 }
@@ -271,6 +274,7 @@ test_protover_all_supported(void *arg)
   tt_assert(protover_all_supported("Fribble=", &msg));
   tt_ptr_op(msg, OP_EQ, NULL);
 
+#ifndef ALL_BUGS_ARE_FATAL
   /* If we get a completely unparseable list, protover_all_supported should
    * hit a fatal assertion for BUG(entries == NULL). */
   tor_capture_bugs_(1);
@@ -282,9 +286,10 @@ test_protover_all_supported(void *arg)
   tor_capture_bugs_(1);
   tt_assert(protover_all_supported("Sleen=1-4294967295", &msg));
   tor_end_capture_bugs_();
+#endif /* !defined(ALL_BUGS_ARE_FATAL) */
 
   /* Protocol name too long */
-#ifndef HAVE_RUST // XXXXXX ?????
+#if !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL)
   tor_capture_bugs_(1);
   tt_assert(protover_all_supported(
                  "DoSaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
@@ -292,7 +297,7 @@ test_protover_all_supported(void *arg)
                  "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
                  "aaaaaaaaaaaa=1-65536", &msg));
   tor_end_capture_bugs_();
-#endif
+#endif /* !defined(HAVE_RUST) && !defined(ALL_BUGS_ARE_FATAL) */
 
  done:
   tor_end_capture_bugs_();
@@ -416,7 +421,7 @@ test_protover_supported_protocols(void *arg)
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
                                             PROTOVER_LINKAUTH_V1));
-#endif
+#endif /* defined(HAVE_WORKING_TOR_TLS_GET_TLSSECRETS) */
   /* Latest LinkAuth is not exposed in the headers. */
   tt_assert(protocol_list_supports_protocol(supported_protocols,
                                             PRT_LINKAUTH,
@@ -587,6 +592,43 @@ test_protover_vote_roundtrip(void *args)
   tor_free(result);
 }
 
+static void
+test_protover_vote_roundtrip_ours(void *args)
+{
+  (void) args;
+  const char *examples[] = {
+    protover_get_supported_protocols(),
+    DIRVOTE_RECOMMEND_RELAY_PROTO,
+    DIRVOTE_RECOMMEND_CLIENT_PROTO,
+    DIRVOTE_REQUIRE_RELAY_PROTO,
+    DIRVOTE_REQUIRE_CLIENT_PROTO,
+  };
+  unsigned u;
+  smartlist_t *votes = smartlist_new();
+  char *result = NULL;
+
+  for (u = 0; u < ARRAY_LENGTH(examples); ++u) {
+    tt_assert(examples[u]);
+    const char *input = examples[u];
+    const char *expected_output = examples[u];
+
+    smartlist_add(votes, (void*)input);
+    result = protover_compute_vote(votes, 1);
+    if (expected_output != NULL) {
+      tt_str_op(result, OP_EQ, expected_output);
+    } else {
+      tt_str_op(result, OP_EQ, "");
+    }
+
+    smartlist_clear(votes);
+    tor_free(result);
+  }
+
+ done:
+  smartlist_free(votes);
+  tor_free(result);
+}
+
 #define PV_TEST(name, flags)                       \
   { #name, test_protover_ ##name, (flags), NULL, NULL }
 
@@ -600,5 +642,6 @@ struct testcase_t protover_tests[] = {
   PV_TEST(supports_version, 0),
   PV_TEST(supported_protocols, 0),
   PV_TEST(vote_roundtrip, 0),
+  PV_TEST(vote_roundtrip_ours, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_pt.c b/src/test/test_pt.c
index 1f9786648a..893fec3674 100644
--- a/src/test/test_pt.c
+++ b/src/test/test_pt.c
@@ -1,28 +1,30 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 #define PT_PRIVATE
-#define UTIL_PRIVATE
 #define STATEFILE_PRIVATE
-#define CONTROL_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define CONTROL_EVENTS_PRIVATE
+#define PROCESS_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
-#include "app/config/confparse.h"
+#include "lib/confmgt/confmgt.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "feature/client/transports.h"
 #include "core/or/circuitbuild.h"
 #include "app/config/statefile.h"
 #include "test/test.h"
-#include "lib/process/subprocess.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/resolve.h"
+#include "lib/process/process.h"
 
 #include "app/config/or_state_st.h"
 
+#include "test/log_test_helpers.h"
+
 static void
 reset_mp(managed_proxy_t *mp)
 {
@@ -288,41 +290,35 @@ test_pt_get_extrainfo_string(void *arg)
   tor_free(s);
 }
 
-#ifdef _WIN32
-#define STDIN_HANDLE HANDLE*
-#else
-#define STDIN_HANDLE int
-#endif
-
-static smartlist_t *
-tor_get_lines_from_handle_replacement(STDIN_HANDLE handle,
-                                      enum stream_status *stream_status_out)
+static int
+process_read_stdout_replacement(process_t *process, buf_t *buffer)
 {
+  (void)process;
   static int times_called = 0;
-  smartlist_t *retval_sl = smartlist_new();
-
-  (void) handle;
-  (void) stream_status_out;
 
   /* Generate some dummy CMETHOD lines the first 5 times. The 6th
      time, send 'CMETHODS DONE' to finish configuring the proxy. */
-  if (times_called++ != 5) {
-    smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d",
+  times_called++;
+
+  if (times_called <= 5) {
+    buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n",
                            times_called, times_called);
-  } else {
-    smartlist_add_strdup(retval_sl, "SMETHODS DONE");
+  } else if (times_called <= 6) {
+    buf_add_string(buffer, "SMETHODS DONE\n");
+  } else if (times_called <= 7) {
+    buf_add_string(buffer, "LOG SEVERITY=error MESSAGE=\"Oh noes, something "
+                           "bad happened. What do we do!?\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=warning MESSAGE=\"warning msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=notice MESSAGE=\"notice msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=info MESSAGE=\"info msg\"\n");
+    buf_add_string(buffer, "LOG SEVERITY=debug MESSAGE=\"debug msg\"\n");
+  } else if (times_called <= 8) {
+    buf_add_string(buffer, "STATUS TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\n");
+    buf_add_string(buffer, "STATUS TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\n");
   }
 
-  return retval_sl;
-}
-
-/* NOP mock */
-static void
-tor_process_handle_destroy_replacement(process_handle_t *process_handle,
-                                       int also_terminate_process)
-{
-  (void) process_handle;
-  (void) also_terminate_process;
+  return (int)buf_datalen(buffer);
 }
 
 static or_state_t *dummy_state = NULL;
@@ -355,12 +351,9 @@ test_pt_configure_proxy(void *arg)
   managed_proxy_t *mp = NULL;
   (void) arg;
 
-  dummy_state = tor_malloc_zero(sizeof(or_state_t));
+  dummy_state = or_state_new();
 
-  MOCK(tor_get_lines_from_handle,
-       tor_get_lines_from_handle_replacement);
-  MOCK(tor_process_handle_destroy,
-       tor_process_handle_destroy_replacement);
+  MOCK(process_read_stdout, process_read_stdout_replacement);
   MOCK(get_or_state,
        get_or_state_replacement);
   MOCK(queue_control_event_string,
@@ -372,24 +365,34 @@ test_pt_configure_proxy(void *arg)
   mp->conf_state = PT_PROTO_ACCEPTING_METHODS;
   mp->transports = smartlist_new();
   mp->transports_to_launch = smartlist_new();
-  mp->process_handle = tor_malloc_zero(sizeof(process_handle_t));
   mp->argv = tor_malloc_zero(sizeof(char*)*2);
   mp->argv[0] = tor_strdup("<testcase>");
   mp->is_server = 1;
 
+  /* Configure the process. */
+  mp->process = process_new("");
+  process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback);
+  process_set_data(mp->process, mp);
+
   /* Test the return value of configure_proxy() by calling it some
      times while it is uninitialized and then finally finalizing its
      configuration. */
   for (i = 0 ; i < 5 ; i++) {
+    /* force a read from our mocked stdout reader. */
+    process_notify_event_stdout(mp->process);
+    /* try to configure our proxy. */
     retval = configure_proxy(mp);
     /* retval should be zero because proxy hasn't finished configuring yet */
     tt_int_op(retval, OP_EQ, 0);
     /* check the number of registered transports */
-    tt_assert(smartlist_len(mp->transports) == i+1);
+    tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1);
     /* check that the mp is still waiting for transports */
     tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS);
   }
 
+  /* Get the SMETHOD DONE written to the process. */
+  process_notify_event_stdout(mp->process);
+
   /* this last configure_proxy() should finalize the proxy configuration. */
   retval = configure_proxy(mp);
   /* retval should be 1 since the proxy finished configuring */
@@ -412,6 +415,49 @@ test_pt_configure_proxy(void *arg)
   tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ,
             "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n");
 
+  /* Get the log message out. */
+  setup_full_capture_of_logs(LOG_ERR);
+  process_notify_event_stdout(mp->process);
+  expect_single_log_msg_containing("Oh noes, something bad happened");
+  teardown_capture_of_logs();
+
+  tt_int_op(controlevent_n, OP_EQ, 10);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 10);
+  tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=error "
+            "MESSAGE=\"Oh noes, "
+            "something bad happened. What do we do!?\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 6), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=warning "
+            "MESSAGE=\"warning msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 7), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=notice "
+            "MESSAGE=\"notice msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 8), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=info "
+            "MESSAGE=\"info msg\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 9), OP_EQ,
+            "650 PT_LOG PT=<testcase> SEVERITY=debug "
+            "MESSAGE=\"debug msg\"\r\n");
+
+  /* Get the STATUS messages out. */
+  process_notify_event_stdout(mp->process);
+
+  tt_int_op(controlevent_n, OP_EQ, 13);
+  tt_int_op(controlevent_event, OP_EQ, EVENT_PT_STATUS);
+  tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 13);
+
+  tt_str_op(smartlist_get(controlevent_msgs, 10), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=a K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 11), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=b K_1=a K_2=b K_3=\"foo bar\"\r\n");
+  tt_str_op(smartlist_get(controlevent_msgs, 12), OP_EQ,
+            "650 PT_STATUS "
+            "PT=<testcase> TRANSPORT=c K_1=a K_2=b K_3=\"foo bar\"\r\n");
+
   { /* check that the transport info were saved properly in the tor state */
     config_line_t *transport_in_state = NULL;
     smartlist_t *transport_info_sl = smartlist_new();
@@ -434,9 +480,9 @@ test_pt_configure_proxy(void *arg)
   }
 
  done:
+  teardown_capture_of_logs();
   or_state_free(dummy_state);
-  UNMOCK(tor_get_lines_from_handle);
-  UNMOCK(tor_process_handle_destroy);
+  UNMOCK(process_read_stdout);
   UNMOCK(get_or_state);
   UNMOCK(queue_control_event_string);
   if (controlevent_msgs) {
@@ -449,7 +495,7 @@ test_pt_configure_proxy(void *arg)
     smartlist_free(mp->transports);
   }
   smartlist_free(mp->transports_to_launch);
-  tor_free(mp->process_handle);
+  process_free(mp->process);
   tor_free(mp->argv[0]);
   tor_free(mp->argv);
   tor_free(mp);
@@ -533,8 +579,10 @@ test_get_pt_proxy_uri(void *arg)
     tor_free(uri);
 }
 
+#ifndef COCCI
 #define PT_LEGACY(name)                                               \
-  { #name, test_pt_ ## name , 0, NULL, NULL }
+  { (#name), test_pt_ ## name , 0, NULL, NULL }
+#endif
 
 struct testcase_t pt_tests[] = {
   PT_LEGACY(parsing),
diff --git a/src/test/test_ptr_slow.c b/src/test/test_ptr_slow.c
new file mode 100644
index 0000000000..25b893c4c0
--- /dev/null
+++ b/src/test/test_ptr_slow.c
@@ -0,0 +1,106 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+#include "core/or/or.h"
+#include "test/test.h"
+#include "test/ptr_helpers.h"
+
+#include <stdint.h>
+#include <limits.h>
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_int_voidptr_roundtrip(int a)
+{
+  intptr_t ap = (intptr_t)a;
+  void *b = cast_intptr_to_voidstar(ap);
+  intptr_t c = cast_voidstar_to_intptr(b);
+  void *d = cast_intptr_to_voidstar(c);
+
+  tt_assert(ap == c);
+  tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_int_voidstar_interop(void *arg)
+{
+  int a;
+  (void)arg;
+
+  for (a = -1024; a <= 1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MIN; a <= INT_MIN+1024; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  for (a = INT_MAX-1024; a < INT_MAX; a++) {
+    assert_int_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_int_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+/** Assert that <b>a</b> can be cast to void * and back. */
+static void
+assert_uint_voidptr_roundtrip(unsigned int a)
+{
+ uintptr_t ap = (uintptr_t)a;
+ void *b = cast_uintptr_to_voidstar(ap);
+ uintptr_t c = cast_voidstar_to_uintptr(b);
+ void *d = cast_uintptr_to_voidstar(c);
+
+ tt_assert(ap == c);
+ tt_assert(b == d);
+
+ done:
+  return;
+}
+
+/** Test for possibility of casting `int` to `void *` and back. */
+static void
+test_uint_voidstar_interop(void *arg)
+{
+  unsigned int a;
+  (void)arg;
+
+  for (a = 0; a <= 1024; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  for (a = UINT_MAX-1024; a < UINT_MAX; a++) {
+    assert_uint_voidptr_roundtrip(a);
+  }
+
+  a = 1;
+  for (unsigned long i = 0; i < sizeof(int) * 8; i++) {
+    assert_uint_voidptr_roundtrip(a);
+    a = (a << 1);
+  }
+}
+
+struct testcase_t slow_ptr_tests[] = {
+  { .name = "int_voidstar_interop",
+    .fn = test_int_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  { .name = "uint_voidstar_interop",
+    .fn = test_uint_voidstar_interop,
+    .flags = 0,
+    .setup = NULL,
+    .setup_data = NULL },
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_build.c b/src/test/test_pubsub_build.c
new file mode 100644
index 0000000000..5f9005926c
--- /dev/null
+++ b/src/test/test_pubsub_build.c
@@ -0,0 +1,578 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+#define PUBSUB_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/cc/torint.h"
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_macros.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include "test/log_test_helpers.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_int_fmt(msg_aux_data_t aux)
+{
+  int val = (int) aux.u64;
+  char *r=NULL;
+  tor_asprintf(&r, "%d", val);
+  return r;
+}
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+
+static dispatch_typefns_t intfns = {
+  .fmt_fn = ex_int_fmt
+};
+
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+DECLARE_MESSAGE_INT(bunch_of_coconuts, int, int);
+DECLARE_PUBLISH(bunch_of_coconuts);
+DECLARE_SUBSCRIBE(bunch_of_coconuts, coconut_recipient_cb);
+
+DECLARE_MESSAGE(yes_we_have_no, string, char *);
+DECLARE_PUBLISH(yes_we_have_no);
+DECLARE_SUBSCRIBE(yes_we_have_no, absent_item_cb);
+
+static void
+coconut_recipient_cb(const msg_t *m, int n_coconuts)
+{
+  (void)m;
+  (void)n_coconuts;
+}
+
+static void
+absent_item_cb(const msg_t *m, const char *fruitname)
+{
+  (void)m;
+  (void)fruitname;
+}
+
+#define FLAG_SKIP 99999
+
+static void
+seed_dispatch_builder(pubsub_builder_t *b,
+                      unsigned fl1, unsigned fl2, unsigned fl3, unsigned fl4)
+{
+  pubsub_connector_t *c = NULL;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    DISPATCH_REGISTER_TYPE(c, int, &intfns);
+    if (fl1 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, fl1);
+    if (fl2 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, yes_we_have_no, fl2);
+    pubsub_connector_free(c);
+  }
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys2"));
+    DISPATCH_REGISTER_TYPE(c, string, &stringfns);
+    if (fl3 != FLAG_SKIP)
+      DISPATCH_ADD_PUB_(c, main, yes_we_have_no, fl3);
+    if (fl4 != FLAG_SKIP)
+      DISPATCH_ADD_SUB_(c, main, bunch_of_coconuts, fl4);
+    pubsub_connector_free(c);
+  }
+}
+
+static void
+seed_pubsub_builder_basic(pubsub_builder_t *b)
+{
+  seed_dispatch_builder(b, 0, 0, 0, 0);
+}
+
+/* Regular builder with valid types and messages.
+ */
+static void
+test_pubsub_build_types_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  pubsub_items_t *items = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  dispatcher = pubsub_builder_finalize(b, &items);
+  b = NULL;
+  tt_assert(dispatcher);
+  tt_assert(items);
+  tt_int_op(smartlist_len(items->items), OP_EQ, 4);
+
+  // Make sure that the bindings got build correctly.
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, dispatcher);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+  tt_int_op(dispatcher->n_types, OP_GE, 2);
+  tt_assert(dispatcher->typefns);
+
+  tt_assert(dispatcher->typefns[get_msg_type_id("int")].fmt_fn == ex_int_fmt);
+  tt_assert(dispatcher->typefns[get_msg_type_id("string")].fmt_fn ==
+            ex_str_fmt);
+
+  // Now clear the bindings, like we would do before freeing the
+  // the dispatcher.
+  pubsub_items_clear_bindings(items);
+  SMARTLIST_FOREACH_BEGIN(items->items, pubsub_cfg_t *, item) {
+    if (item->is_publish) {
+      tt_assert(item->pub_binding);
+      tt_ptr_op(item->pub_binding->dispatch_ptr, OP_EQ, NULL);
+    }
+  } SMARTLIST_FOREACH_END(item);
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  pubsub_items_free(items);
+}
+
+/* We fail if the same type is defined in two places with different functions.
+ */
+static void
+test_pubsub_build_types_decls_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    // Extra declaration of int: we don't allow this.
+    DISPATCH_REGISTER_TYPE(c, int, &stringfns);
+    pubsub_connector_free(c);
+  }
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+  // expect_log_msg_containing("(int) declared twice"); // XXXX
+
+ done:
+  pubsub_connector_free(c);
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* If a message ID exists but nobody is publishing or subscribing to it,
+ * that's okay. */
+static void
+test_pubsub_build_unused_message(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  // This message isn't actually generated by anyone, but that will be fine:
+  // we just log it at info.
+  get_message_id("unused");
+  setup_capture_of_logs(LOG_INFO);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+  expect_log_msg_containing(
+     "Nobody is publishing or subscribing to message");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Publishing or subscribing to a message with no subscribers / publishers
+ * should fail and warn. */
+static void
+test_pubsub_build_missing_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, FLAG_SKIP, FLAG_SKIP);
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing(
+       "Message \"bunch_of_coconuts\" has publishers, but no subscribers.");
+  expect_log_msg_containing(
+       "Message \"yes_we_have_no\" has subscribers, but no publishers.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Make sure that a stub publisher or subscriber prevents an error from
+ * happening even if there are no other publishers/subscribers for a message
+ */
+static void
+test_pubsub_build_stub_pubsub(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, 0, 0, DISP_FLAG_STUB, DISP_FLAG_STUB);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // no subscribers
+  tt_ptr_op(NULL, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* Only one channel per msg id. */
+static void
+test_pubsub_build_channels_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("int"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent channels.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* Only one type per msg id. */
+static void
+test_pubsub_build_types_conflict(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("problems"));
+    /* Usually the DISPATCH_ADD_PUB macro would keep us from using
+     * the wrong channel */
+    pubsub_add_pub_(c, &btmp, get_channel_id("hithere"),
+                    get_message_id("bunch_of_coconuts"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is associated "
+                            "with multiple inconsistent message types.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* The same module can't publish and subscribe the same message */
+static void
+test_pubsub_build_pubsub_same(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys1"));
+    // already publishing this.
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("Message \"bunch_of_coconuts\" is published "
+                            "and subscribed by the same subsystem \"sys1\".");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+/* More than one subsystem may publish or subscribe, and that's okay. */
+static void
+test_pubsub_build_pubsub_multi(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+  pub_binding_t btmp;
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    pubsub_add_pub_(c, &btmp, get_channel_id("main"),
+                    get_message_id("yes_we_have_no"),
+                    get_msg_type_id("string"),
+                    0 /* flags */,
+                    "somewhere.c", 22);
+    pubsub_connector_free(c);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 2 subscribers.
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(2, OP_EQ, ent->n_enabled);
+  tt_int_op(2, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+static void
+some_other_coconut_hook(const msg_t *m)
+{
+  (void)m;
+}
+
+/* Subscribe hooks should be build correctly when there are a bunch of
+ * them. */
+static void
+test_pubsub_build_sub_many(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+  char *sysname = NULL;
+  b = pubsub_builder_new();
+  seed_pubsub_builder_basic(b);
+
+  int i;
+  for (i = 1; i < 100; ++i) {
+    tor_asprintf(&sysname, "system%d",i);
+    c = pubsub_connector_for_subsystem(b, get_subsys_id(sysname));
+    if (i % 7) {
+      DISPATCH_ADD_SUB(c, main, bunch_of_coconuts);
+    } else {
+      pubsub_add_sub_(c, some_other_coconut_hook,
+                      get_channel_id("main"),
+                      get_message_id("bunch_of_coconuts"),
+                      get_msg_type_id("int"),
+                      0 /* flags */,
+                      "somewhere.c", 22);
+    }
+    pubsub_connector_free(c);
+    tor_free(sysname);
+  };
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  dtbl_entry_t *ent =
+    dispatcher->table[get_message_id("bunch_of_coconuts")];
+  tt_int_op(100, OP_EQ, ent->n_enabled);
+  tt_int_op(100, OP_EQ, ent->n_fns);
+  tt_ptr_op(ent->rcv[0].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[1].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[76].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+  tt_ptr_op(ent->rcv[77].fn, OP_EQ, some_other_coconut_hook);
+  tt_ptr_op(ent->rcv[78].fn, OP_EQ, recv_fn__bunch_of_coconuts);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  tor_free(sysname);
+}
+
+/* It's fine to declare the excl flag. */
+static void
+test_pubsub_build_excl_ok(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+
+  b = pubsub_builder_new();
+  // Try one excl/excl pair and one excl/non pair.
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, 0,
+                        DISP_FLAG_EXCL, DISP_FLAG_EXCL);
+
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher);
+
+  // 1 subscribers
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("yes_we_have_no")]->n_enabled);
+  // 1 subscriber.
+  tt_int_op(1, OP_EQ,
+            dispatcher->table[get_message_id("bunch_of_coconuts")]->n_enabled);
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+}
+
+/* but if you declare the excl flag, you need to mean it. */
+static void
+test_pubsub_build_excl_bad(void *arg)
+{
+  (void)arg;
+  pubsub_builder_t *b = NULL;
+  dispatch_t *dispatcher = NULL;
+  pubsub_connector_t *c = NULL;
+
+  b = pubsub_builder_new();
+  seed_dispatch_builder(b, DISP_FLAG_EXCL, DISP_FLAG_EXCL,
+                        0, 0);
+
+  {
+    c = pubsub_connector_for_subsystem(b, get_subsys_id("sys3"));
+    DISPATCH_ADD_PUB_(c, main, bunch_of_coconuts, 0);
+    DISPATCH_ADD_SUB_(c, main, yes_we_have_no, 0);
+    pubsub_connector_free(c);
+  };
+
+  setup_full_capture_of_logs(LOG_WARN);
+  dispatcher = pubsub_builder_finalize(b, NULL);
+  b = NULL;
+  tt_assert(dispatcher == NULL);
+
+  expect_log_msg_containing("has multiple publishers, but at least one is "
+                            "marked as exclusive.");
+  expect_log_msg_containing("has multiple subscribers, but at least one is "
+                            "marked as exclusive.");
+
+ done:
+  pubsub_builder_free(b);
+  dispatch_free(dispatcher);
+  teardown_capture_of_logs();
+}
+
+#define T(name, flags)                                          \
+  { #name, test_pubsub_build_ ## name , (flags), NULL, NULL }
+
+struct testcase_t pubsub_build_tests[] = {
+  T(types_ok, TT_FORK),
+  T(types_decls_conflict, TT_FORK),
+  T(unused_message, TT_FORK),
+  T(missing_pubsub, TT_FORK),
+  T(stub_pubsub, TT_FORK),
+  T(channels_conflict, TT_FORK),
+  T(types_conflict, TT_FORK),
+  T(pubsub_same, TT_FORK),
+  T(pubsub_multi, TT_FORK),
+  T(sub_many, TT_FORK),
+  T(excl_ok, TT_FORK),
+  T(excl_bad, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_pubsub_msg.c b/src/test/test_pubsub_msg.c
new file mode 100644
index 0000000000..3054db885d
--- /dev/null
+++ b/src/test/test_pubsub_msg.c
@@ -0,0 +1,305 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#define DISPATCH_PRIVATE
+
+#include "test/test.h"
+
+#include "lib/dispatch/dispatch.h"
+#include "lib/dispatch/dispatch_naming.h"
+#include "lib/dispatch/dispatch_st.h"
+#include "lib/dispatch/msgtypes.h"
+#include "lib/pubsub/pubsub_flags.h"
+#include "lib/pubsub/pub_binding_st.h"
+#include "lib/pubsub/pubsub_build.h"
+#include "lib/pubsub/pubsub_builder_st.h"
+#include "lib/pubsub/pubsub_connect.h"
+#include "lib/pubsub/pubsub_publish.h"
+
+#include "lib/log/escape.h"
+#include "lib/malloc/malloc.h"
+#include "lib/string/printf.h"
+
+#include <stdio.h>
+#include <string.h>
+
+static char *
+ex_str_fmt(msg_aux_data_t aux)
+{
+  return esc_for_log(aux.ptr);
+}
+static void
+ex_str_free(msg_aux_data_t aux)
+{
+  tor_free_(aux.ptr);
+}
+static dispatch_typefns_t stringfns = {
+  .free_fn = ex_str_free,
+  .fmt_fn = ex_str_fmt
+};
+
+// We're using the lowest-level publish/subscribe logic here, to avoid the
+// pubsub_macros.h macros and just test the dispatch core.  We'll use a string
+// type for everything.
+
+#define DECLARE_MESSAGE(suffix)                         \
+  static pub_binding_t pub_binding_##suffix;            \
+  static int msg_received_##suffix = 0;                 \
+  static void recv_msg_##suffix(const msg_t *m) {       \
+    (void)m;                                            \
+    ++msg_received_##suffix;                            \
+  }                                                     \
+  EAT_SEMICOLON
+
+#define ADD_PUBLISH(binding_suffix, subsys, channel, msg, flags)        \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_pub_(con, &pub_binding_##binding_suffix,                 \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define ADD_SUBSCRIBE(hook_suffix, subsys, channel, msg, flags)         \
+  STMT_BEGIN {                                                          \
+    con = pubsub_connector_for_subsystem(builder,                       \
+                                           get_subsys_id(#subsys));     \
+    pubsub_add_sub_(con, recv_msg_##hook_suffix,                        \
+                      get_channel_id(#channel),                         \
+                      get_message_id(#msg), get_msg_type_id("string"),  \
+                      (flags), __FILE__, __LINE__);                     \
+    pubsub_connector_free(con);                                         \
+  } STMT_END
+
+#define SEND(binding_suffix, val)                          \
+  STMT_BEGIN {                                             \
+    msg_aux_data_t data_;                                  \
+    data_.ptr = tor_strdup(val);                           \
+    pubsub_pub_(&pub_binding_##binding_suffix, data_);     \
+  } STMT_END
+
+DECLARE_MESSAGE(msg1);
+DECLARE_MESSAGE(msg2);
+DECLARE_MESSAGE(msg3);
+DECLARE_MESSAGE(msg4);
+DECLARE_MESSAGE(msg5);
+
+static smartlist_t *strings_received = NULL;
+static void
+recv_msg_copy_string(const msg_t *m)
+{
+  const char *s = m->aux_data__.ptr;
+  smartlist_add(strings_received, tor_strdup(s));
+}
+
+static void *
+setup_dispatcher(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  pubsub_builder_t *builder = pubsub_builder_new();
+  pubsub_connector_t *con;
+
+  {
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id("types"));
+    pubsub_connector_register_type_(con,
+                                    get_msg_type_id("string"),
+                                    &stringfns,
+                                    "nowhere.c", 99);
+    pubsub_connector_free(con);
+  }
+  // message1 has one publisher and one subscriber.
+  ADD_PUBLISH(msg1, sys1, main, message1, 0);
+  ADD_SUBSCRIBE(msg1, sys2, main, message1, 0);
+
+  // message2 has a publisher and a stub subscriber.
+  ADD_PUBLISH(msg2, sys1, main, message2, 0);
+  ADD_SUBSCRIBE(msg2, sys2, main, message2, DISP_FLAG_STUB);
+
+  // message3 has a publisher and three subscribers.
+  ADD_PUBLISH(msg3, sys1, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys2, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys3, main, message3, 0);
+  ADD_SUBSCRIBE(msg3, sys4, main, message3, 0);
+
+  // message4 has one publisher and two subscribers, but it's on another
+  // channel.
+  ADD_PUBLISH(msg4, sys2, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys1, other, message4, 0);
+  ADD_SUBSCRIBE(msg4, sys3, other, message4, 0);
+
+  // message5 has a huge number of recipients.
+  ADD_PUBLISH(msg5, sys3, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys4, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys5, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys6, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys7, main, message5, 0);
+  ADD_SUBSCRIBE(msg5, sys8, main, message5, 0);
+  for (int i = 0; i < 1000-5; ++i) {
+    char *sys;
+    tor_asprintf(&sys, "xsys-%d", i);
+    con = pubsub_connector_for_subsystem(builder, get_subsys_id(sys));
+    pubsub_add_sub_(con, recv_msg_copy_string,
+                    get_channel_id("main"),
+                    get_message_id("message5"),
+                    get_msg_type_id("string"), 0, "here", 100);
+    pubsub_connector_free(con);
+    tor_free(sys);
+  }
+
+  return pubsub_builder_finalize(builder, NULL);
+}
+
+static int
+cleanup_dispatcher(const struct testcase_t *testcase, void *dispatcher_)
+{
+  (void)testcase;
+  dispatch_t *dispatcher = dispatcher_;
+  dispatch_free(dispatcher);
+  return 1;
+}
+
+static const struct testcase_setup_t dispatcher_setup = {
+  setup_dispatcher, cleanup_dispatcher
+};
+
+static void
+test_pubsub_msg_minimal(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  SEND(msg1, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(1, OP_EQ, msg_received_msg1); // we got the message!
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_send_to_stub(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg2);
+  SEND(msg2, "hello silence");
+  tt_int_op(0, OP_EQ, msg_received_msg2); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(0, OP_EQ, msg_received_msg2); // doesn't arrive -- stub hook.
+
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_cancel_msgs(void *arg)
+{
+  dispatch_t *d = arg;
+
+  tt_int_op(0, OP_EQ, msg_received_msg1);
+  for (int i = 0; i < 100; ++i) {
+    SEND(msg1, "hello world");
+  }
+  tt_int_op(0, OP_EQ, msg_received_msg1); // hasn't actually arrived yet.
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 10));
+  tt_int_op(10, OP_EQ, msg_received_msg1); // we got the message 10 times.
+
+  // At this point, the dispatcher will be freed with queued, undelivered
+  // messages.
+ done:
+  ;
+}
+
+struct alertfn_target {
+  dispatch_t *d;
+  channel_id_t ch;
+  int count;
+};
+static void
+alertfn_generic(dispatch_t *d, channel_id_t ch, void *arg)
+{
+  struct alertfn_target *t = arg;
+  tt_ptr_op(d, OP_EQ, t->d);
+  tt_int_op(ch, OP_EQ, t->ch);
+  ++t->count;
+ done:
+  ;
+}
+
+static void
+test_pubsub_msg_alertfns(void *arg)
+{
+  dispatch_t *d = arg;
+  struct alertfn_target ch1_a = { d, get_channel_id("main"), 0 };
+  struct alertfn_target ch2_a = { d, get_channel_id("other"), 0 };
+
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("main"),
+                                  alertfn_generic, &ch1_a));
+  tt_int_op(0, OP_EQ,
+            dispatch_set_alert_fn(d, get_channel_id("other"),
+                                  alertfn_generic, &ch2_a));
+
+  SEND(msg3, "hello");
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg3, "world");
+  tt_int_op(ch1_a.count, OP_EQ, 1); // only the first message sends an alert
+  tt_int_op(ch2_a.count, OP_EQ, 0); // no alert for 'other'
+
+  SEND(msg4, "worse things happen in C");
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+  // flush the first (main) channel...
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 1000));
+  tt_int_op(6, OP_EQ, msg_received_msg3); // 3 subscribers, 2 instances.
+
+  // now that the main channel is flushed, sending another message on it
+  // starts another alert.
+  tt_int_op(ch1_a.count, OP_EQ, 1);
+  SEND(msg1, "plover");
+  tt_int_op(ch1_a.count, OP_EQ, 2);
+  tt_int_op(ch2_a.count, OP_EQ, 1);
+
+ done:
+  ;
+}
+
+/* try more than N_FAST_FNS hooks on msg5 */
+static void
+test_pubsub_msg_many_hooks(void *arg)
+{
+  dispatch_t *d = arg;
+  strings_received = smartlist_new();
+
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  SEND(msg5, "hello world");
+  tt_int_op(0, OP_EQ, msg_received_msg5);
+  tt_int_op(0, OP_EQ, smartlist_len(strings_received));
+
+  tt_int_op(0, OP_EQ, dispatch_flush(d, get_channel_id("main"), 100000));
+  tt_int_op(5, OP_EQ, msg_received_msg5);
+  tt_int_op(995, OP_EQ, smartlist_len(strings_received));
+
+ done:
+  SMARTLIST_FOREACH(strings_received, char *, s, tor_free(s));
+  smartlist_free(strings_received);
+}
+
+#define T(name)                                                 \
+  { #name, test_pubsub_msg_ ## name , TT_FORK,                \
+      &dispatcher_setup, NULL }
+
+struct testcase_t pubsub_msg_tests[] = {
+  T(minimal),
+  T(send_to_stub),
+  T(cancel_msgs),
+  T(alertfns),
+  T(many_hooks),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_rebind.py b/src/test/test_rebind.py
index 30a587858f..3fc3deb68e 100644
--- a/src/test/test_rebind.py
+++ b/src/test/test_rebind.py
@@ -1,4 +1,7 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import errno
 import logging
@@ -22,9 +25,10 @@ def skip(msg):
 
 def try_connecting_to_socksport():
     socks_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    if socks_socket.connect_ex(('127.0.0.1', socks_port)):
+    e = socks_socket.connect_ex(('127.0.0.1', socks_port))
+    if e:
         tor_process.terminate()
-        fail('Cannot connect to SOCKSPort')
+        fail('Cannot connect to SOCKSPort: error ' + os.strerror(e))
     socks_socket.close()
 
 def wait_for_log(s):
diff --git a/src/test/test_rebind.sh b/src/test/test_rebind.sh
index ea2012957e..879008c1c1 100755
--- a/src/test/test_rebind.sh
+++ b/src/test/test_rebind.sh
@@ -1,24 +1,60 @@
 #!/bin/sh
 
+umask 077
+set -e
 set -x
 
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
 UNAME_OS=$(uname -s | cut -d_ -f1)
 if test "$UNAME_OS" = 'CYGWIN' || \
    test "$UNAME_OS" = 'MSYS' || \
    test "$UNAME_OS" = 'MINGW'; then
   if test "$APPVEYOR" = 'True'; then
-    echo "This test is disabled on Windows CI, as it requires firewall examptions. Skipping." >&2
+    echo "This test is disabled on Windows CI, as it requires firewall exemptions. Skipping." >&2
     exit 77
   fi
 fi
 
-exitcode=0
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "${TOR_BINARY}" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
 
 tmpdir=
-clean () { test -n "$tmpdir" && test -d "$tmpdir" && rm -rf "$tmpdir" || :; }
+clean () {
+  if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then
+    rm -rf "$tmpdir"
+  fi
+}
+
 trap clean EXIT HUP INT TERM
 
-tmpdir="`mktemp -d -t tor_rebind_test.XXXXXX`"
+tmpdir="$(mktemp -d -t tor_rebind_test.XXXXXX)"
 if [ -z "$tmpdir" ]; then
   echo >&2 mktemp failed
   exit 2
@@ -27,6 +63,6 @@ elif [ ! -d "$tmpdir" ]; then
   exit 3
 fi
 
-"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TESTING_TOR_BINARY}" "$tmpdir"
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_rebind.py" "${TOR_BINARY}" "$tmpdir"
 
 exit $?
diff --git a/src/test/test_relay.c b/src/test/test_relay.c
index 0b7a7be332..066aeaa7b3 100644
--- a/src/test/test_relay.c
+++ b/src/test/test_relay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define CIRCUITBUILD_PRIVATE
@@ -10,7 +10,6 @@
 #include "core/or/channeltls.h"
 #include "feature/stats/rephist.h"
 #include "core/or/relay.h"
-#include "feature/stats/rephist.h"
 #include "lib/container/order.h"
 /* For init/free stuff */
 #include "core/or/scheduler.h"
@@ -21,42 +20,10 @@
 /* Test suite stuff */
 #include "test/test.h"
 #include "test/fakechans.h"
-
-static or_circuit_t * new_fake_orcirc(channel_t *nchan, channel_t *pchan);
+#include "test/fakecircs.h"
 
 static void test_relay_append_cell_to_circuit_queue(void *arg);
 
-static or_circuit_t *
-new_fake_orcirc(channel_t *nchan, channel_t *pchan)
-{
-  or_circuit_t *orcirc = NULL;
-  circuit_t *circ = NULL;
-
-  orcirc = tor_malloc_zero(sizeof(*orcirc));
-  circ = &(orcirc->base_);
-  circ->magic = OR_CIRCUIT_MAGIC;
-
-  circuit_set_n_circid_chan(circ, get_unique_circ_id_by_chan(nchan), nchan);
-  cell_queue_init(&(circ->n_chan_cells));
-
-  circ->n_hop = NULL;
-  circ->streams_blocked_on_n_chan = 0;
-  circ->streams_blocked_on_p_chan = 0;
-  circ->n_delete_pending = 0;
-  circ->p_delete_pending = 0;
-  circ->received_destroy = 0;
-  circ->state = CIRCUIT_STATE_OPEN;
-  circ->purpose = CIRCUIT_PURPOSE_OR;
-  circ->package_window = CIRCWINDOW_START_MAX;
-  circ->deliver_window = CIRCWINDOW_START_MAX;
-  circ->n_chan_create_cell = NULL;
-
-  circuit_set_p_circid_chan(orcirc, get_unique_circ_id_by_chan(pchan), pchan);
-  cell_queue_init(&(orcirc->p_chan_cells));
-
-  return orcirc;
-}
-
 static void
 assert_circuit_ok_mock(const circuit_t *c)
 {
@@ -145,7 +112,7 @@ test_relay_close_circuit(void *arg)
     cell_queue_clear(&orcirc->base_.n_chan_cells);
     cell_queue_clear(&orcirc->p_chan_cells);
   }
-  tor_free(orcirc);
+  free_fake_orcirc(orcirc);
   free_fake_channel(nchan);
   free_fake_channel(pchan);
   UNMOCK(assert_circuit_ok);
@@ -218,7 +185,7 @@ test_relay_append_cell_to_circuit_queue(void *arg)
     cell_queue_clear(&orcirc->base_.n_chan_cells);
     cell_queue_clear(&orcirc->p_chan_cells);
   }
-  tor_free(orcirc);
+  free_fake_orcirc(orcirc);
   free_fake_channel(nchan);
   free_fake_channel(pchan);
 
diff --git a/src/test/test_relaycell.c b/src/test/test_relaycell.c
index 0623583511..da9e791fb6 100644
--- a/src/test/test_relaycell.c
+++ b/src/test/test_relaycell.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /* Unit tests for handling different kinds of relay cell */
@@ -17,6 +17,7 @@
 #include "core/or/circuitbuild.h"
 #include "core/or/circuitlist.h"
 #include "core/or/connection_edge.h"
+#include "core/or/sendme.h"
 #include "core/or/relay.h"
 #include "test/test.h"
 #include "test/log_test_helpers.h"
@@ -29,7 +30,6 @@
 #include "core/or/half_edge_st.h"
 
 #include "feature/client/circpathbias.h"
-#include "core/or/connection_edge.h"
 
 static int srm_ncalls;
 static entry_connection_t *srm_conn;
@@ -812,7 +812,11 @@ test_circbw_relay(void *arg)
   ASSERT_UNCOUNTED_BW();
 
   /* Sendme on circuit with non-full window: counted */
-  PACK_CELL(0, RELAY_COMMAND_SENDME, "Data1234");
+  PACK_CELL(0, RELAY_COMMAND_SENDME, "");
+  /* Recording a cell, the window is updated after decryption so off by one in
+   * order to record and then we process it with the proper window. */
+  circ->cpath->package_window = 901;
+  sendme_record_cell_digest_on_circ(TO_CIRCUIT(circ), circ->cpath);
   circ->cpath->package_window = 900;
   connection_edge_process_relay_cell(&cell, TO_CIRCUIT(circ), edgeconn,
                                      circ->cpath);
diff --git a/src/test/test_relaycrypt.c b/src/test/test_relaycrypt.c
index fe6889e521..737c243e2d 100644
--- a/src/test/test_relaycrypt.c
+++ b/src/test/test_relaycrypt.c
@@ -1,8 +1,10 @@
 /* Copyright 2001-2004 Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
+#define CRYPT_PATH_PRIVATE
+
 #include "core/or/or.h"
 #include "core/or/circuitbuild.h"
 #define CIRCUITLIST_PRIVATE
@@ -10,7 +12,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "core/or/relay.h"
 #include "core/crypto/relay_crypto.h"
-
+#include "core/or/crypt_path.h"
 #include "core/or/cell_st.h"
 #include "core/or/or_circuit_st.h"
 #include "core/or/origin_circuit_st.h"
@@ -49,10 +51,10 @@ testing_circuitset_setup(const struct testcase_t *testcase)
   cs->origin_circ->base_.purpose = CIRCUIT_PURPOSE_C_GENERAL;
   for (i=0; i<3; ++i) {
     crypt_path_t *hop = tor_malloc_zero(sizeof(*hop));
-    relay_crypto_init(&hop->crypto, KEY_MATERIAL[i], sizeof(KEY_MATERIAL[i]),
-                      0, 0);
+    relay_crypto_init(&hop->pvt_crypto, KEY_MATERIAL[i],
+                      sizeof(KEY_MATERIAL[i]), 0, 0);
     hop->state = CPATH_STATE_OPEN;
-    onion_append_to_cpath(&cs->origin_circ->cpath, hop);
+    cpath_extend_linked_list(&cs->origin_circ->cpath, hop);
     tt_ptr_op(hop, OP_EQ, cs->origin_circ->cpath->prev);
   }
 
diff --git a/src/test/test_rendcache.c b/src/test/test_rendcache.c
index 8b0e2df485..06167635c1 100644
--- a/src/test/test_rendcache.c
+++ b/src/test/test_rendcache.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -21,8 +21,6 @@
 #include "test/rend_test_helpers.h"
 #include "test/log_test_helpers.h"
 
-#define NS_MODULE rend_cache
-
 static const int RECENT_TIME = -10;
 static const int TIME_IN_THE_PAST = -(REND_CACHE_MAX_AGE + \
                                       REND_CACHE_MAX_SKEW + 60);
@@ -369,13 +367,12 @@ test_rend_cache_store_v2_desc_as_client_with_different_time(void *data)
   rend_data_free(mock_rend_query);
 }
 
-#define NS_SUBMODULE lookup_v2_desc_as_dir
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t *rcache_lookup_v2_as_dir_get_my_routerinfo(void);
 
 static routerinfo_t *mock_routerinfo;
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+rcache_lookup_v2_as_dir_get_my_routerinfo(void)
 {
   if (!mock_routerinfo) {
     mock_routerinfo = tor_malloc(sizeof(routerinfo_t));
@@ -395,7 +392,8 @@ test_rend_cache_lookup_v2_desc_as_dir(void *data)
 
   (void)data;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_lookup_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -418,20 +416,17 @@ test_rend_cache_lookup_v2_desc_as_dir(void *data)
   tt_assert(ret_desc);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   tor_free(mock_routerinfo);
   rend_cache_free_all();
   rend_encoded_v2_service_descriptor_free(desc_holder);
   tor_free(service_id);
 }
 
-#undef NS_SUBMODULE
-
-#define NS_SUBMODULE store_v2_desc_as_dir
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t *rcache_store_v2_as_dir_get_my_routerinfo(void);
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+rcache_store_v2_as_dir_get_my_routerinfo(void)
 {
   return mock_routerinfo;
 }
@@ -444,7 +439,8 @@ test_rend_cache_store_v2_desc_as_dir(void *data)
   rend_encoded_v2_service_descriptor_t *desc_holder = NULL;
   char *service_id = NULL;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_store_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -485,7 +481,7 @@ test_rend_cache_store_v2_desc_as_dir(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   rend_encoded_v2_service_descriptor_free(desc_holder);
   tor_free(service_id);
   rend_cache_free_all();
@@ -505,7 +501,8 @@ test_rend_cache_store_v2_desc_as_dir_with_different_time(void *data)
   rend_encoded_v2_service_descriptor_t *desc_holder_newer;
   rend_encoded_v2_service_descriptor_t *desc_holder_older;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_store_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -543,7 +540,7 @@ test_rend_cache_store_v2_desc_as_dir_with_different_time(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   rend_cache_free_all();
   rend_service_descriptor_free(generated);
   tor_free(service_id);
@@ -568,7 +565,8 @@ test_rend_cache_store_v2_desc_as_dir_with_different_content(void *data)
   rend_encoded_v2_service_descriptor_t *desc_holder_one = NULL;
   rend_encoded_v2_service_descriptor_t *desc_holder_two = NULL;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rcache_store_v2_as_dir_get_my_routerinfo);
 
   rend_cache_init();
 
@@ -602,7 +600,7 @@ test_rend_cache_store_v2_desc_as_dir_with_different_content(void *data)
   tt_int_op(ret, OP_EQ, 0);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
   rend_cache_free_all();
   rend_service_descriptor_free(generated);
   tor_free(service_id);
@@ -613,8 +611,6 @@ test_rend_cache_store_v2_desc_as_dir_with_different_content(void *data)
   rend_encoded_v2_service_descriptor_free(desc_holder_two);
 }
 
-#undef NS_SUBMODULE
-
 static void
 test_rend_cache_init(void *data)
 {
@@ -1077,8 +1073,6 @@ test_rend_cache_intro_failure_note(void *data)
   rend_cache_free_all();
 }
 
-#define NS_SUBMODULE clean_v2_descs_as_dir
-
 static void
 test_rend_cache_clean_v2_descs_as_dir(void *data)
 {
@@ -1120,8 +1114,6 @@ test_rend_cache_clean_v2_descs_as_dir(void *data)
   rend_cache_free_all();
 }
 
-#undef NS_SUBMODULE
-
 static void
 test_rend_cache_entry_allocation(void *data)
 {
diff --git a/src/test/test_replay.c b/src/test/test_replay.c
index 28a508bf4d..1487b0a29d 100644
--- a/src/test/test_replay.c
+++ b/src/test/test_replay.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2012-2019, The Tor Project, Inc. */
+/* Copyright (c) 2012-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define REPLAYCACHE_PRIVATE
diff --git a/src/test/test_rng.c b/src/test/test_rng.c
new file mode 100644
index 0000000000..ebaffb74f5
--- /dev/null
+++ b/src/test/test_rng.c
@@ -0,0 +1,59 @@
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/*
+ * Example usage:
+ *
+ * ./src/test/test-rng --emit | dieharder -g 200 -a
+ *
+ * Remember, dieharder can tell you that your RNG is completely broken, but if
+ * your RNG is not _completely_ broken, dieharder cannot tell you whether your
+ * RNG is actually secure.
+ */
+
+#include "orconfig.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+
+#include "lib/crypt_ops/crypto_rand.h"
+
+int
+main(int argc, char **argv)
+{
+  uint8_t buf[0x123];
+
+  if (argc != 2 || strcmp(argv[1], "--emit")) {
+    fprintf(stderr, "If you want me to fill stdout with a bunch of random "
+            "bytes, you need to say --emit.\n");
+    return 1;
+  }
+
+  if (crypto_seed_rng() < 0) {
+    fprintf(stderr, "Can't seed RNG.\n");
+    return 1;
+  }
+
+#if 0
+  while (1) {
+    crypto_rand(buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+#endif /* 0 */
+
+  crypto_fast_rng_t *rng = crypto_fast_rng_new();
+  while (1) {
+    crypto_fast_rng_getbytes(rng, buf, sizeof(buf));
+    if (write(1 /*stdout*/, buf, sizeof(buf)) != sizeof(buf)) {
+      fprintf(stderr, "write() failed: %s\n", strerror(errno));
+      return 1;
+    }
+  }
+}
diff --git a/src/test/test_router.c b/src/test/test_router.c
index e0d3adfdbd..cf0c2b3dd1 100644
--- a/src/test/test_router.c
+++ b/src/test/test_router.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* Copyright (c) 2017, isis agora lovecruft  */
 /* See LICENSE for licensing information     */
 
@@ -9,13 +9,20 @@
 
 #define CONFIG_PRIVATE
 #define CONNECTION_PRIVATE
+#define ROUTER_PRIVATE
+
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "core/mainloop/mainloop.h"
 #include "core/mainloop/connection.h"
 #include "feature/hibernate/hibernate.h"
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/nodelist.h"
 #include "feature/nodelist/routerinfo_st.h"
 #include "feature/nodelist/routerlist.h"
+#include "feature/nodelist/routerstatus_st.h"
 #include "feature/relay/router.h"
 #include "feature/stats/rephist.h"
 #include "lib/crypt_ops/crypto_curve25519.h"
@@ -28,12 +35,13 @@
 #include "test/test.h"
 #include "test/log_test_helpers.h"
 
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static const routerinfo_t * rtr_tests_router_get_my_routerinfo(void);
+ATTR_UNUSED static int rtr_tests_router_get_my_routerinfo_called = 0;
 
 static routerinfo_t* mock_routerinfo;
 
 static const routerinfo_t*
-NS(router_get_my_routerinfo)(void)
+rtr_tests_router_get_my_routerinfo(void)
 {
   crypto_pk_t* ident_key;
   crypto_pk_t* tap_key;
@@ -83,20 +91,24 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
   char* found = NULL;
   (void)arg;
 
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(router_get_my_routerinfo,
+       rtr_tests_router_get_my_routerinfo);
 
   options->ORPort_set = 1;
   options->BridgeRelay = 1;
 
   /* Generate keys which router_dump_router_to_string() expects to exist. */
-  tt_int_op(0, ==, curve25519_keypair_generate(&ntor_keypair, 0));
-  tt_int_op(0, ==, ed25519_keypair_generate(&signing_keypair, 0));
+  tt_int_op(0, OP_EQ, curve25519_keypair_generate(&ntor_keypair, 0));
+  tt_int_op(0, OP_EQ, ed25519_keypair_generate(&signing_keypair, 0));
 
   /* Set up part of our routerinfo_t so that we don't trigger any other
    * assertions in router_dump_router_to_string(). */
   router = (routerinfo_t*)router_get_my_routerinfo();
-  tt_ptr_op(router, !=, NULL);
+  tt_ptr_op(router, OP_NE, NULL);
 
+  /* The real router_get_my_routerinfo() looks up onion_curve25519_pkey using
+   * get_current_curve25519_keypair(), but we don't initialise static data in
+   * this test. */
   router->onion_curve25519_pkey = &ntor_keypair.pubkey;
 
   /* Generate our server descriptor and ensure that the substring
@@ -109,12 +121,12 @@ test_router_dump_router_to_string_no_bridge_distribution_method(void *arg)
                                       &ntor_keypair,
                                       &signing_keypair);
   crypto_pk_free(onion_pkey);
-  tt_ptr_op(desc, !=, NULL);
+  tt_ptr_op(desc, OP_NE, NULL);
   found = strstr(desc, needle);
-  tt_ptr_op(found, !=, NULL);
+  tt_ptr_op(found, OP_NE, NULL);
 
  done:
-  NS_UNMOCK(router_get_my_routerinfo);
+  UNMOCK(router_get_my_routerinfo);
 
   tor_free(desc);
 }
@@ -237,6 +249,247 @@ test_router_check_descriptor_bandwidth_changed(void *arg)
   UNMOCK(we_are_hibernating);
 }
 
+static networkstatus_t *mock_ns = NULL;
+static networkstatus_t *
+mock_networkstatus_get_live_consensus(time_t now)
+{
+  (void)now;
+  return mock_ns;
+}
+
+static routerstatus_t *mock_rs = NULL;
+static const routerstatus_t *
+mock_networkstatus_vote_find_entry(networkstatus_t *ns, const char *digest)
+{
+  (void)ns;
+  (void)digest;
+  return mock_rs;
+}
+
+static void
+test_router_mark_if_too_old(void *arg)
+{
+  (void)arg;
+  time_t now = approx_time();
+  MOCK(networkstatus_get_live_consensus,
+       mock_networkstatus_get_live_consensus);
+  MOCK(networkstatus_vote_find_entry, mock_networkstatus_vote_find_entry);
+
+  routerstatus_t rs;
+  networkstatus_t ns;
+  memset(&rs, 0, sizeof(rs));
+  memset(&ns, 0, sizeof(ns));
+  mock_ns = &ns;
+  mock_ns->valid_after = now-3600;
+  mock_rs = &rs;
+  mock_rs->published_on = now - 10;
+
+  // no reason to mark this time.
+  desc_clean_since = now-10;
+  desc_dirty_reason = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+
+  // Doesn't appear in consensus?  Still don't mark it.
+  mock_ns = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now-10);
+  mock_ns = &ns;
+
+  // No new descriptor in a long time?  Mark it.
+  desc_clean_since = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ, "time for new descriptor");
+
+  // Version in consensus published a long time ago?  We won't mark it
+  // if it's been clean for only a short time.
+  desc_clean_since = now - 10;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 3600 * 96;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, now - 10);
+
+  // ... but if it's been clean a while, we mark.
+  desc_clean_since = now - 2 * 3600;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "version listed in consensus is quite old");
+
+  // same deal if we're marked stale.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs->published_on = now - 10;
+  mock_rs->is_staledesc = 1;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "listed as stale in consensus");
+
+  // same deal if we're absent from the consensus.
+  desc_clean_since = now - 2 * 3600;
+  desc_dirty_reason = NULL;
+  mock_rs = NULL;
+  mark_my_descriptor_dirty_if_too_old(now);
+  tt_i64_op(desc_clean_since, OP_EQ, 0);
+  tt_str_op(desc_dirty_reason, OP_EQ,
+            "not listed in consensus");
+
+ done:
+  UNMOCK(networkstatus_get_live_consensus);
+  UNMOCK(networkstatus_vote_find_entry);
+}
+
+static node_t fake_node;
+static const node_t *
+mock_node_get_by_nickname(const char *name, unsigned flags)
+{
+  (void)flags;
+  if (!strcasecmp(name, "crumpet"))
+    return &fake_node;
+  else
+    return NULL;
+}
+
+static void
+test_router_get_my_family(void *arg)
+{
+  (void)arg;
+  or_options_t *options = options_new();
+  smartlist_t *sl = NULL;
+  char *join = NULL;
+  // Overwrite the result of router_get_my_identity_digest().  This
+  // happens to be okay, but only for testing.
+  set_server_identity_key_digest_testing(
+                                   (const uint8_t*)"holeinthebottomofthe");
+
+  setup_capture_of_logs(LOG_WARN);
+
+  // No family listed -- so there's no list.
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+#define CLEAR() do {                                    \
+    if (sl) {                                           \
+      SMARTLIST_FOREACH(sl, char *, cp, tor_free(cp));  \
+      smartlist_free(sl);                               \
+    }                                                   \
+    tor_free(join);                                     \
+    mock_clean_saved_logs();                            \
+  } while (0)
+
+  // Add a single nice friendly hex member.  This should be enough
+  // to have our own ID added.
+  tt_ptr_op(options->MyFamily, OP_EQ, NULL);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 2);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Add a hex member with a ~.  The ~ part should get removed.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$0123456789abcdef0123456789abcdef01234567~Muffin");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 3);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA");
+  expect_no_log_entry();
+  CLEAR();
+
+  // Nickname lookup will fail, so a nickname will appear verbatim.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "BAGEL");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  expect_single_log_msg_containing(
+           "There is a router named \"BAGEL\" in my declared family, but "
+           "I have no descriptor for it.");
+  CLEAR();
+
+  // A bogus digest should fail entirely.
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$painauchocolat");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 4);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.
+  expect_single_log_msg_containing(
+           "There is a router named \"$painauchocolat\" in my declared "
+           "family, but that isn't a legal digest or nickname. Skipping it.");
+  CLEAR();
+
+  // Let's introduce a node we can look up by nickname
+  memset(&fake_node, 0, sizeof(fake_node));
+  memcpy(fake_node.identity, "whydoyouasknonononon", DIGEST_LEN);
+  MOCK(node_get_by_nickname, mock_node_get_by_nickname);
+
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "CRUmpeT");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_NE, NULL);
+  tt_int_op(smartlist_len(sl), OP_EQ, 5);
+  join = smartlist_join_strings(sl, " ", 0, NULL);
+  tt_str_op(join, OP_EQ,
+            "$0123456789ABCDEF0123456789ABCDEF01234567 "
+            "$686F6C65696E746865626F74746F6D6F66746865 "
+            "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E "
+            "$AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA "
+            "bagel");
+  // "BAGEL" is still there, but it won't make a warning, because we already
+  // warned about it.  Some with "$painauchocolat".
+  expect_single_log_msg_containing(
+           "There is a router named \"CRUmpeT\" in my declared "
+           "family, but it wasn't listed by digest. Please consider saying "
+           "$776879646F796F7561736B6E6F6E6F6E6F6E6F6E instead, if that's "
+           "what you meant.");
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+  // Try a singleton list containing only us: It should give us NULL.
+  config_free_lines(options->MyFamily);
+  config_line_append(&options->MyFamily, "MyFamily",
+                     "$686F6C65696E746865626F74746F6D6F66746865");
+  sl = get_my_declared_family(options);
+  tt_ptr_op(sl, OP_EQ, NULL);
+  expect_no_log_entry();
+
+ done:
+  or_options_free(options);
+  teardown_capture_of_logs();
+  CLEAR();
+  UNMOCK(node_get_by_nickname);
+
+#undef CLEAR
+}
+
 static smartlist_t *fake_connection_array = NULL;
 static smartlist_t *
 mock_get_connection_array(void)
@@ -354,6 +607,8 @@ test_router_get_advertised_or_port_localhost(void *arg)
 struct testcase_t router_tests[] = {
   ROUTER_TEST(check_descriptor_bandwidth_changed, TT_FORK),
   ROUTER_TEST(dump_router_to_string_no_bridge_distribution_method, TT_FORK),
+  ROUTER_TEST(mark_if_too_old, TT_FORK),
+  ROUTER_TEST(get_my_family, TT_FORK),
   ROUTER_TEST(get_advertised_or_port, TT_FORK),
   ROUTER_TEST(get_advertised_or_port_localhost, TT_FORK),
   END_OF_TESTCASES
diff --git a/src/test/test_routerkeys.c b/src/test/test_routerkeys.c
index 727fa5660f..fc437dccc0 100644
--- a/src/test/test_routerkeys.c
+++ b/src/test/test_routerkeys.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -399,7 +399,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -409,7 +409,7 @@ test_routerkeys_ed_key_init_split(void *arg)
   tt_assert(kp2 != NULL);
   tt_assert(cert == NULL);
   tt_mem_op(&kp1->pubkey, OP_EQ, &kp2->pubkey, sizeof(kp2->pubkey));
-  tt_assert(tor_mem_is_zero((char*)kp2->seckey.seckey,
+  tt_assert(fast_mem_is_zero((char*)kp2->seckey.seckey,
                             sizeof(kp2->seckey.seckey)));
   ed25519_keypair_free(kp2); kp2 = NULL;
 
@@ -455,11 +455,11 @@ test_routerkeys_ed_keys_init_all(void *arg)
   options->TestingLinkKeySlop = 2*3600;
 
 #ifdef _WIN32
-  mkdir(dir);
-  mkdir(keydir);
+  tt_int_op(0, OP_EQ, mkdir(dir));
+  tt_int_op(0, OP_EQ, mkdir(keydir));
 #else
-  mkdir(dir, 0700);
-  mkdir(keydir, 0700);
+  tt_int_op(0, OP_EQ, mkdir(dir, 0700));
+  tt_int_op(0, OP_EQ, mkdir(keydir, 0700));
 #endif /* defined(_WIN32) */
 
   options->DataDirectory = dir;
diff --git a/src/test/test_routerlist.c b/src/test/test_routerlist.c
index 95c9176faa..f2a83c18a3 100644
--- a/src/test/test_routerlist.c
+++ b/src/test/test_routerlist.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -46,7 +46,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "test/test.h"
 #include "test/test_dir_common.h"
@@ -265,7 +265,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   /* Init SR subsystem. */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -275,7 +277,9 @@ test_router_pick_directory_server_impl(void *arg)
 
   construct_consensus(&consensus_text_md, now);
   tt_assert(consensus_text_md);
-  con_md = networkstatus_parse_vote_from_string(consensus_text_md, NULL,
+  con_md = networkstatus_parse_vote_from_string(consensus_text_md,
+                                                strlen(consensus_text_md),
+                                                NULL,
                                                 NS_TYPE_CONSENSUS);
   tt_assert(con_md);
   tt_int_op(con_md->flavor,OP_EQ, FLAV_MICRODESC);
@@ -301,7 +305,6 @@ test_router_pick_directory_server_impl(void *arg)
   tt_assert(!networkstatus_consensus_is_bootstrapping(con_md->valid_until
                                                       + 24*60*60));
   /* These times are outside the test validity period */
-  tt_assert(networkstatus_consensus_is_bootstrapping(now));
   tt_assert(networkstatus_consensus_is_bootstrapping(now + 2*24*60*60));
   tt_assert(networkstatus_consensus_is_bootstrapping(now - 2*24*60*60));
 
@@ -475,7 +478,9 @@ test_directory_guard_fetch_with_no_dirinfo(void *arg)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -626,7 +631,7 @@ mock_clock_skew_warning(const connection_t *conn, long apparent_skew,
   (void)conn;
   mock_apparent_skew = apparent_skew;
   tt_int_op(trusted, OP_EQ, 1);
-  tt_int_op(domain, OP_EQ, LD_GENERAL);
+  tt_i64_op(domain, OP_EQ, LD_GENERAL);
   tt_str_op(received, OP_EQ, "microdesc flavor consensus");
   tt_str_op(source, OP_EQ, "CONSENSUS");
  done:
@@ -648,7 +653,9 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
 
   /* Initialize the SRV subsystem */
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   sr_init(0);
   UNMOCK(get_my_v3_authority_cert);
 
@@ -662,7 +669,8 @@ test_skew_common(void *arg, time_t now, unsigned long *offset)
   MOCK(clock_skew_warning, mock_clock_skew_warning);
   /* Caller will call teardown_capture_of_logs() */
   setup_capture_of_logs(LOG_WARN);
-  retval = networkstatus_set_current_consensus(consensus, "microdesc", 0,
+  retval = networkstatus_set_current_consensus(consensus, strlen(consensus),
+                                               "microdesc", 0,
                                                NULL);
 
  done:
diff --git a/src/test/test_routerset.c b/src/test/test_routerset.c
index c45f0e1595..892ac6e210 100644
--- a/src/test/test_routerset.c
+++ b/src/test/test_routerset.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define ROUTERSET_PRIVATE
@@ -18,17 +18,13 @@
 
 #include "test/test.h"
 
-#define NS_MODULE routerset
-
-#define NS_SUBMODULE routerset_new
-
 /*
  * Functional (blackbox) test to determine that each member of the routerset
  * is non-NULL
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_new(void *arg)
 {
   routerset_t *rs;
   (void)arg;
@@ -46,15 +42,12 @@ NS(test_main)(void *arg)
     routerset_free(rs);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_get_countryname
-
 /*
  * Functional test to strip the braces from a "{xx}" country code string.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_countryname(void *arg)
 {
   const char *input;
   char *name;
@@ -91,257 +84,272 @@ NS(test_main)(void *arg)
     tor_free(name);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, geoip_not_loaded)
-
 /*
  * Structural (whitebox) test for routerset_refresh_counties, when the GeoIP DB
  * is not loaded.
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
+static int rset_refresh_geoip_not_loaded_geoip_is_loaded(sa_family_t family);
+static int rset_refresh_geoip_not_loaded_geoip_is_loaded_called = 0;
+static int rset_refresh_geoip_not_loaded_geoip_get_n_countries(void);
+static int rset_refresh_geoip_not_loaded_geoip_get_n_countries_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_geoip_not_loaded(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
+  MOCK(geoip_is_loaded,
+       rset_refresh_geoip_not_loaded_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_geoip_not_loaded_geoip_get_n_countries);
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_EQ, NULL);
   tt_int_op(set->n_countries, OP_EQ, 0);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 0);
+  tt_int_op(rset_refresh_geoip_not_loaded_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_geoip_not_loaded_geoip_get_n_countries_called,
+            OP_EQ, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_geoip_not_loaded_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_geoip_not_loaded_geoip_is_loaded_called++;
 
   return 0;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_geoip_not_loaded_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_geoip_not_loaded_geoip_get_n_countries_called++;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, no_countries)
-
 /*
  * Structural test for routerset_refresh_counties, when there are no countries.
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static int rset_refresh_no_countries_geoip_is_loaded(sa_family_t family);
+static int rset_refresh_no_countries_geoip_is_loaded_called = 0;
+static int rset_refresh_no_countries_geoip_get_n_countries(void);
+static int rset_refresh_no_countries_geoip_get_n_countries_called = 0;
+static country_t rset_refresh_no_countries_geoip_get_country(
+                                                    const char *country);
+static int rset_refresh_no_countries_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_no_countries(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_refresh_no_countries_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_no_countries_geoip_get_n_countries);
+  MOCK(geoip_get_country,
+       rset_refresh_no_countries_geoip_get_country);
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_NE, NULL);
   tt_int_op(set->n_countries, OP_EQ, 1);
   tt_int_op((unsigned int)(*set->countries), OP_EQ, 0);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country), OP_EQ, 0);
+  tt_int_op(rset_refresh_no_countries_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_no_countries_geoip_get_n_countries_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_no_countries_geoip_get_country_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
-    NS_UNMOCK(geoip_get_country);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_get_country);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_no_countries_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_no_countries_geoip_is_loaded_called++;
 
   return 1;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_no_countries_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_no_countries_geoip_get_n_countries_called++;
 
   return 1;
 }
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+rset_refresh_no_countries_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  rset_refresh_no_countries_geoip_get_country_called++;
 
   return 1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, one_valid_country)
-
 /*
  * Structural test for routerset_refresh_counties, with one valid country.
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static int rset_refresh_one_valid_country_geoip_is_loaded(sa_family_t family);
+static int rset_refresh_one_valid_country_geoip_is_loaded_called = 0;
+static int rset_refresh_one_valid_country_geoip_get_n_countries(void);
+static int rset_refresh_one_valid_country_geoip_get_n_countries_called = 0;
+static country_t rset_refresh_one_valid_country_geoip_get_country(
+                                                    const char *country);
+static int rset_refresh_one_valid_country_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_one_valid_country(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_refresh_one_valid_country_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_one_valid_country_geoip_get_n_countries);
+  MOCK(geoip_get_country,
+       rset_refresh_one_valid_country_geoip_get_country);
   smartlist_add(set->country_names, tor_strndup("foo", 3));
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_NE, NULL);
   tt_int_op(set->n_countries, OP_EQ, 2);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country), OP_EQ, 1);
+  tt_int_op(rset_refresh_one_valid_country_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_one_valid_country_geoip_get_n_countries_called,
+            OP_EQ, 1);
+  tt_int_op(rset_refresh_one_valid_country_geoip_get_country_called, OP_EQ, 1);
   tt_int_op((unsigned int)(*set->countries), OP_NE, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
-    NS_UNMOCK(geoip_get_country);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_get_country);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_one_valid_country_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_one_valid_country_geoip_is_loaded_called++;
 
   return 1;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_one_valid_country_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_one_valid_country_geoip_get_n_countries_called++;
 
   return 2;
 }
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+rset_refresh_one_valid_country_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  rset_refresh_one_valid_country_geoip_get_country_called++;
 
   return 1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_refresh_counties, one_invalid_country)
-
 /*
  * Structural test for routerset_refresh_counties, with one invalid
  * country code..
  */
 
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
-NS_DECL(int, geoip_get_n_countries, (void));
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static int rset_refresh_one_invalid_country_geoip_is_loaded(
+                                               sa_family_t family);
+static int rset_refresh_one_invalid_country_geoip_is_loaded_called = 0;
+static int rset_refresh_one_invalid_country_geoip_get_n_countries(void);
+static int rset_refresh_one_invalid_country_geoip_get_n_countries_called = 0;
+static country_t rset_refresh_one_invalid_country_geoip_get_country(
+                                               const char *country);
+static int rset_refresh_one_invalid_country_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_refresh_one_invalid_country(void *arg)
 {
   routerset_t *set = routerset_new();
   (void)arg;
 
-  NS_MOCK(geoip_is_loaded);
-  NS_MOCK(geoip_get_n_countries);
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_refresh_one_invalid_country_geoip_is_loaded);
+  MOCK(geoip_get_n_countries,
+       rset_refresh_one_invalid_country_geoip_get_n_countries);
+  MOCK(geoip_get_country,
+       rset_refresh_one_invalid_country_geoip_get_country);
   smartlist_add(set->country_names, tor_strndup("foo", 3));
 
   routerset_refresh_countries(set);
 
   tt_ptr_op(set->countries, OP_NE, NULL);
   tt_int_op(set->n_countries, OP_EQ, 2);
-  tt_int_op(CALLED(geoip_is_loaded), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_n_countries), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country), OP_EQ, 1);
+  tt_int_op(rset_refresh_one_invalid_country_geoip_is_loaded_called, OP_EQ, 1);
+  tt_int_op(rset_refresh_one_invalid_country_geoip_get_n_countries_called,
+            OP_EQ, 1);
+  tt_int_op(rset_refresh_one_invalid_country_geoip_get_country_called,
+            OP_EQ, 1);
   tt_int_op((unsigned int)(*set->countries), OP_EQ, 0);
 
   done:
-    NS_UNMOCK(geoip_is_loaded);
-    NS_UNMOCK(geoip_get_n_countries);
-    NS_UNMOCK(geoip_get_country);
+    UNMOCK(geoip_is_loaded);
+    UNMOCK(geoip_get_n_countries);
+    UNMOCK(geoip_get_country);
     routerset_free(set);
 }
 
 static int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_refresh_one_invalid_country_geoip_is_loaded(sa_family_t family)
 {
   (void)family;
-  CALLED(geoip_is_loaded)++;
+  rset_refresh_one_invalid_country_geoip_is_loaded_called++;
 
   return 1;
 }
 
 static int
-NS(geoip_get_n_countries)(void)
+rset_refresh_one_invalid_country_geoip_get_n_countries(void)
 {
-  CALLED(geoip_get_n_countries)++;
+  rset_refresh_one_invalid_country_geoip_get_n_countries_called++;
 
   return 2;
 }
 
 static country_t
-NS(geoip_get_country)(const char *countrycode)
+rset_refresh_one_invalid_country_geoip_get_country(const char *countrycode)
 {
   (void)countrycode;
-  CALLED(geoip_get_country)++;
+  rset_refresh_one_invalid_country_geoip_get_country_called++;
 
   return -1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, malformed)
-
 /*
  * Functional test, with a malformed string to parse.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_malformed(void *arg)
 {
   routerset_t *set = routerset_new();
   const char *s = "_";
@@ -356,16 +364,13 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, valid_hexdigest)
-
 /*
  * Functional test for routerset_parse, that routerset_parse returns 0
  * on a valid hexdigest entry.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_valid_hexdigest(void *arg)
 {
   routerset_t *set;
   const char *s;
@@ -382,15 +387,12 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, valid_nickname)
-
 /*
  * Functional test for routerset_parse, when given a valid nickname as input.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_valid_nickname(void *arg)
 {
   routerset_t *set;
   const char *s;
@@ -407,15 +409,12 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, get_countryname)
-
 /*
  * Functional test for routerset_parse, when given a valid countryname.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_get_countryname(void *arg)
 {
   routerset_t *set;
   const char *s;
@@ -432,158 +431,158 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, policy_wildcard)
-
 /*
  * Structural test for routerset_parse, when given a valid wildcard policy.
  */
 
-NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-    (const char *s, int assume_action, int *malformed_list));
+static addr_policy_t * rset_parse_policy_wildcard_parse_item_from_string(
+               const char *s, int assume_action, int *malformed_list);
+static int rset_parse_policy_wildcard_parse_item_from_string_called = 0;
 
-static addr_policy_t *NS(mock_addr_policy);
+static addr_policy_t *rset_parse_policy_wildcard_mock_addr_policy;
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_policy_wildcard(void *arg)
 {
   routerset_t *set;
   const char *s;
   int r;
   (void)arg;
 
-  NS_MOCK(router_parse_addr_policy_item_from_string);
-  NS(mock_addr_policy) = tor_malloc_zero(sizeof(addr_policy_t));
+  MOCK(router_parse_addr_policy_item_from_string,
+       rset_parse_policy_wildcard_parse_item_from_string);
+  rset_parse_policy_wildcard_mock_addr_policy =
+    tor_malloc_zero(sizeof(addr_policy_t));
 
   set = routerset_new();
   s = "*";
   r = routerset_parse(set, s, "");
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(smartlist_len(set->policies), OP_NE, 0);
-  tt_int_op(CALLED(router_parse_addr_policy_item_from_string), OP_EQ, 1);
+  tt_int_op(rset_parse_policy_wildcard_parse_item_from_string_called,
+            OP_EQ, 1);
 
   done:
     routerset_free(set);
 }
 
 addr_policy_t *
-NS(router_parse_addr_policy_item_from_string)(const char *s,
+rset_parse_policy_wildcard_parse_item_from_string(const char *s,
                                               int assume_action,
                                               int *malformed_list)
 {
   (void)s;
   (void)assume_action;
   (void)malformed_list;
-  CALLED(router_parse_addr_policy_item_from_string)++;
+  rset_parse_policy_wildcard_parse_item_from_string_called++;
 
-  return NS(mock_addr_policy);
+  return rset_parse_policy_wildcard_mock_addr_policy;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, policy_ipv4)
-
 /*
  * Structural test for routerset_parse, when given a valid IPv4 address
  * literal policy.
  */
 
-NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-        (const char *s, int assume_action, int *bogus));
+static addr_policy_t * rset_parse_policy_ipv4_parse_item_from_string(
+              const char *s, int assume_action, int *bogus);
+static int rset_parse_policy_ipv4_parse_item_from_string_called = 0;
 
-static addr_policy_t *NS(mock_addr_policy);
+static addr_policy_t *rset_parse_policy_ipv4_mock_addr_policy;
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_policy_ipv4(void *arg)
 {
   routerset_t *set;
   const char *s;
   int r;
   (void)arg;
 
-  NS_MOCK(router_parse_addr_policy_item_from_string);
-  NS(mock_addr_policy) = tor_malloc_zero(sizeof(addr_policy_t));
+  MOCK(router_parse_addr_policy_item_from_string,
+       rset_parse_policy_ipv4_parse_item_from_string);
+  rset_parse_policy_ipv4_mock_addr_policy =
+    tor_malloc_zero(sizeof(addr_policy_t));
 
   set = routerset_new();
   s = "127.0.0.1";
   r = routerset_parse(set, s, "");
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(smartlist_len(set->policies), OP_NE, 0);
-  tt_int_op(CALLED(router_parse_addr_policy_item_from_string), OP_EQ, 1);
+  tt_int_op(rset_parse_policy_ipv4_parse_item_from_string_called, OP_EQ, 1);
 
  done:
   routerset_free(set);
 }
 
 addr_policy_t *
-NS(router_parse_addr_policy_item_from_string)(const char *s, int assume_action,
-                                              int *bogus)
+rset_parse_policy_ipv4_parse_item_from_string(
+                                      const char *s, int assume_action,
+                                      int *bogus)
 {
   (void)s;
   (void)assume_action;
-  CALLED(router_parse_addr_policy_item_from_string)++;
+  rset_parse_policy_ipv4_parse_item_from_string_called++;
   *bogus = 0;
 
-  return NS(mock_addr_policy);
+  return rset_parse_policy_ipv4_mock_addr_policy;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_parse, policy_ipv6)
-
 /*
  * Structural test for routerset_parse, when given a valid IPv6 address
  * literal policy.
  */
 
-NS_DECL(addr_policy_t *, router_parse_addr_policy_item_from_string,
-        (const char *s, int assume_action, int *bad));
+static addr_policy_t * rset_parse_policy_ipv6_parse_item_from_string(
+               const char *s, int assume_action, int *bad);
+static int rset_parse_policy_ipv6_parse_item_from_string_called = 0;
 
-static addr_policy_t *NS(mock_addr_policy);
+static addr_policy_t *rset_parse_policy_ipv6_mock_addr_policy;
 
 static void
-NS(test_main)(void *arg)
+test_rset_parse_policy_ipv6(void *arg)
 {
   routerset_t *set;
   const char *s;
   int r;
   (void)arg;
 
-  NS_MOCK(router_parse_addr_policy_item_from_string);
-  NS(mock_addr_policy) = tor_malloc_zero(sizeof(addr_policy_t));
+  MOCK(router_parse_addr_policy_item_from_string,
+       rset_parse_policy_ipv6_parse_item_from_string);
+  rset_parse_policy_ipv6_mock_addr_policy =
+    tor_malloc_zero(sizeof(addr_policy_t));
 
   set = routerset_new();
   s = "::1";
   r = routerset_parse(set, s, "");
   tt_int_op(r, OP_EQ, 0);
   tt_int_op(smartlist_len(set->policies), OP_NE, 0);
-  tt_int_op(CALLED(router_parse_addr_policy_item_from_string), OP_EQ, 1);
+  tt_int_op(rset_parse_policy_ipv6_parse_item_from_string_called, OP_EQ, 1);
 
  done:
   routerset_free(set);
 }
 
 addr_policy_t *
-NS(router_parse_addr_policy_item_from_string)(const char *s,
+rset_parse_policy_ipv6_parse_item_from_string(const char *s,
                                               int assume_action, int *bad)
 {
   (void)s;
   (void)assume_action;
-  CALLED(router_parse_addr_policy_item_from_string)++;
+  rset_parse_policy_ipv6_parse_item_from_string_called++;
   *bad = 0;
 
-  return NS(mock_addr_policy);
+  return rset_parse_policy_ipv6_mock_addr_policy;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_union, source_bad)
-
 /*
  * Structural test for routerset_union, when given a bad source argument.
  */
 
-NS_DECL(smartlist_t *, smartlist_new, (void));
+static smartlist_t * rset_union_source_bad_smartlist_new(void);
+static int rset_union_source_bad_smartlist_new_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_union_source_bad(void *arg)
 {
   routerset_t *set, *bad_set;
   (void)arg;
@@ -593,16 +592,17 @@ NS(test_main)(void *arg)
   smartlist_free(bad_set->list);
   bad_set->list = NULL;
 
-  NS_MOCK(smartlist_new);
+  MOCK(smartlist_new,
+       rset_union_source_bad_smartlist_new);
 
   routerset_union(set, NULL);
-  tt_int_op(CALLED(smartlist_new), OP_EQ, 0);
+  tt_int_op(rset_union_source_bad_smartlist_new_called, OP_EQ, 0);
 
   routerset_union(set, bad_set);
-  tt_int_op(CALLED(smartlist_new), OP_EQ, 0);
+  tt_int_op(rset_union_source_bad_smartlist_new_called, OP_EQ, 0);
 
   done:
-    NS_UNMOCK(smartlist_new);
+    UNMOCK(smartlist_new);
     routerset_free(set);
 
     /* Just recreate list, so we can simply use routerset_free. */
@@ -611,22 +611,19 @@ NS(test_main)(void *arg)
 }
 
 static smartlist_t *
-NS(smartlist_new)(void)
+rset_union_source_bad_smartlist_new(void)
 {
-  CALLED(smartlist_new)++;
+  rset_union_source_bad_smartlist_new_called++;
 
   return NULL;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_union, one)
-
 /*
  * Functional test for routerset_union.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_union_one(void *arg)
 {
   routerset_t *src = routerset_new();
   routerset_t *tgt;
@@ -643,15 +640,12 @@ NS(test_main)(void *arg)
     routerset_free(tgt);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_is_list
-
 /*
  * Functional tests for routerset_is_list.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_is_list(void *arg)
 {
   routerset_t *set;
   addr_policy_t *policy;
@@ -696,15 +690,12 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_needs_geoip
-
 /*
  * Functional tests for routerset_needs_geoip.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_needs_geoip(void *arg)
 {
   routerset_t *set;
   int needs_geoip;
@@ -731,15 +722,12 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_is_empty
-
 /*
  * Functional tests for routerset_is_empty.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_is_empty(void *arg)
 {
   routerset_t *set = NULL;
   int is_empty;
@@ -765,16 +753,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, null_set_or_null_set_list)
-
 /*
  * Functional test for routerset_contains, when given a NULL set or the
  * set has a NULL list.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_set_or_list(void *arg)
 {
   routerset_t *set = NULL;
   int contains;
@@ -794,16 +779,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_null_nickname)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset but a
  * NULL nickname.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_nickname(void *arg)
 {
   routerset_t *set = routerset_new();
   char *nickname = NULL;
@@ -819,16 +801,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_nickname)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the nickname is in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_nickname(void *arg)
 {
   routerset_t *set = routerset_new();
   const char *nickname;
@@ -845,16 +824,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_no_nickname)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the nickname is not in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_no_nickname(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -869,16 +845,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_digest)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the digest is contained in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_digest(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -894,16 +867,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_no_digest)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the digest is not contained in the routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_no_digest(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -920,16 +890,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_null_digest)
-
 /*
  * Functional test for routerset_contains, when given a valid routerset
  * and the digest is NULL.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_digest(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
@@ -945,34 +912,34 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_addr)
-
 /*
  * Structural test for routerset_contains, when given a valid routerset
  * and the address is rejected by policy.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
+static addr_policy_result_t rset_contains_addr_cmp_addr_to_policy(
+              const tor_addr_t *addr, uint16_t port,
+              const smartlist_t *policy);
+static int rset_contains_addr_cmp_addr_to_policy_called = 0;
 
 static tor_addr_t MOCK_TOR_ADDR;
 #define MOCK_TOR_ADDR_PTR (&MOCK_TOR_ADDR)
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_addr(void *arg)
 {
   routerset_t *set = routerset_new();
   tor_addr_t *addr = MOCK_TOR_ADDR_PTR;
   int contains;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_contains_addr_cmp_addr_to_policy);
 
   contains = routerset_contains(set, addr, 0, NULL, NULL, 0);
   routerset_free(set);
 
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
+  tt_int_op(rset_contains_addr_cmp_addr_to_policy_called, OP_EQ, 1);
   tt_int_op(contains, OP_EQ, 3);
 
   done:
@@ -980,12 +947,12 @@ NS(test_main)(void *arg)
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_contains_addr_cmp_addr_to_policy(const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_contains_addr_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
   return ADDR_POLICY_REJECTED;
 
@@ -993,31 +960,31 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_no_addr)
-
 /*
  * Structural test for routerset_contains, when given a valid routerset
  * and the address is not rejected by policy.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
+static addr_policy_result_t rset_contains_no_addr_cmp_addr_to_policy(
+              const tor_addr_t *addr, uint16_t port,
+              const smartlist_t *policy);
+static int rset_contains_no_addr_cmp_addr_to_policy_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_no_addr(void *arg)
 {
   routerset_t *set = routerset_new();
   tor_addr_t *addr = MOCK_TOR_ADDR_PTR;
   int contains;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_contains_no_addr_cmp_addr_to_policy);
 
   contains = routerset_contains(set, addr, 0, NULL, NULL, 0);
   routerset_free(set);
 
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
+  tt_int_op(rset_contains_no_addr_cmp_addr_to_policy_called, OP_EQ, 1);
   tt_int_op(contains, OP_EQ, 0);
 
   done:
@@ -1025,12 +992,12 @@ NS(test_main)(void *arg)
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_contains_no_addr_cmp_addr_to_policy(const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_contains_no_addr_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   return ADDR_POLICY_ACCEPTED;
@@ -1039,25 +1006,25 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
       return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, set_and_null_addr)
-
 /*
  * Structural test for routerset_contains, when given a valid routerset
  * and the address is NULL.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
+static addr_policy_result_t rset_contains_null_addr_cmp_addr_to_policy(
+                 const tor_addr_t *addr, uint16_t port,
+                 const smartlist_t *policy);
+static int rset_contains_null_addr_cmp_addr_to_policy_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_null_addr(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_contains_null_addr_cmp_addr_to_policy);
 
   contains = routerset_contains(set, NULL, 0, NULL, NULL, 0);
   routerset_free(set);
@@ -1069,12 +1036,13 @@ NS(test_main)(void *arg)
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_contains_null_addr_cmp_addr_to_policy(
+    const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_contains_null_addr_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   return ADDR_POLICY_ACCEPTED;
@@ -1083,27 +1051,30 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, countries_no_geoip)
-
 /*
  * Structural test for routerset_contains, when there is no matching country
  * for the address.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
-NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+static addr_policy_result_t rset_countries_no_geoip_cmp_addr_to_policy(
+                  const tor_addr_t *addr, uint16_t port,
+                  const smartlist_t *policy);
+static int rset_countries_no_geoip_cmp_addr_to_policy_called = 0;
+static int rset_countries_no_geoip_geoip_get_country_by_addr(
+                  const tor_addr_t *addr);
+static int rset_countries_no_geoip_geoip_get_country_by_addr_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_countries_no_geoip(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains = 1;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
-  NS_MOCK(geoip_get_country_by_addr);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_countries_no_geoip_cmp_addr_to_policy);
+  MOCK(geoip_get_country_by_addr,
+       rset_countries_no_geoip_geoip_get_country_by_addr);
 
   set->countries = bitarray_init_zero(1);
   bitarray_set(set->countries, 1);
@@ -1111,20 +1082,23 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(contains, OP_EQ, 0);
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country_by_addr), OP_EQ, 1);
+  tt_int_op(rset_countries_no_geoip_cmp_addr_to_policy_called,
+            OP_EQ, 1);
+  tt_int_op(rset_countries_no_geoip_geoip_get_country_by_addr_called,
+            OP_EQ, 1);
 
   done:
     ;
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_countries_no_geoip_cmp_addr_to_policy(
+    const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_countries_no_geoip_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
@@ -1132,36 +1106,39 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
 }
 
 int
-NS(geoip_get_country_by_addr)(const tor_addr_t *addr)
+rset_countries_no_geoip_geoip_get_country_by_addr(const tor_addr_t *addr)
 {
-  CALLED(geoip_get_country_by_addr)++;
+  rset_countries_no_geoip_geoip_get_country_by_addr_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
     return -1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains, countries_geoip)
-
 /*
  * Structural test for routerset_contains, when there a matching country
  * for the address.
  */
 
-NS_DECL(addr_policy_result_t, compare_tor_addr_to_addr_policy,
-    (const tor_addr_t *addr, uint16_t port, const smartlist_t *policy));
-NS_DECL(int, geoip_get_country_by_addr, (const tor_addr_t *addr));
+static addr_policy_result_t rset_countries_geoip_cmp_addr_to_policy(
+              const tor_addr_t *addr, uint16_t port,
+              const smartlist_t *policy);
+static int rset_countries_geoip_cmp_addr_to_policy_called = 0;
+static int rset_countries_geoip_geoip_get_country_by_addr(
+                                          const tor_addr_t *addr);
+static int rset_countries_geoip_geoip_get_country_by_addr_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_countries_geoip(void *arg)
 {
   routerset_t *set = routerset_new();
   int contains = 1;
   (void)arg;
 
-  NS_MOCK(compare_tor_addr_to_addr_policy);
-  NS_MOCK(geoip_get_country_by_addr);
+  MOCK(compare_tor_addr_to_addr_policy,
+       rset_countries_geoip_cmp_addr_to_policy);
+  MOCK(geoip_get_country_by_addr,
+       rset_countries_geoip_geoip_get_country_by_addr);
 
   set->n_countries = 2;
   set->countries = bitarray_init_zero(1);
@@ -1170,20 +1147,24 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(contains, OP_EQ, 2);
-  tt_int_op(CALLED(compare_tor_addr_to_addr_policy), OP_EQ, 1);
-  tt_int_op(CALLED(geoip_get_country_by_addr), OP_EQ, 1);
+  tt_int_op(
+     rset_countries_geoip_cmp_addr_to_policy_called,
+     OP_EQ, 1);
+  tt_int_op(rset_countries_geoip_geoip_get_country_by_addr_called,
+            OP_EQ, 1);
 
   done:
     ;
 }
 
 addr_policy_result_t
-NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
+rset_countries_geoip_cmp_addr_to_policy(
+    const tor_addr_t *addr, uint16_t port,
     const smartlist_t *policy)
 {
   (void)port;
   (void)policy;
-  CALLED(compare_tor_addr_to_addr_policy)++;
+  rset_countries_geoip_cmp_addr_to_policy_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
@@ -1191,25 +1172,22 @@ NS(compare_tor_addr_to_addr_policy)(const tor_addr_t *addr, uint16_t port,
 }
 
 int
-NS(geoip_get_country_by_addr)(const tor_addr_t *addr)
+rset_countries_geoip_geoip_get_country_by_addr(const tor_addr_t *addr)
 {
-  CALLED(geoip_get_country_by_addr)++;
+  rset_countries_geoip_geoip_get_country_by_addr_called++;
   tt_ptr_op(addr, OP_EQ, MOCK_TOR_ADDR_PTR);
 
   done:
     return 1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, only_flag_and_no_ccs)
-
 /*
  * Functional test for routerset_add_unknown_ccs, where only_if_some_cc_set
  * is set and there are no country names.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_only_flag(void *arg)
 {
   routerset_t *set = routerset_new();
   routerset_t **setp = &set;
@@ -1224,26 +1202,26 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, creates_set)
-
 /*
  * Functional test for routerset_add_unknown_ccs, where the set argument
  * is created if passed in as NULL.
  */
 
 /* The mock is only used to stop the test from asserting erroneously. */
-NS_DECL(country_t, geoip_get_country, (const char *country));
+static country_t rset_add_unknown_ccs_creates_set_geoip_get_country(
+                                                     const char *country);
+static int rset_add_unknown_ccs_creates_set_geoip_get_country_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_creates_set(void *arg)
 {
   routerset_t *set = NULL;
   routerset_t **setp = &set;
   int r;
   (void)arg;
 
-  NS_MOCK(geoip_get_country);
+  MOCK(geoip_get_country,
+       rset_add_unknown_ccs_creates_set_geoip_get_country);
 
   r = routerset_add_unknown_ccs(setp, 0);
 
@@ -1256,35 +1234,38 @@ NS(test_main)(void *arg)
 }
 
 country_t
-NS(geoip_get_country)(const char *country)
+rset_add_unknown_ccs_creates_set_geoip_get_country(const char *country)
 {
   (void)country;
-  CALLED(geoip_get_country)++;
+  rset_add_unknown_ccs_creates_set_geoip_get_country_called++;
 
   return -1;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, add_unknown)
-
 /*
  * Structural test for routerset_add_unknown_ccs, that the "{??}"
  * country code is added to the list.
  */
 
-NS_DECL(country_t, geoip_get_country, (const char *country));
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
+static country_t rset_add_unknown_ccs_add_unknown_geoip_get_country(
+                                                 const char *country);
+static int rset_add_unknown_ccs_add_unknown_geoip_get_country_called = 0;
+static int rset_add_unknown_ccs_add_unknown_geoip_is_loaded(
+                                                 sa_family_t family);
+static int rset_add_unknown_ccs_add_unknown_geoip_is_loaded_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_add_unknown(void *arg)
 {
   routerset_t *set = routerset_new();
   routerset_t **setp = &set;
   int r;
   (void)arg;
 
-  NS_MOCK(geoip_get_country);
-  NS_MOCK(geoip_is_loaded);
+  MOCK(geoip_get_country,
+       rset_add_unknown_ccs_add_unknown_geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_add_unknown_ccs_add_unknown_geoip_is_loaded);
 
   r = routerset_add_unknown_ccs(setp, 0);
 
@@ -1298,11 +1279,11 @@ NS(test_main)(void *arg)
 }
 
 country_t
-NS(geoip_get_country)(const char *country)
+rset_add_unknown_ccs_add_unknown_geoip_get_country(const char *country)
 {
   int arg_is_qq, arg_is_a1;
 
-  CALLED(geoip_get_country)++;
+  rset_add_unknown_ccs_add_unknown_geoip_get_country_called++;
 
   arg_is_qq = !strcmp(country, "??");
   arg_is_a1 = !strcmp(country, "A1");
@@ -1317,9 +1298,9 @@ NS(geoip_get_country)(const char *country)
 }
 
 int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_add_unknown_ccs_add_unknown_geoip_is_loaded(sa_family_t family)
 {
-  CALLED(geoip_is_loaded)++;
+  rset_add_unknown_ccs_add_unknown_geoip_is_loaded_called++;
 
   tt_int_op(family, OP_EQ, AF_INET);
 
@@ -1327,27 +1308,29 @@ NS(geoip_is_loaded)(sa_family_t family)
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_add_unknown_ccs, add_a1)
-
 /*
  * Structural test for routerset_add_unknown_ccs, that the "{a1}"
  * country code is added to the list.
  */
 
-NS_DECL(country_t, geoip_get_country, (const char *country));
-NS_DECL(int, geoip_is_loaded, (sa_family_t family));
+static country_t rset_add_unknown_ccs_add_a1_geoip_get_country(
+                                           const char *country);
+static int rset_add_unknown_ccs_add_a1_geoip_get_country_called = 0;
+static int rset_add_unknown_ccs_add_a1_geoip_is_loaded(sa_family_t family);
+static int rset_add_unknown_ccs_add_a1_geoip_is_loaded_called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_add_unknown_ccs_add_a1(void *arg)
 {
   routerset_t *set = routerset_new();
   routerset_t **setp = &set;
   int r;
   (void)arg;
 
-  NS_MOCK(geoip_get_country);
-  NS_MOCK(geoip_is_loaded);
+  MOCK(geoip_get_country,
+       rset_add_unknown_ccs_add_a1_geoip_get_country);
+  MOCK(geoip_is_loaded,
+       rset_add_unknown_ccs_add_a1_geoip_is_loaded);
 
   r = routerset_add_unknown_ccs(setp, 0);
 
@@ -1361,11 +1344,11 @@ NS(test_main)(void *arg)
 }
 
 country_t
-NS(geoip_get_country)(const char *country)
+rset_add_unknown_ccs_add_a1_geoip_get_country(const char *country)
 {
   int arg_is_qq, arg_is_a1;
 
-  CALLED(geoip_get_country)++;
+  rset_add_unknown_ccs_add_a1_geoip_get_country_called++;
 
   arg_is_qq = !strcmp(country, "??");
   arg_is_a1 = !strcmp(country, "A1");
@@ -1380,9 +1363,9 @@ NS(geoip_get_country)(const char *country)
 }
 
 int
-NS(geoip_is_loaded)(sa_family_t family)
+rset_add_unknown_ccs_add_a1_geoip_is_loaded(sa_family_t family)
 {
-  CALLED(geoip_is_loaded)++;
+  rset_add_unknown_ccs_add_a1_geoip_is_loaded_called++;
 
   tt_int_op(family, OP_EQ, AF_INET);
 
@@ -1390,15 +1373,12 @@ NS(geoip_is_loaded)(sa_family_t family)
     return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_contains_extendinfo
-
 /*
  * Functional test for routerset_contains_extendinfo.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_extendinfo(void *arg)
 {
   routerset_t *set = routerset_new();
   extend_info_t ei;
@@ -1418,15 +1398,12 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_contains_router
-
 /*
  * Functional test for routerset_contains_router.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_router(void *arg)
 {
   routerset_t *set = routerset_new();
   routerinfo_t ri;
@@ -1446,9 +1423,6 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_contains_routerstatus
-
 /*
  * Functional test for routerset_contains_routerstatus.
  */
@@ -1458,7 +1432,7 @@ NS(test_main)(void *arg)
 // a bit more or test a bit more.
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_routerstatus(void *arg)
 {
   routerset_t *set = routerset_new();
   routerstatus_t rs;
@@ -1479,46 +1453,41 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains_node, none)
-
 /*
  * Functional test for routerset_contains_node, when the node has no
  * routerset or routerinfo.
  */
 
-static node_t NS(mock_node);
+static node_t rset_contains_none_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_none(void *arg)
 {
   routerset_t *set = routerset_new();
   int r;
   (void)arg;
 
-  memset(&NS(mock_node), 0, sizeof(NS(mock_node)));
-  NS(mock_node).ri = NULL;
-  NS(mock_node).rs = NULL;
+  memset(&rset_contains_none_mock_node, 0,
+         sizeof(rset_contains_none_mock_node));
+  rset_contains_none_mock_node.ri = NULL;
+  rset_contains_none_mock_node.rs = NULL;
 
-  r = routerset_contains_node(set, &NS(mock_node));
+  r = routerset_contains_node(set, &rset_contains_none_mock_node);
   tt_int_op(r, OP_EQ, 0);
 
   done:
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains_node, routerstatus)
-
 /*
  * Functional test for routerset_contains_node, when the node has a
  * routerset and no routerinfo.
  */
 
-static node_t NS(mock_node);
+static node_t rset_contains_rs_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_rs(void *arg)
 {
   routerset_t *set = routerset_new();
   int r;
@@ -1530,27 +1499,24 @@ NS(test_main)(void *arg)
 
   strncpy(rs.nickname, nickname, sizeof(rs.nickname) - 1);
   rs.nickname[sizeof(rs.nickname) - 1] = '\0';
-  memset(&NS(mock_node), 0, sizeof(NS(mock_node)));
-  NS(mock_node).ri = NULL;
-  NS(mock_node).rs = &rs;
+  memset(&rset_contains_rs_mock_node, 0, sizeof(rset_contains_rs_mock_node));
+  rset_contains_rs_mock_node.ri = NULL;
+  rset_contains_rs_mock_node.rs = &rs;
 
-  r = routerset_contains_node(set, &NS(mock_node));
+  r = routerset_contains_node(set, &rset_contains_rs_mock_node);
 
   tt_int_op(r, OP_EQ, 4);
   done:
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_contains_node, routerinfo)
-
 /*
  * Functional test for routerset_contains_node, when the node has no
  * routerset and a routerinfo.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_contains_routerinfo(void *arg)
 {
   routerset_t *set = routerset_new();
   int r;
@@ -1573,16 +1539,13 @@ NS(test_main)(void *arg)
     routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, no_routerset)
-
 /*
  * Functional test for routerset_get_all_nodes, when routerset is NULL or
  * the routerset list is NULL.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_no_routerset(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = NULL;
@@ -1606,30 +1569,29 @@ NS(test_main)(void *arg)
   smartlist_free(out);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, list_with_no_nodes)
-
 /*
  * Structural test for routerset_get_all_nodes, when the routerset list
  * is empty.
  */
 
-NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, unsigned flags));
-static const char *NS(mock_nickname);
+static const node_t * rset_get_all_l_no_nodes_node_get_by_nickname(
+                                       const char *nickname, unsigned flags);
+static int rset_get_all_l_no_nodes_node_get_by_nickname_called = 0;
+static const char *rset_get_all_l_no_nodes_mock_nickname;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_l_no_nodes(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = routerset_new();
   int out_len;
   (void)arg;
 
-  NS_MOCK(node_get_by_nickname);
+  MOCK(node_get_by_nickname,
+       rset_get_all_l_no_nodes_node_get_by_nickname);
 
-  NS(mock_nickname) = "foo";
-  smartlist_add_strdup(set->list, NS(mock_nickname));
+  rset_get_all_l_no_nodes_mock_nickname = "foo";
+  smartlist_add_strdup(set->list, rset_get_all_l_no_nodes_mock_nickname);
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1638,49 +1600,49 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(out_len, OP_EQ, 0);
-  tt_int_op(CALLED(node_get_by_nickname), OP_EQ, 1);
+  tt_int_op(rset_get_all_l_no_nodes_node_get_by_nickname_called, OP_EQ, 1);
 
   done:
     ;
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, unsigned flags)
+rset_get_all_l_no_nodes_node_get_by_nickname(const char *nickname,
+                                             unsigned flags)
 {
-  CALLED(node_get_by_nickname)++;
-  tt_str_op(nickname, OP_EQ, NS(mock_nickname));
+  rset_get_all_l_no_nodes_node_get_by_nickname_called++;
+  tt_str_op(nickname, OP_EQ, rset_get_all_l_no_nodes_mock_nickname);
   tt_uint_op(flags, OP_EQ, 0);
 
   done:
     return NULL;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, list_flag_not_running)
-
 /*
  * Structural test for routerset_get_all_nodes, with the running_only flag
  * is set but the nodes are not running.
  */
 
-NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, unsigned flags));
-static const char *NS(mock_nickname);
-static node_t NS(mock_node);
+static const node_t * rset_get_all_l_not_running_node_get_by_nickname(
+                                       const char *nickname, unsigned flags);
+static int rset_get_all_l_not_running_node_get_by_nickname_called = 0;
+static const char *rset_get_all_l_not_running_mock_nickname;
+static node_t rset_get_all_l_not_running_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_l_not_running(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = routerset_new();
   int out_len;
   (void)arg;
 
-  NS_MOCK(node_get_by_nickname);
+  MOCK(node_get_by_nickname,
+       rset_get_all_l_not_running_node_get_by_nickname);
 
-  NS(mock_node).is_running = 0;
-  NS(mock_nickname) = "foo";
-  smartlist_add_strdup(set->list, NS(mock_nickname));
+  rset_get_all_l_not_running_mock_node.is_running = 0;
+  rset_get_all_l_not_running_mock_nickname = "foo";
+  smartlist_add_strdup(set->list, rset_get_all_l_not_running_mock_nickname);
 
   routerset_get_all_nodes(out, set, NULL, 1);
   out_len = smartlist_len(out);
@@ -1689,37 +1651,36 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(out_len, OP_EQ, 0);
-  tt_int_op(CALLED(node_get_by_nickname), OP_EQ, 1);
+  tt_int_op(rset_get_all_l_not_running_node_get_by_nickname_called, OP_EQ, 1);
 
   done:
     ;
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, unsigned flags)
+rset_get_all_l_not_running_node_get_by_nickname(const char *nickname,
+                                                unsigned flags)
 {
-  CALLED(node_get_by_nickname)++;
-  tt_str_op(nickname, OP_EQ, NS(mock_nickname));
+  rset_get_all_l_not_running_node_get_by_nickname_called++;
+  tt_str_op(nickname, OP_EQ, rset_get_all_l_not_running_mock_nickname);
   tt_int_op(flags, OP_EQ, 0);
 
   done:
-    return &NS(mock_node);
+    return &rset_get_all_l_not_running_mock_node;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, list)
-
 /*
  * Structural test for routerset_get_all_nodes.
  */
 
-NS_DECL(const node_t *, node_get_by_nickname,
-    (const char *nickname, unsigned flags));
-static char *NS(mock_nickname);
-static node_t NS(mock_node);
+static const node_t * rset_get_all_list_node_get_by_nickname(
+                             const char *nickname, unsigned flags);
+static int rset_get_all_list_node_get_by_nickname_called = 0;
+static char *rset_get_all_list_mock_nickname;
+static node_t rset_get_all_list_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_list(void *arg)
 {
   smartlist_t *out = smartlist_new();
   routerset_t *set = routerset_new();
@@ -1727,10 +1688,11 @@ NS(test_main)(void *arg)
   node_t *ent;
   (void)arg;
 
-  NS_MOCK(node_get_by_nickname);
+  MOCK(node_get_by_nickname,
+       rset_get_all_list_node_get_by_nickname);
 
-  NS(mock_nickname) = tor_strdup("foo");
-  smartlist_add(set->list, NS(mock_nickname));
+  rset_get_all_list_mock_nickname = tor_strdup("foo");
+  smartlist_add(set->list, rset_get_all_list_mock_nickname);
 
   routerset_get_all_nodes(out, set, NULL, 0);
   out_len = smartlist_len(out);
@@ -1740,127 +1702,122 @@ NS(test_main)(void *arg)
   routerset_free(set);
 
   tt_int_op(out_len, OP_EQ, 1);
-  tt_ptr_op(ent, OP_EQ, &NS(mock_node));
-  tt_int_op(CALLED(node_get_by_nickname), OP_EQ, 1);
+  tt_ptr_op(ent, OP_EQ, &rset_get_all_list_mock_node);
+  tt_int_op(rset_get_all_list_node_get_by_nickname_called, OP_EQ, 1);
 
   done:
     ;
 }
 
 const node_t *
-NS(node_get_by_nickname)(const char *nickname, unsigned flags)
+rset_get_all_list_node_get_by_nickname(const char *nickname, unsigned flags)
 {
-  CALLED(node_get_by_nickname)++;
-  tt_str_op(nickname, OP_EQ, NS(mock_nickname));
+  rset_get_all_list_node_get_by_nickname_called++;
+  tt_str_op(nickname, OP_EQ, rset_get_all_list_mock_nickname);
   tt_int_op(flags, OP_EQ, 0);
 
   done:
-    return &NS(mock_node);
+    return &rset_get_all_list_mock_node;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, nodelist_with_no_nodes)
-
 /*
  * Structural test for routerset_get_all_nodes, when the nodelist has no nodes.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
-
-static smartlist_t *NS(mock_smartlist);
+static const smartlist_t * rset_get_all_n_no_nodes_nodelist_get_list(void);
+static int rset_get_all_n_no_nodes_nodelist_get_list_called = 0;
 
+static smartlist_t *rset_get_all_n_no_nodes_mock_smartlist;
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_n_no_nodes(void *arg)
 {
   routerset_t *set = routerset_new();
   smartlist_t *out = smartlist_new();
   int r;
   (void)arg;
 
-  NS_MOCK(nodelist_get_list);
+  MOCK(nodelist_get_list,
+       rset_get_all_n_no_nodes_nodelist_get_list);
 
   smartlist_add_strdup(set->country_names, "{xx}");
-  NS(mock_smartlist) = smartlist_new();
+  rset_get_all_n_no_nodes_mock_smartlist = smartlist_new();
 
   routerset_get_all_nodes(out, set, NULL, 1);
   r = smartlist_len(out);
   routerset_free(set);
   smartlist_free(out);
-  smartlist_free(NS(mock_smartlist));
+  smartlist_free(rset_get_all_n_no_nodes_mock_smartlist);
 
   tt_int_op(r, OP_EQ, 0);
-  tt_int_op(CALLED(nodelist_get_list), OP_EQ, 1);
+  tt_int_op(rset_get_all_n_no_nodes_nodelist_get_list_called, OP_EQ, 1);
 
   done:
     ;
 }
 
-smartlist_t *
-NS(nodelist_get_list)(void)
+const smartlist_t *
+rset_get_all_n_no_nodes_nodelist_get_list(void)
 {
-  CALLED(nodelist_get_list)++;
+  rset_get_all_n_no_nodes_nodelist_get_list_called++;
 
-  return NS(mock_smartlist);
+  return rset_get_all_n_no_nodes_mock_smartlist;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_get_all_nodes, nodelist_flag_not_running)
-
 /*
  * Structural test for routerset_get_all_nodes, with a non-list routerset
  * the running_only flag is set, but the nodes are not running.
  */
 
-NS_DECL(smartlist_t *, nodelist_get_list, (void));
+static const smartlist_t * rset_get_all_n_not_running_nodelist_get_list(void);
+static int rset_get_all_n_not_running_nodelist_get_list_called = 0;
 
-static smartlist_t *NS(mock_smartlist);
-static node_t NS(mock_node);
+static smartlist_t *rset_get_all_n_not_running_mock_smartlist;
+static node_t rset_get_all_n_not_running_mock_node;
 
 static void
-NS(test_main)(void *arg)
+test_rset_get_all_n_not_running(void *arg)
 {
   routerset_t *set = routerset_new();
   smartlist_t *out = smartlist_new();
   int r;
   (void)arg;
 
-  NS_MOCK(nodelist_get_list);
+  MOCK(nodelist_get_list,
+       rset_get_all_n_not_running_nodelist_get_list);
 
   smartlist_add_strdup(set->country_names, "{xx}");
-  NS(mock_smartlist) = smartlist_new();
-  NS(mock_node).is_running = 0;
-  smartlist_add(NS(mock_smartlist), (void *)&NS(mock_node));
+  rset_get_all_n_not_running_mock_smartlist = smartlist_new();
+  rset_get_all_n_not_running_mock_node.is_running = 0;
+  smartlist_add(rset_get_all_n_not_running_mock_smartlist,
+                (void *)&rset_get_all_n_not_running_mock_node);
 
   routerset_get_all_nodes(out, set, NULL, 1);
   r = smartlist_len(out);
   routerset_free(set);
   smartlist_free(out);
-  smartlist_free(NS(mock_smartlist));
+  smartlist_free(rset_get_all_n_not_running_mock_smartlist);
 
   tt_int_op(r, OP_EQ, 0);
-  tt_int_op(CALLED(nodelist_get_list), OP_EQ, 1);
+  tt_int_op(rset_get_all_n_not_running_nodelist_get_list_called, OP_EQ, 1);
 
   done:
     ;
 }
 
-smartlist_t *
-NS(nodelist_get_list)(void)
+const smartlist_t *
+rset_get_all_n_not_running_nodelist_get_list(void)
 {
-  CALLED(nodelist_get_list)++;
+  rset_get_all_n_not_running_nodelist_get_list_called++;
 
-  return NS(mock_smartlist);
+  return rset_get_all_n_not_running_mock_smartlist;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_subtract_nodes
-
 /*
  * Functional test for routerset_subtract_nodes.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_subtract_nodes(void *arg)
 {
   routerset_t *set = routerset_new();
   smartlist_t *list = smartlist_new();
@@ -1885,15 +1842,12 @@ NS(test_main)(void *arg)
     smartlist_free(list);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_subtract_nodes, null_routerset)
-
 /*
  * Functional test for routerset_subtract_nodes, with a NULL routerset.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_subtract_nodes_null_routerset(void *arg)
 {
   routerset_t *set = NULL;
   smartlist_t *list = smartlist_new();
@@ -1915,15 +1869,12 @@ NS(test_main)(void *arg)
     smartlist_free(list);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_to_string
-
 /*
  * Functional test for routerset_to_string.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_to_string(void *arg)
 {
   routerset_t *set = NULL;
   char *s = NULL;
@@ -1960,15 +1911,12 @@ NS(test_main)(void *arg)
   routerset_free(set);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, empty_empty)
-
 /*
  * Functional test for routerset_equal, with both routersets empty.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_empty_empty(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -1984,15 +1932,12 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, empty_not_empty)
-
 /*
  * Functional test for routerset_equal, with one routersets empty.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_empty_not_empty(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2008,16 +1953,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, differing_lengths)
-
 /*
  * Functional test for routerset_equal, with the routersets having
  * differing lengths.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_differing_lengths(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2035,16 +1977,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, unequal)
-
 /*
  * Functional test for routerset_equal, with the routersets being
  * different.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_unequal(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2061,16 +2000,13 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_equal, equal)
-
 /*
  * Functional test for routerset_equal, with the routersets being
  * equal.
  */
 
 static void
-NS(test_main)(void *arg)
+test_rset_equal_equal(void *arg)
 {
   routerset_t *a = routerset_new(), *b = routerset_new();
   int r;
@@ -2087,147 +2023,176 @@ NS(test_main)(void *arg)
     ;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(routerset_free, null_routerset)
-
 /*
  * Structural test for routerset_free, where the routerset is NULL.
  */
 
-NS_DECL(void, smartlist_free_, (smartlist_t *sl));
+static void rset_free_null_routerset_smartlist_free_(smartlist_t *sl);
+static int rset_free_null_routerset_smartlist_free__called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_free_null_routerset(void *arg)
 {
   (void)arg;
 
-  NS_MOCK(smartlist_free_);
+  MOCK(smartlist_free_,
+       rset_free_null_routerset_smartlist_free_);
 
   routerset_free_(NULL);
 
-  tt_int_op(CALLED(smartlist_free_), OP_EQ, 0);
+  tt_int_op(rset_free_null_routerset_smartlist_free__called, OP_EQ, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free_)(smartlist_t *s)
+rset_free_null_routerset_smartlist_free_(smartlist_t *s)
 {
   (void)s;
-  CALLED(smartlist_free_)++;
+  rset_free_null_routerset_smartlist_free__called++;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE routerset_free
-
 /*
  * Structural test for routerset_free.
  */
 
-NS_DECL(void, smartlist_free_, (smartlist_t *sl));
-NS_DECL(void, strmap_free_,(strmap_t *map, void (*free_val)(void*)));
-NS_DECL(void, digestmap_free_, (digestmap_t *map, void (*free_val)(void*)));
+static void rset_free_smartlist_free_(smartlist_t *sl);
+static int rset_free_smartlist_free__called = 0;
+static void rset_free_strmap_free_(strmap_t *map, void (*free_val)(void*));
+static int rset_free_strmap_free__called = 0;
+static void rset_free_digestmap_free_(digestmap_t *map,
+                                      void (*free_val)(void*));
+static int rset_free_digestmap_free__called = 0;
 
 static void
-NS(test_main)(void *arg)
+test_rset_free(void *arg)
 {
   routerset_t *routerset = routerset_new();
   (void)arg;
 
-  NS_MOCK(smartlist_free_);
-  NS_MOCK(strmap_free_);
-  NS_MOCK(digestmap_free_);
+  MOCK(smartlist_free_,
+       rset_free_smartlist_free_);
+  MOCK(strmap_free_,
+       rset_free_strmap_free_);
+  MOCK(digestmap_free_,
+       rset_free_digestmap_free_);
 
   routerset_free(routerset);
 
-  tt_int_op(CALLED(smartlist_free_), OP_NE, 0);
-  tt_int_op(CALLED(strmap_free_), OP_NE, 0);
-  tt_int_op(CALLED(digestmap_free_), OP_NE, 0);
+  tt_int_op(rset_free_smartlist_free__called, OP_NE, 0);
+  tt_int_op(rset_free_strmap_free__called, OP_NE, 0);
+  tt_int_op(rset_free_digestmap_free__called, OP_NE, 0);
 
   done:
     ;
 }
 
 void
-NS(smartlist_free_)(smartlist_t *s)
+rset_free_smartlist_free_(smartlist_t *s)
 {
-  CALLED(smartlist_free_)++;
+  rset_free_smartlist_free__called++;
   smartlist_free___real(s);
 }
 
 void
-NS(strmap_free_)(strmap_t *map, void (*free_val)(void*))
+rset_free_strmap_free_(strmap_t *map, void (*free_val)(void*))
 {
-  CALLED(strmap_free_)++;
+  rset_free_strmap_free__called++;
   strmap_free___real(map, free_val);
 }
 
 void
-NS(digestmap_free_)(digestmap_t *map, void (*free_val)(void*))
+rset_free_digestmap_free_(digestmap_t *map, void (*free_val)(void*))
 {
-  CALLED(digestmap_free_)++;
+  rset_free_digestmap_free__called++;
   digestmap_free___real(map, free_val);
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t routerset_tests[] = {
-  TEST_CASE(routerset_new),
-  TEST_CASE(routerset_get_countryname),
-  TEST_CASE(routerset_is_list),
-  TEST_CASE(routerset_needs_geoip),
-  TEST_CASE(routerset_is_empty),
-  TEST_CASE_ASPECT(routerset_contains, null_set_or_null_set_list),
-  TEST_CASE_ASPECT(routerset_contains, set_and_nickname),
-  TEST_CASE_ASPECT(routerset_contains, set_and_null_nickname),
-  TEST_CASE_ASPECT(routerset_contains, set_and_no_nickname),
-  TEST_CASE_ASPECT(routerset_contains, set_and_digest),
-  TEST_CASE_ASPECT(routerset_contains, set_and_no_digest),
-  TEST_CASE_ASPECT(routerset_contains, set_and_null_digest),
-  TEST_CASE_ASPECT(routerset_contains, set_and_addr),
-  TEST_CASE_ASPECT(routerset_contains, set_and_no_addr),
-  TEST_CASE_ASPECT(routerset_contains, set_and_null_addr),
-  TEST_CASE_ASPECT(routerset_contains, countries_no_geoip),
-  TEST_CASE_ASPECT(routerset_contains, countries_geoip),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, only_flag_and_no_ccs),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, creates_set),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, add_unknown),
-  TEST_CASE_ASPECT(routerset_add_unknown_ccs, add_a1),
-  TEST_CASE(routerset_contains_extendinfo),
-  TEST_CASE(routerset_contains_router),
-  TEST_CASE(routerset_contains_routerstatus),
-  TEST_CASE_ASPECT(routerset_contains_node, none),
-  TEST_CASE_ASPECT(routerset_contains_node, routerinfo),
-  TEST_CASE_ASPECT(routerset_contains_node, routerstatus),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, no_routerset),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, list_with_no_nodes),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, list_flag_not_running),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, list),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, nodelist_with_no_nodes),
-  TEST_CASE_ASPECT(routerset_get_all_nodes, nodelist_flag_not_running),
-  TEST_CASE_ASPECT(routerset_refresh_counties, geoip_not_loaded),
-  TEST_CASE_ASPECT(routerset_refresh_counties, no_countries),
-  TEST_CASE_ASPECT(routerset_refresh_counties, one_valid_country),
-  TEST_CASE_ASPECT(routerset_refresh_counties, one_invalid_country),
-  TEST_CASE_ASPECT(routerset_union, source_bad),
-  TEST_CASE_ASPECT(routerset_union, one),
-  TEST_CASE_ASPECT(routerset_parse, malformed),
-  TEST_CASE_ASPECT(routerset_parse, valid_hexdigest),
-  TEST_CASE_ASPECT(routerset_parse, valid_nickname),
-  TEST_CASE_ASPECT(routerset_parse, get_countryname),
-  TEST_CASE_ASPECT(routerset_parse, policy_wildcard),
-  TEST_CASE_ASPECT(routerset_parse, policy_ipv4),
-  TEST_CASE_ASPECT(routerset_parse, policy_ipv6),
-  TEST_CASE(routerset_subtract_nodes),
-  TEST_CASE_ASPECT(routerset_subtract_nodes, null_routerset),
-  TEST_CASE(routerset_to_string),
-  TEST_CASE_ASPECT(routerset_equal, empty_empty),
-  TEST_CASE_ASPECT(routerset_equal, empty_not_empty),
-  TEST_CASE_ASPECT(routerset_equal, differing_lengths),
-  TEST_CASE_ASPECT(routerset_equal, unequal),
-  TEST_CASE_ASPECT(routerset_equal, equal),
-  TEST_CASE_ASPECT(routerset_free, null_routerset),
-  TEST_CASE(routerset_free),
+  { "new", test_rset_new, TT_FORK, NULL, NULL },
+  { "get_countryname", test_rset_get_countryname, TT_FORK, NULL, NULL },
+  { "is_list", test_rset_is_list, TT_FORK, NULL, NULL },
+  { "needs_geoip", test_rset_needs_geoip, TT_FORK, NULL, NULL },
+  { "is_empty", test_rset_is_empty, TT_FORK, NULL, NULL },
+  { "contains_null_set_or_list", test_rset_contains_null_set_or_list,
+    TT_FORK, NULL, NULL },
+  { "contains_nickname", test_rset_contains_nickname, TT_FORK, NULL, NULL },
+  { "contains_null_nickname", test_rset_contains_null_nickname,
+    TT_FORK, NULL, NULL },
+  { "contains_no_nickname", test_rset_contains_no_nickname,
+    TT_FORK, NULL, NULL },
+  { "contains_digest", test_rset_contains_digest, TT_FORK, NULL, NULL },
+  { "contains_no_digest", test_rset_contains_no_digest, TT_FORK, NULL, NULL },
+  { "contains_null_digest", test_rset_contains_null_digest,
+    TT_FORK, NULL, NULL },
+  { "contains_addr", test_rset_contains_addr, TT_FORK, NULL, NULL },
+  { "contains_no_addr", test_rset_contains_no_addr, TT_FORK, NULL, NULL },
+  { "contains_null_addr", test_rset_contains_null_addr, TT_FORK, NULL, NULL },
+  { "contains_countries_no_geoip", test_rset_countries_no_geoip,
+    TT_FORK, NULL, NULL },
+  { "contains_countries_geoip", test_rset_countries_geoip,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_only_flag", test_rset_add_unknown_ccs_only_flag,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_creates_set", test_rset_add_unknown_ccs_creates_set,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_add_unknown", test_rset_add_unknown_ccs_add_unknown,
+    TT_FORK, NULL, NULL },
+  { "add_unknown_ccs_add_a1", test_rset_add_unknown_ccs_add_a1,
+    TT_FORK, NULL, NULL },
+  { "contains_extendinfo", test_rset_contains_extendinfo,
+    TT_FORK, NULL, NULL },
+  { "contains_router", test_rset_contains_router, TT_FORK, NULL, NULL },
+  { "contains_routerstatus", test_rset_contains_routerstatus,
+    TT_FORK, NULL, NULL },
+  { "contains_none", test_rset_contains_none, TT_FORK, NULL, NULL },
+  { "contains_routerinfo", test_rset_contains_routerinfo,
+    TT_FORK, NULL, NULL },
+  { "contains_rs", test_rset_contains_rs, TT_FORK, NULL, NULL },
+  { "get_all_no_routerset", test_rset_get_all_no_routerset,
+    TT_FORK, NULL, NULL },
+  { "get_all_l_no_nodes", test_rset_get_all_l_no_nodes, TT_FORK, NULL, NULL },
+  { "get_all_l_not_running", test_rset_get_all_l_not_running,
+    TT_FORK, NULL, NULL },
+  { "get_all_list", test_rset_get_all_list, TT_FORK, NULL, NULL },
+  { "get_all_n_no_nodes", test_rset_get_all_n_no_nodes, TT_FORK, NULL, NULL },
+  { "get_all_n_not_running", test_rset_get_all_n_not_running,
+    TT_FORK, NULL, NULL },
+  { "refresh_geoip_not_loaded", test_rset_refresh_geoip_not_loaded,
+    TT_FORK, NULL, NULL },
+  { "refresh_no_countries", test_rset_refresh_no_countries,
+    TT_FORK, NULL, NULL },
+  { "refresh_one_valid_country", test_rset_refresh_one_valid_country,
+    TT_FORK, NULL, NULL },
+  { "refresh_one_invalid_country", test_rset_refresh_one_invalid_country,
+    TT_FORK, NULL, NULL },
+  { "union_source_bad", test_rset_union_source_bad, TT_FORK, NULL, NULL },
+  { "union_one", test_rset_union_one, TT_FORK, NULL, NULL },
+  { "parse_malformed", test_rset_parse_malformed, TT_FORK, NULL, NULL },
+  { "parse_valid_hexdigest", test_rset_parse_valid_hexdigest,
+    TT_FORK, NULL, NULL },
+  { "parse_valid_nickname", test_rset_parse_valid_nickname,
+    TT_FORK, NULL, NULL },
+  { "parse_get_countryname", test_rset_parse_get_countryname,
+    TT_FORK, NULL, NULL },
+  { "parse_policy_wildcard", test_rset_parse_policy_wildcard,
+    TT_FORK, NULL, NULL },
+  { "parse_policy_ipv4", test_rset_parse_policy_ipv4, TT_FORK, NULL, NULL },
+  { "parse_policy_ipv6", test_rset_parse_policy_ipv6, TT_FORK, NULL, NULL },
+  { "subtract_nodes", test_rset_subtract_nodes, TT_FORK, NULL, NULL },
+  { "subtract_nodes_null_routerset", test_rset_subtract_nodes_null_routerset,
+    TT_FORK, NULL, NULL },
+  { "to_string", test_rset_to_string, TT_FORK, NULL, NULL },
+  { "equal_empty_empty", test_rset_equal_empty_empty, TT_FORK, NULL, NULL },
+  { "equal_empty_not_empty", test_rset_equal_empty_not_empty,
+    TT_FORK, NULL, NULL },
+  { "equal_differing_lengths", test_rset_equal_differing_lengths,
+    TT_FORK, NULL, NULL },
+  { "equal_unequal", test_rset_equal_unequal, TT_FORK, NULL, NULL },
+  { "equal_equal", test_rset_equal_equal, TT_FORK, NULL, NULL },
+  { "free_null_routerset", test_rset_free_null_routerset,
+    TT_FORK, NULL, NULL },
+  { "free", test_rset_free, TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_rust.sh b/src/test/test_rust.sh
index 00b3e88d37..804d2ada36 100755
--- a/src/test/test_rust.sh
+++ b/src/test/test_rust.sh
@@ -14,11 +14,12 @@ rustc_host=$(rustc -vV | grep host | sed 's/host: //')
 
 for cargo_toml_dir in "${abs_top_srcdir:-../../..}"/src/rust/*; do
     if [ -e "${cargo_toml_dir}/Cargo.toml" ]; then
+        # shellcheck disable=SC2086
 	cd "${abs_top_builddir:-../../..}/src/rust" && \
 	    CARGO_TARGET_DIR="${abs_top_builddir:-../../..}/src/rust/target" \
-	    "${CARGO:-cargo}" test ${CARGO_ONLINE-"--frozen"} \
+	    "${CARGO:-cargo}" test "${CARGO_ONLINE-'--frozen'}" \
             --features "test_linking_hack" \
-            --target $rustc_host \
+            --target "$rustc_host" \
 	    ${EXTRA_CARGO_OPTIONS} \
 	    --manifest-path "${cargo_toml_dir}/Cargo.toml" || exitcode=1
     fi
diff --git a/src/test/test_scheduler.c b/src/test/test_scheduler.c
index bf9c6a49cd..9ec15948e8 100644
--- a/src/test/test_scheduler.c
+++ b/src/test/test_scheduler.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -6,8 +6,8 @@
 #include <math.h>
 
 #define SCHEDULER_KIST_PRIVATE
-#define TOR_CHANNEL_INTERNAL_
-#define CHANNEL_PRIVATE_
+#define CHANNEL_OBJECT_PRIVATE
+#define CHANNEL_FILE_PRIVATE
 #include "core/or/or.h"
 #include "app/config/config.h"
 #include "lib/evloop/compat_libevent.h"
@@ -15,7 +15,7 @@
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"
-#define SCHEDULER_PRIVATE_
+#define SCHEDULER_PRIVATE
 #include "core/or/scheduler.h"
 
 /* Test suite stuff */
@@ -848,8 +848,8 @@ test_scheduler_initfree(void *arg)
 {
   (void)arg;
 
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
+  tt_ptr_op(channels_pending, OP_EQ, NULL);
+  tt_ptr_op(run_sched_ev, OP_EQ, NULL);
 
   MOCK(get_options, mock_get_options);
   set_scheduler_options(SCHEDULER_KIST);
@@ -858,17 +858,17 @@ test_scheduler_initfree(void *arg)
 
   scheduler_init();
 
-  tt_ptr_op(channels_pending, !=, NULL);
-  tt_ptr_op(run_sched_ev, !=, NULL);
+  tt_ptr_op(channels_pending, OP_NE, NULL);
+  tt_ptr_op(run_sched_ev, OP_NE, NULL);
   /* We have specified nothing in the torrc and there's no consensus so the
    * KIST scheduler is what should be in use */
-  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
-  tt_int_op(sched_run_interval, ==, 10);
+  tt_ptr_op(the_scheduler, OP_EQ, get_kist_scheduler());
+  tt_int_op(sched_run_interval, OP_EQ, 10);
 
   scheduler_free_all();
 
-  tt_ptr_op(channels_pending, ==, NULL);
-  tt_ptr_op(run_sched_ev, ==, NULL);
+  tt_ptr_op(channels_pending, OP_EQ, NULL);
+  tt_ptr_op(run_sched_ev, OP_EQ, NULL);
 
  done:
   UNMOCK(get_options);
@@ -890,11 +890,11 @@ test_scheduler_can_use_kist(void *arg)
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
 #ifdef HAVE_KIST_SUPPORT
-  tt_int_op(res_should, ==, 1);
+  tt_int_op(res_should, OP_EQ, 1);
 #else /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
 #endif /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_freq, ==, 1234);
+  tt_int_op(res_freq, OP_EQ, 1234);
 
   /* Test defer to consensus, but no consensus available */
   clear_options();
@@ -902,11 +902,11 @@ test_scheduler_can_use_kist(void *arg)
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
 #ifdef HAVE_KIST_SUPPORT
-  tt_int_op(res_should, ==, 1);
+  tt_int_op(res_should, OP_EQ, 1);
 #else /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
 #endif /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_freq, ==, 10);
+  tt_int_op(res_freq, OP_EQ, 10);
 
   /* Test defer to consensus, and kist consensus available */
   MOCK(networkstatus_get_param, mock_kist_networkstatus_get_param);
@@ -915,11 +915,11 @@ test_scheduler_can_use_kist(void *arg)
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
 #ifdef HAVE_KIST_SUPPORT
-  tt_int_op(res_should, ==, 1);
+  tt_int_op(res_should, OP_EQ, 1);
 #else /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_should, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
 #endif /* HAVE_KIST_SUPPORT */
-  tt_int_op(res_freq, ==, 12);
+  tt_int_op(res_freq, OP_EQ, 12);
   UNMOCK(networkstatus_get_param);
 
   /* Test defer to consensus, and vanilla consensus available */
@@ -928,8 +928,8 @@ test_scheduler_can_use_kist(void *arg)
   mocked_options.KISTSchedRunInterval = 0;
   res_should = scheduler_can_use_kist();
   res_freq = kist_scheduler_run_interval();
-  tt_int_op(res_should, ==, 0);
-  tt_int_op(res_freq, ==, 0);
+  tt_int_op(res_should, OP_EQ, 0);
+  tt_int_op(res_freq, OP_EQ, 0);
   UNMOCK(networkstatus_get_param);
 
  done:
@@ -956,7 +956,7 @@ test_scheduler_ns_changed(void *arg)
   set_scheduler_options(SCHEDULER_KIST);
   set_scheduler_options(SCHEDULER_VANILLA);
 
-  tt_ptr_op(the_scheduler, ==, NULL);
+  tt_ptr_op(the_scheduler, OP_EQ, NULL);
 
   /* Change from vanilla to kist via consensus */
   the_scheduler = get_vanilla_scheduler();
@@ -964,9 +964,9 @@ test_scheduler_ns_changed(void *arg)
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
 #ifdef HAVE_KIST_SUPPORT
-  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_kist_scheduler());
 #else
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 #endif
 
   /* Change from kist to vanilla via consensus */
@@ -974,7 +974,7 @@ test_scheduler_ns_changed(void *arg)
   MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 
   /* Doesn't change when using KIST */
   the_scheduler = get_kist_scheduler();
@@ -982,9 +982,9 @@ test_scheduler_ns_changed(void *arg)
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
 #ifdef HAVE_KIST_SUPPORT
-  tt_ptr_op(the_scheduler, ==, get_kist_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_kist_scheduler());
 #else
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 #endif
 
   /* Doesn't change when using vanilla */
@@ -992,7 +992,7 @@ test_scheduler_ns_changed(void *arg)
   MOCK(networkstatus_get_param, mock_vanilla_networkstatus_get_param);
   scheduler_notify_networkstatus_changed();
   UNMOCK(networkstatus_get_param);
-  tt_ptr_op(the_scheduler, ==, get_vanilla_scheduler());
+  tt_ptr_op(the_scheduler, OP_EQ, get_vanilla_scheduler());
 
  done:
   UNMOCK(get_options);
diff --git a/src/test/test_sendme.c b/src/test/test_sendme.c
new file mode 100644
index 0000000000..2ff4809124
--- /dev/null
+++ b/src/test/test_sendme.c
@@ -0,0 +1,365 @@
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/* Unit tests for handling different kinds of relay cell */
+
+#define CIRCUITLIST_PRIVATE
+#define NETWORKSTATUS_PRIVATE
+#define SENDME_PRIVATE
+#define RELAY_PRIVATE
+
+#include "core/or/circuit_st.h"
+#include "core/or/or_circuit_st.h"
+#include "core/or/origin_circuit_st.h"
+#include "core/or/circuitlist.h"
+#include "core/or/relay.h"
+#include "core/or/sendme.h"
+
+#include "feature/nodelist/networkstatus.h"
+#include "feature/nodelist/networkstatus_st.h"
+
+#include "lib/crypt_ops/crypto_digest.h"
+
+#include "test/test.h"
+#include "test/log_test_helpers.h"
+
+static void
+setup_mock_consensus(void)
+{
+  current_md_consensus = current_ns_consensus =
+    tor_malloc_zero(sizeof(networkstatus_t));
+  current_md_consensus->net_params = smartlist_new();
+  current_md_consensus->routerstatus_list = smartlist_new();
+}
+
+static void
+free_mock_consensus(void)
+{
+  SMARTLIST_FOREACH(current_md_consensus->routerstatus_list, void *, r,
+                    tor_free(r));
+  smartlist_free(current_md_consensus->routerstatus_list);
+  smartlist_free(current_ns_consensus->net_params);
+  tor_free(current_ns_consensus);
+}
+
+static void
+test_v1_record_digest(void *arg)
+{
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  /* Create our dummy circuit. */
+  or_circ = or_circuit_new(1, NULL);
+  /* Points it to the OR circuit now. */
+  circ = TO_CIRCUIT(or_circ);
+
+  /* The package window has to be a multiple of CIRCWINDOW_INCREMENT minus 1
+   * in order to catched the CIRCWINDOW_INCREMENT-nth cell. Try something that
+   * shouldn't be noted. */
+  circ->package_window = CIRCWINDOW_INCREMENT;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(!circ->sendme_last_digests);
+
+  /* This should work now. Package window at CIRCWINDOW_INCREMENT + 1. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_assert(circ->sendme_last_digests);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* Next cell in the package window shouldn't do anything. */
+  circ->package_window++;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+
+  /* The next CIRCWINDOW_INCREMENT should add one more digest. */
+  circ->package_window = (CIRCWINDOW_INCREMENT * 2) + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 2);
+
+ done:
+  circuit_free_(circ);
+}
+
+static void
+test_v1_consensus_params(void *arg)
+{
+  (void) arg;
+
+  setup_mock_consensus();
+  tt_assert(current_md_consensus);
+
+  /* Both zeroes. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=0");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 0);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Both ones. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 1);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Different values from each other. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_emit_min_version=1");
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=0");
+  tt_int_op(get_emit_min_version(), OP_EQ, 1);
+  tt_int_op(get_accept_min_version(), OP_EQ, 0);
+  smartlist_clear(current_md_consensus->net_params);
+
+  /* Validate is the cell version is coherent with our internal default value
+   * and the one in the consensus. */
+  smartlist_add(current_md_consensus->net_params,
+                (void *) "sendme_accept_min_version=1");
+  /* Minimum acceptable value is 1. */
+  tt_int_op(cell_version_can_be_handled(1), OP_EQ, true);
+  /* Minimum acceptable value is 1 so a cell version of 0 is refused. */
+  tt_int_op(cell_version_can_be_handled(0), OP_EQ, false);
+
+ done:
+  free_mock_consensus();
+}
+
+static void
+test_v1_build_cell(void *arg)
+{
+  uint8_t payload[RELAY_PAYLOAD_SIZE], digest[DIGEST_LEN];
+  ssize_t ret;
+  crypto_digest_t *cell_digest = NULL;
+  or_circuit_t *or_circ = NULL;
+  circuit_t *circ = NULL;
+
+  (void) arg;
+
+  or_circ = or_circuit_new(1, NULL);
+  circ = TO_CIRCUIT(or_circ);
+  circ->sendme_last_digests = smartlist_new();
+
+  cell_digest = crypto_digest_new();
+  tt_assert(cell_digest);
+  crypto_digest_add_bytes(cell_digest, "AAAAAAAAAAAAAAAAAAAA", 20);
+  crypto_digest_get_digest(cell_digest, (char *) digest, sizeof(digest));
+  smartlist_add(circ->sendme_last_digests, tor_memdup(digest, sizeof(digest)));
+
+  /* SENDME v1 payload is 3 bytes + 20 bytes digest. See spec. */
+  ret = build_cell_payload_v1(digest, payload);
+  tt_int_op(ret, OP_EQ, 23);
+
+  /* Validation. */
+
+  /* An empty payload means SENDME version 0 thus valid. */
+  tt_int_op(sendme_is_valid(circ, payload, 0), OP_EQ, true);
+  /* Current phoney digest should have been popped. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+  /* An unparseable cell means invalid. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, (const uint8_t *) "A", 1), OP_EQ, false);
+  expect_log_msg_containing("Unparseable SENDME cell received. "
+                            "Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* No cell digest recorded for this. */
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  expect_log_msg_containing("We received a SENDME but we have no cell digests "
+                            "to match. Closing circuit.");
+  teardown_capture_of_logs();
+
+  /* Note the wrong digest in the circuit, cell should fail validation. */
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  setup_full_capture_of_logs(LOG_INFO);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, false);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+  expect_log_msg_containing("SENDME v1 cell digest do not match.");
+  teardown_capture_of_logs();
+
+  /* Record the cell digest into the circuit, cell should validate. */
+  memcpy(or_circ->crypto.sendme_digest, digest, sizeof(digest));
+  circ->package_window = CIRCWINDOW_INCREMENT + 1;
+  sendme_record_cell_digest_on_circ(circ, NULL);
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 1);
+  tt_int_op(sendme_is_valid(circ, payload, sizeof(payload)), OP_EQ, true);
+  /* After a validation, the last digests is always popped out. */
+  tt_int_op(smartlist_len(circ->sendme_last_digests), OP_EQ, 0);
+
+ done:
+  crypto_digest_free(cell_digest);
+  circuit_free_(circ);
+}
+
+static void
+test_cell_payload_pad(void *arg)
+{
+  size_t pad_offset, payload_len, expected_offset;
+
+  (void) arg;
+
+  /* Offset should be 0, not enough room for padding. */
+  payload_len = RELAY_PAYLOAD_SIZE;
+  pad_offset = get_pad_cell_offset(payload_len);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Still no room because we keep 4 extra bytes. */
+  pad_offset = get_pad_cell_offset(payload_len - 4);
+  tt_int_op(pad_offset, OP_EQ, 0);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* We should have 1 byte of padding. Meaning, the offset should be the
+   * CELL_PAYLOAD_SIZE minus 1 byte. */
+  expected_offset = CELL_PAYLOAD_SIZE - 1;
+  pad_offset = get_pad_cell_offset(payload_len - 5);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Now some arbitrary small payload length. The cell size is header + 10 +
+   * extra 4 bytes we keep so the offset should be there. */
+  expected_offset = RELAY_HEADER_SIZE + 10 + 4;
+  pad_offset = get_pad_cell_offset(10);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+  /* Data length of 0. */
+  expected_offset = RELAY_HEADER_SIZE + 4;
+  pad_offset = get_pad_cell_offset(0);
+  tt_int_op(pad_offset, OP_EQ, expected_offset);
+  tt_int_op(CELL_PAYLOAD_SIZE - pad_offset, OP_LE, CELL_PAYLOAD_SIZE);
+
+ done:
+  ;
+}
+
+static void
+test_cell_version_validation(void *arg)
+{
+  (void) arg;
+
+  /* We currently only support up to SENDME_MAX_SUPPORTED_VERSION so we are
+   * going to test the boundaries there. */
+
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION));
+
+  /* Version below our supported should pass. */
+  tt_assert(cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION - 1));
+
+  /* Extra version from our supported should fail. */
+  tt_assert(!cell_version_can_be_handled(SENDME_MAX_SUPPORTED_VERSION + 1));
+
+  /* Simple check for version 0. */
+  tt_assert(cell_version_can_be_handled(0));
+
+  /* We MUST handle the default cell version that we emit or accept. */
+  tt_assert(cell_version_can_be_handled(SENDME_EMIT_MIN_VERSION_DEFAULT));
+  tt_assert(cell_version_can_be_handled(SENDME_ACCEPT_MIN_VERSION_DEFAULT));
+
+ done:
+  ;
+}
+
+/* check our decisions about how much stuff to put into relay cells. */
+static void
+test_package_payload_len(void *arg)
+{
+  (void)arg;
+  /* this is not a real circuit: it only has the fields needed for this
+   * test. */
+  circuit_t *c = tor_malloc_zero(sizeof(circuit_t));
+
+  /* check initial conditions. */
+  circuit_reset_sendme_randomness(c);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE, CIRCWINDOW_INCREMENT / 2);
+  tt_int_op(c->send_randomness_after_n_cells, OP_LT, CIRCWINDOW_INCREMENT);
+
+  /* We have a bunch of cells before we need to send randomness, so the first
+   * few can be packaged full. */
+  int initial = c->send_randomness_after_n_cells;
+  size_t n = connection_edge_get_inbuf_bytes_to_package(10000, 0, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  n = connection_edge_get_inbuf_bytes_to_package(95000, 1, c);
+  tt_uint_op(RELAY_PAYLOAD_SIZE, OP_EQ, n);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial isn't set, we won't package a partially full cell at
+   * all. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 0, c);
+  tt_int_op(n, OP_EQ, 0);
+  /* no change in our state, since nothing was sent. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 2);
+
+  /* If package_partial is set and the partial cell is not going to have
+   * _enough_ randomness, we package it, but we don't consider ourselves to
+   * have sent a sufficiently random cell. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-1, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-1);
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 3);
+
+  /* Make sure we set have_set_sufficiently_random_cell as appropriate. */
+  n = connection_edge_get_inbuf_bytes_to_package(RELAY_PAYLOAD_SIZE-64, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE-64);
+  tt_assert(c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_EQ, initial - 4);
+
+  /* Now let's look at what happens when we get down to zero. Since we have
+   * sent a sufficiently random cell, we will not force this one to have a gap.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  tt_int_op(n, OP_EQ, RELAY_PAYLOAD_SIZE);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* What would happen if we hadn't sent a sufficiently random cell? */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(10000, 1, c);
+  const size_t reduced_payload_size = RELAY_PAYLOAD_SIZE - 4 - 16;
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+  /* Now these will be reset. */
+  tt_assert(! c->have_sent_sufficiently_random_cell);
+  tt_int_op(c->send_randomness_after_n_cells, OP_GE,
+            CIRCWINDOW_INCREMENT / 2 - 1);
+
+  /* Here is a fun case: if it's time to package a small cell, then
+   * package_partial==0 should mean we accept that many bytes.
+   */
+  c->send_randomness_after_n_cells = 0;
+  n = connection_edge_get_inbuf_bytes_to_package(reduced_payload_size, 0, c);
+  tt_int_op(n, OP_EQ, reduced_payload_size);
+
+ done:
+  tor_free(c);
+}
+
+struct testcase_t sendme_tests[] = {
+  { "v1_record_digest", test_v1_record_digest, TT_FORK,
+    NULL, NULL },
+  { "v1_consensus_params", test_v1_consensus_params, TT_FORK,
+    NULL, NULL },
+  { "v1_build_cell", test_v1_build_cell, TT_FORK,
+    NULL, NULL },
+  { "cell_payload_pad", test_cell_payload_pad, TT_FORK,
+    NULL, NULL },
+  { "cell_version_validation", test_cell_version_validation, TT_FORK,
+    NULL, NULL },
+  { "package_payload_len", test_package_payload_len, 0, NULL, NULL },
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_shared_random.c b/src/test/test_shared_random.c
index 413dfbeb03..9e49e835c9 100644
--- a/src/test/test_shared_random.c
+++ b/src/test/test_shared_random.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2016-2019, The Tor Project, Inc. */
+/* Copyright (c) 2016-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define SHARED_RANDOM_PRIVATE
@@ -21,7 +21,7 @@
 #include "feature/nodelist/dirlist.h"
 #include "feature/dirparse/authcert_parse.h"
 #include "feature/hs_common/shared_random_client.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
 
 #include "feature/dirclient/dir_server_st.h"
 #include "feature/nodelist/networkstatus_st.h"
@@ -58,14 +58,17 @@ trusteddirserver_get_by_v3_auth_digest_m(const char *digest)
 }
 
 /* Setup a minimal dirauth environment by initializing the SR state and
- * making sure the options are set to be an authority directory. */
+ * making sure the options are set to be an authority directory.
+ * You must only call this function once per process. */
 static void
 init_authority_state(void)
 {
   MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
 
   or_options_t *options = get_options_mutable();
-  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+  mock_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                               strlen(AUTHORITY_CERT_1),
+                                               NULL);
   tt_assert(mock_cert);
   options->AuthoritativeDir = 1;
   tt_int_op(load_ed_keys(options, time(NULL)), OP_GE, 0);
@@ -201,7 +204,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     /* Compare it with the correct result */
@@ -213,7 +216,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 19:22:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -224,7 +227,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -235,7 +238,7 @@ test_get_state_valid_until_time(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     valid_until_time = get_state_valid_until_time(current_time);
 
     format_iso_time(tbuf, valid_until_time);
@@ -275,7 +278,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:01 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -287,7 +290,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 23:59:59 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -299,7 +302,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:00:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -323,12 +326,13 @@ test_get_start_time_of_current_run(void *arg)
 
     retval = parse_rfc1123_time("Mon, 19 Apr 2015 23:00:00 UTC",
                                 &mock_consensus.valid_after);
+    tt_int_op(retval, OP_EQ, 0);
 
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:08:00 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
     update_approx_time(current_time);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
 
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
@@ -336,7 +340,7 @@ test_get_start_time_of_current_run(void *arg)
     format_iso_time(tbuf, run_start_time);
     tt_str_op("2015-04-19 00:00:00", OP_EQ, tbuf);
     /* Check that voting_schedule.interval_starts is at 01:00 (see above) */
-    time_t interval_starts = voting_schedule_get_next_valid_after_time();
+    time_t interval_starts = dirauth_sched_get_next_valid_after_time();
     format_iso_time(tbuf, interval_starts);
     tt_str_op("2015-04-20 01:00:00", OP_EQ, tbuf);
   }
@@ -356,7 +360,7 @@ test_get_start_time_of_current_run(void *arg)
     retval = parse_rfc1123_time("Mon, 20 Apr 2015 00:15:32 UTC",
                                 &current_time);
     tt_int_op(retval, OP_EQ, 0);
-    voting_schedule_recalculate_timing(get_options(), current_time);
+    dirauth_sched_recalculate_timing(get_options(), current_time);
     run_start_time = sr_state_get_start_time_of_current_protocol_run();
 
     /* Compare it with the correct result */
@@ -388,13 +392,13 @@ test_get_start_time_functions(void *arg)
   tt_int_op(retval, OP_EQ, 0);
   time_t now = mock_consensus.valid_after;
 
-  voting_schedule_recalculate_timing(get_options(), now);
+  dirauth_sched_recalculate_timing(get_options(), now);
   time_t start_time_of_protocol_run =
     sr_state_get_start_time_of_current_protocol_run();
   tt_assert(start_time_of_protocol_run);
 
   /* Check that the round start time of the beginning of the run, is itself */
-  tt_int_op(get_start_time_of_current_round(), OP_EQ,
+  tt_int_op(dirauth_sched_get_cur_valid_after_time(), OP_EQ,
             start_time_of_protocol_run);
 
  done:
@@ -438,7 +442,9 @@ test_sr_commit(void *arg)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -457,12 +463,12 @@ test_sr_commit(void *arg)
     /* We should have a reveal value. */
     tt_assert(commit_has_reveal_value(our_commit));
     /* We should have a random value. */
-    tt_assert(!tor_mem_is_zero((char *) our_commit->random_number,
+    tt_assert(!fast_mem_is_zero((char *) our_commit->random_number,
                                sizeof(our_commit->random_number)));
     /* Commit and reveal timestamp should be the same. */
     tt_u64_op(our_commit->commit_ts, OP_EQ, our_commit->reveal_ts);
     /* We should have a hashed reveal. */
-    tt_assert(!tor_mem_is_zero(our_commit->hashed_reveal,
+    tt_assert(!fast_mem_is_zero(our_commit->hashed_reveal,
                                sizeof(our_commit->hashed_reveal)));
     /* Do we have a valid encoded commit and reveal. Note the following only
      * tests if the generated values are correct. Their could be a bug in
@@ -849,7 +855,9 @@ test_sr_setup_commits(void)
   {  /* Setup a minimal dirauth environment for this test  */
     or_options_t *options = get_options_mutable();
 
-    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1, NULL);
+    auth_cert = authority_cert_parse_from_string(AUTHORITY_CERT_1,
+                                                 strlen(AUTHORITY_CERT_1),
+                                                 NULL);
     tt_assert(auth_cert);
 
     options->AuthoritativeDir = 1;
@@ -1087,73 +1095,99 @@ test_sr_get_majority_srv_from_votes(void *arg)
   smartlist_free(votes);
 }
 
+/* Testing sr_srv_dup(). */
 static void
-test_utils(void *arg)
+test_sr_svr_dup(void *arg)
 {
-  (void) arg;
+  (void)arg;
 
-  /* Testing srv_dup(). */
-  {
-    sr_srv_t *srv = NULL, *dup_srv = NULL;
-    const char *srv_value =
-      "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
-    srv = tor_malloc_zero(sizeof(*srv));
-    srv->num_reveals = 42;
-    memcpy(srv->value, srv_value, sizeof(srv->value));
-    dup_srv = srv_dup(srv);
-    tt_assert(dup_srv);
-    tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
-    tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
-    tor_free(srv);
-    tor_free(dup_srv);
-  }
+  sr_srv_t *srv = NULL, *dup_srv = NULL;
+  const char *srv_value =
+    "1BDB7C3E973936E4D13A49F37C859B3DC69C429334CF9412E3FEF6399C52D47A";
+  srv = tor_malloc_zero(sizeof(*srv));
+  srv->num_reveals = 42;
+  memcpy(srv->value, srv_value, sizeof(srv->value));
+  dup_srv = sr_srv_dup(srv);
+  tt_assert(dup_srv);
+  tt_u64_op(dup_srv->num_reveals, OP_EQ, srv->num_reveals);
+  tt_mem_op(dup_srv->value, OP_EQ, srv->value, sizeof(srv->value));
 
-  /* Testing commitments_are_the_same(). Currently, the check is to test the
-   * value of the encoded commit so let's make sure that actually works. */
-  {
-    /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
-     * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
-     * which comes down to SR_COMMIT_BASE64_LEN + 1. */
-    const char *payload =
-      "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
-      "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
-      "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
-      "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
-    sr_commit_t commit1, commit2;
-    memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
-    memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
-    /* Let's corrupt one of them. */
-    memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
-    tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
-  }
+ done:
+  tor_free(srv);
+  tor_free(dup_srv);
+}
 
-  /* Testing commit_is_authoritative(). */
-  {
-    crypto_pk_t *k = crypto_pk_new();
-    char digest[DIGEST_LEN];
-    sr_commit_t commit;
+/* Testing commitments_are_the_same(). Currently, the check is to test the
+ * value of the encoded commit so let's make sure that actually works. */
+static void
+test_commitments_are_the_same(void *arg)
+{
+  (void)arg;
+
+  /* Payload of 57 bytes that is the length of sr_commit_t->encoded_commit.
+  * 56 bytes of payload and a NUL terminated byte at the end ('\x00')
+  * which comes down to SR_COMMIT_BASE64_LEN + 1. */
+  const char *payload =
+  "\x5d\xb9\x60\xb6\xcc\x51\x68\x52\x31\xd9\x88\x88\x71\x71\xe0\x30"
+  "\x59\x55\x7f\xcd\x61\xc0\x4b\x05\xb8\xcd\xc1\x48\xe9\xcd\x16\x1f"
+  "\x70\x15\x0c\xfc\xd3\x1a\x75\xd0\x93\x6c\xc4\xe0\x5c\xbe\xe2\x18"
+  "\xc7\xaf\x72\xb6\x7c\x9b\x52\x00";
+  sr_commit_t commit1, commit2;
+  memcpy(commit1.encoded_commit, payload, sizeof(commit1.encoded_commit));
+  memcpy(commit2.encoded_commit, payload, sizeof(commit2.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 1);
+  /* Let's corrupt one of them. */
+  memset(commit1.encoded_commit, 'A', sizeof(commit1.encoded_commit));
+  tt_int_op(commitments_are_the_same(&commit1, &commit2), OP_EQ, 0);
 
-    tt_assert(!crypto_pk_generate_key(k));
+ done:
+  return;
+}
 
-    tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
-    memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
-    /* Change the pubkey. */
-    memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
-    tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
-    crypto_pk_free(k);
-  }
+/* Testing commit_is_authoritative(). */
+static void
+test_commit_is_authoritative(void *arg)
+{
+  (void)arg;
 
-  /* Testing get_phase_str(). */
-  {
-    tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
-    tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
-  }
+  crypto_pk_t *k = crypto_pk_new();
+  char digest[DIGEST_LEN];
+  sr_commit_t commit;
+
+  tt_assert(!crypto_pk_generate_key(k));
+
+  tt_int_op(0, OP_EQ, crypto_pk_get_digest(k, digest));
+  memcpy(commit.rsa_identity, digest, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 1);
+  /* Change the pubkey. */
+  memset(commit.rsa_identity, 0, sizeof(commit.rsa_identity));
+  tt_int_op(commit_is_authoritative(&commit, digest), OP_EQ, 0);
+
+ done:
+  crypto_pk_free(k);
+}
+
+static void
+test_get_phase_str(void *arg)
+{
+  (void)arg;
+
+  tt_str_op(get_phase_str(SR_PHASE_REVEAL), OP_EQ, "reveal");
+  tt_str_op(get_phase_str(SR_PHASE_COMMIT), OP_EQ, "commit");
+
+ done:
+  return;
+}
+
+/* Test utils that depend on authority state */
+static void
+test_utils_auth(void *arg)
+{
+  (void)arg;
+  init_authority_state();
 
   /* Testing phase transition */
   {
-    init_authority_state();
     set_sr_phase(SR_PHASE_COMMIT);
     tt_int_op(is_phase_transition(SR_PHASE_REVEAL), OP_EQ, 1);
     tt_int_op(is_phase_transition(SR_PHASE_COMMIT), OP_EQ, 0);
@@ -1164,8 +1198,193 @@ test_utils(void *arg)
     tt_int_op(is_phase_transition(42), OP_EQ, 1);
   }
 
+  /* Testing get, set, delete, clean SRVs */
+
+  {
+    /* Just set the previous SRV */
+    test_sr_setup_srv(0);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Delete the SRVs one at a time */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    state_del_previous_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    state_del_current_srv();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now set the SRVs to NULL instead */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And in the opposite order */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And both at once */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_clean_srvs();
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+
+    /* And do the gets and sets multiple times */
+    test_sr_setup_srv(1);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    sr_state_set_current_srv(NULL);
+    sr_state_set_previous_srv(NULL);
+    sr_state_set_current_srv(NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_EQ, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
+  }
+
+  {
+    /* Now copy the values across */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the previous: the protocol goes the other way */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_previous_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is the same */
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  {
+    /* Now copy a value onto itself */
+    test_sr_setup_srv(1);
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Take a copy of the old value */
+    sr_srv_t old_current_srv;
+    memcpy(&old_current_srv, sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is different */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Set the current to the current: the protocol never replaces an SRV with
+     * the same value */
+    sr_state_set_current_srv(sr_srv_dup(sr_state_get_current_srv()));
+    /* Check that the pointers are non-NULL, and different from each other */
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_current_srv(), OP_NE, NULL);
+    tt_ptr_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv());
+    /* Check that the content is different between current and previous */
+    tt_mem_op(sr_state_get_previous_srv(), OP_NE,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+    /* Check that the content is the same as the old content */
+    tt_mem_op(&old_current_srv, OP_EQ,
+              sr_state_get_current_srv(), sizeof(sr_srv_t));
+  }
+
+  /* I don't think we can say "expect a BUG()" in our tests. */
+#if 0
+  {
+    /* Now copy a value onto itself without sr_srv_dup().
+     * This should fail with a BUG() warning. */
+    test_sr_setup_srv(1);
+    sr_state_set_current_srv(sr_state_get_current_srv());
+    sr_state_set_previous_srv(sr_state_get_previous_srv());
+  }
+#endif /* 0 */
+
  done:
-  return;
+  sr_state_free_all();
 }
 
 static void
@@ -1173,6 +1392,7 @@ test_state_transition(void *arg)
 {
   sr_state_t *state = NULL;
   time_t now = time(NULL);
+  sr_srv_t *cur = NULL;
 
   (void) arg;
 
@@ -1211,44 +1431,47 @@ test_state_transition(void *arg)
 
   /* Test SRV rotation in our state. */
   {
-    const sr_srv_t *cur, *prev;
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     tt_assert(cur);
-    /* After, current srv should be the previous and then set to NULL. */
+    /* After, the previous SRV should be the same as the old current SRV, and
+     * the current SRV should be set to NULL */
     state_rotate_srv();
-    prev = sr_state_get_previous_srv();
-    tt_assert(prev == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     tt_ptr_op(sr_state_get_current_srv(), OP_EQ, NULL);
     sr_state_clean_srvs();
+    tor_free(cur);
   }
 
   /* New protocol run. */
   {
-    const sr_srv_t *cur;
     /* Setup some new SRVs so we can confirm that a new protocol run
      * actually makes them rotate and compute new ones. */
     test_sr_setup_srv(1);
-    cur = sr_state_get_current_srv();
-    tt_assert(cur);
+    tt_assert(sr_state_get_current_srv());
+    /* Take a copy of the data, because the state owns the pointer */
+    cur = sr_srv_dup(sr_state_get_current_srv());
     set_sr_phase(SR_PHASE_REVEAL);
     MOCK(get_my_v3_authority_cert, get_my_v3_authority_cert_m);
     new_protocol_run(now);
     UNMOCK(get_my_v3_authority_cert);
     /* Rotation happened. */
-    tt_assert(sr_state_get_previous_srv() == cur);
+    tt_mem_op(sr_state_get_previous_srv(), OP_EQ, cur, sizeof(sr_srv_t));
     /* We are going into COMMIT phase so we had to rotate our SRVs. Usually
      * our current SRV would be NULL but a new protocol run should make us
      * compute a new SRV. */
     tt_assert(sr_state_get_current_srv());
     /* Also, make sure we did change the current. */
-    tt_assert(sr_state_get_current_srv() != cur);
+    tt_mem_op(sr_state_get_current_srv(), OP_NE, cur, sizeof(sr_srv_t));
     /* We should have our commitment alone. */
     tt_int_op(digestmap_size(state->commits), OP_EQ, 1);
     tt_int_op(state->n_reveal_rounds, OP_EQ, 0);
     tt_int_op(state->n_commit_rounds, OP_EQ, 0);
     /* 46 here since we were at 45 just before. */
     tt_u64_op(state->n_protocol_runs, OP_EQ, 46);
+    tor_free(cur);
   }
 
   /* Cleanup of SRVs. */
@@ -1259,6 +1482,7 @@ test_state_transition(void *arg)
   }
 
  done:
+  tor_free(cur);
   sr_state_free_all();
 }
 
@@ -1454,7 +1678,13 @@ struct testcase_t sr_tests[] = {
   { "sr_compute_srv", test_sr_compute_srv, TT_FORK, NULL, NULL },
   { "sr_get_majority_srv_from_votes", test_sr_get_majority_srv_from_votes,
     TT_FORK, NULL, NULL },
-  { "utils", test_utils, TT_FORK, NULL, NULL },
+  { "sr_svr_dup", test_sr_svr_dup, TT_FORK, NULL, NULL },
+  { "commitments_are_the_same", test_commitments_are_the_same, TT_FORK, NULL,
+    NULL },
+  { "commit_is_authoritative", test_commit_is_authoritative, TT_FORK, NULL,
+    NULL },
+  { "get_phase_str", test_get_phase_str, TT_FORK, NULL, NULL },
+  { "utils_auth", test_utils_auth, TT_FORK, NULL, NULL },
   { "state_transition", test_state_transition, TT_FORK, NULL, NULL },
   { "state_update", test_state_update, TT_FORK,
     NULL, NULL },
diff --git a/src/test/test_slow.c b/src/test/test_slow.c
index bda67b2d92..49b1066dac 100644
--- a/src/test/test_slow.c
+++ b/src/test/test_slow.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -20,7 +20,9 @@
 
 struct testgroup_t testgroups[] = {
   { "slow/crypto/", slow_crypto_tests },
-  { "slow/util/", slow_util_tests },
+  { "slow/process/", slow_process_tests },
+  { "slow/prob_distr/", slow_stochastic_prob_distr_tests },
+  { "slow/ptr/", slow_ptr_tests },
   END_OF_GROUPS
 };
 
diff --git a/src/test/test_socks.c b/src/test/test_socks.c
index 3686e1036b..4a465c7361 100644
--- a/src/test/test_socks.c
+++ b/src/test/test_socks.c
@@ -1,10 +1,10 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_socks.h"
@@ -399,6 +399,43 @@ test_socks_5_supported_commands(void *ptr)
 
   tt_int_op(0,OP_EQ, buf_datalen(buf));
 
+  socks_request_clear(socks);
+
+  /* SOCKS 5 Send RESOLVE_PTR [F1] for an IPv6 address */
+  ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\xF1\x00\x04"
+           "\x20\x01\x0d\xb8\x85\xa3\x00\x00\x00\x00\x8a\x2e\x03\x70\x73\x34"
+           "\x12\x34");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+  tt_str_op("[2001:db8:85a3::8a2e:370:7334]",OP_EQ, socks->address);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+
+  socks_request_clear(socks);
+
+  /* SOCKS 5 Send RESOLVE_PTR [F1] for a an IPv6 address written as a
+   * string with brackets */
+  ADD_DATA(buf, "\x05\x01\x00");
+  ADD_DATA(buf, "\x05\xF1\x00\x03\x1e");
+  ADD_DATA(buf, "[2001:db8:85a3::8a2e:370:7334]");
+  ADD_DATA(buf, "\x12\x34");
+  tt_int_op(fetch_from_buf_socks(buf, socks, get_options()->TestSocks,
+                                 get_options()->SafeSocks),
+            OP_EQ, 1);
+  tt_int_op(5,OP_EQ, socks->socks_version);
+  tt_int_op(2,OP_EQ, socks->replylen);
+  tt_int_op(5,OP_EQ, socks->reply[0]);
+  tt_int_op(0,OP_EQ, socks->reply[1]);
+  tt_str_op("[2001:db8:85a3::8a2e:370:7334]",OP_EQ, socks->address);
+
+  tt_int_op(0,OP_EQ, buf_datalen(buf));
+
  done:
   ;
 }
diff --git a/src/test/test_stats.c b/src/test/test_stats.c
new file mode 100644
index 0000000000..291473ebc9
--- /dev/null
+++ b/src/test/test_stats.c
@@ -0,0 +1,258 @@
+/* Copyright (c) 2001-2004, Roger Dingledine.
+ * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_stats.c
+ * \brief Unit tests for the statistics (reputation history) module.
+ **/
+
+#include "orconfig.h"
+#include "lib/crypt_ops/crypto_rand.h"
+#include "app/config/or_state_st.h"
+#include "test/rng_test_helpers.h"
+
+#include <stdio.h>
+
+#ifdef _WIN32
+/* For mkdir() */
+#include <direct.h>
+#else
+#include <dirent.h>
+#endif /* defined(_WIN32) */
+
+#include <math.h>
+
+/* These macros pull in declarations for some functions and structures that
+ * are typically file-private. */
+#define CIRCUITSTATS_PRIVATE
+#define CIRCUITLIST_PRIVATE
+#define MAINLOOP_PRIVATE
+#define STATEFILE_PRIVATE
+
+#include "core/or/or.h"
+#include "lib/err/backtrace.h"
+#include "lib/buf/buffers.h"
+#include "core/or/circuitstats.h"
+#include "app/config/config.h"
+#include "test/test.h"
+#include "core/mainloop/mainloop.h"
+#include "lib/memarea/memarea.h"
+#include "feature/stats/rephist.h"
+#include "app/config/statefile.h"
+
+/** Run unit tests for some stats code. */
+static void
+test_stats(void *arg)
+{
+  time_t now = 1281533250; /* 2010-08-11 13:27:30 UTC */
+  char *s = NULL;
+  int i;
+
+  /* Start with testing exit port statistics; we shouldn't collect exit
+   * stats without initializing them. */
+  (void)arg;
+  rep_hist_note_exit_stream_opened(80);
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note some streams and bytes, and generate history
+   * string. */
+  rep_hist_exit_stats_init(now);
+  rep_hist_note_exit_stream_opened(80);
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  rep_hist_note_exit_stream_opened(443);
+  rep_hist_note_exit_bytes(443, 100, 10000);
+  rep_hist_note_exit_bytes(443, 100, 10000);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "exit-kibibytes-written 80=1,443=1,other=0\n"
+             "exit-kibibytes-read 80=10,443=20,other=0\n"
+             "exit-streams-opened 80=4,443=4,other=0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Add a few bytes on 10 more ports and ensure that only the top 10
+   * ports are contained in the history string. */
+  for (i = 50; i < 60; i++) {
+    rep_hist_note_exit_bytes(i, i, i);
+    rep_hist_note_exit_stream_opened(i);
+  }
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "exit-kibibytes-written 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
+             "59=1,80=1,443=1,other=1\n"
+             "exit-kibibytes-read 52=1,53=1,54=1,55=1,56=1,57=1,58=1,"
+             "59=1,80=10,443=20,other=1\n"
+             "exit-streams-opened 52=4,53=4,54=4,55=4,56=4,57=4,58=4,"
+             "59=4,80=4,443=4,other=4\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add some bytes, and ensure we don't generate
+   * a history string. */
+  rep_hist_exit_stats_term();
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add some bytes, reset stats, and see what history we
+   * get when observing no streams or bytes at all. */
+  rep_hist_exit_stats_init(now);
+  rep_hist_note_exit_stream_opened(80);
+  rep_hist_note_exit_bytes(80, 100, 10000);
+  rep_hist_reset_exit_stats(now);
+  s = rep_hist_format_exit_stats(now + 86400);
+  tt_str_op("exit-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "exit-kibibytes-written other=0\n"
+             "exit-kibibytes-read other=0\n"
+             "exit-streams-opened other=0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Continue with testing connection statistics; we shouldn't collect
+   * conn stats without initializing them. */
+  rep_hist_note_or_conn_bytes(1, 20, 400, now);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, note bytes, and generate history string. */
+  rep_hist_conn_stats_init(now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,1,0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add some bytes, and ensure we don't generate
+   * a history string. */
+  rep_hist_conn_stats_term();
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add some bytes, reset stats, and see what history we
+   * get when observing no bytes at all. */
+  rep_hist_conn_stats_init(now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now);
+  rep_hist_note_or_conn_bytes(1, 30000, 400000, now + 5);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 10);
+  rep_hist_note_or_conn_bytes(2, 400000, 30000, now + 15);
+  rep_hist_reset_conn_stats(now);
+  s = rep_hist_format_conn_stats(now + 86400);
+  tt_str_op("conn-bi-direct 2010-08-12 13:27:30 (86400 s) 0,0,0,0\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Continue with testing buffer statistics; we shouldn't collect buffer
+   * stats without initializing them. */
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Initialize stats, add statistics for a single circuit, and generate
+   * the history string. */
+  rep_hist_buffer_stats_init(now);
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "cell-processed-cells 20,0,0,0,0,0,0,0,0,0\n"
+             "cell-queued-cells 2.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
+                               "0.00,0.00\n"
+             "cell-time-in-queue 2,0,0,0,0,0,0,0,0,0\n"
+             "cell-circuits-per-decile 1\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Add nineteen more circuit statistics to the one that's already in the
+   * history to see that the math works correctly. */
+  for (i = 21; i < 30; i++)
+    rep_hist_add_buffer_stats(2.0, 2.0, i);
+  for (i = 20; i < 30; i++)
+    rep_hist_add_buffer_stats(3.5, 3.5, i);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "cell-processed-cells 29,28,27,26,25,24,23,22,21,20\n"
+             "cell-queued-cells 2.75,2.75,2.75,2.75,2.75,2.75,2.75,2.75,"
+                               "2.75,2.75\n"
+             "cell-time-in-queue 3,3,3,3,3,3,3,3,3,3\n"
+             "cell-circuits-per-decile 2\n",OP_EQ, s);
+  tor_free(s);
+
+  /* Stop collecting stats, add statistics for one circuit, and ensure we
+   * don't generate a history string. */
+  rep_hist_buffer_stats_term();
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_ptr_op(s, OP_EQ, NULL);
+
+  /* Re-start stats, add statistics for one circuit, reset stats, and make
+   * sure that the history has all zeros. */
+  rep_hist_buffer_stats_init(now);
+  rep_hist_add_buffer_stats(2.0, 2.0, 20);
+  rep_hist_reset_buffer_stats(now);
+  s = rep_hist_format_buffer_stats(now + 86400);
+  tt_str_op("cell-stats-end 2010-08-12 13:27:30 (86400 s)\n"
+             "cell-processed-cells 0,0,0,0,0,0,0,0,0,0\n"
+             "cell-queued-cells 0.00,0.00,0.00,0.00,0.00,0.00,0.00,0.00,"
+                               "0.00,0.00\n"
+             "cell-time-in-queue 0,0,0,0,0,0,0,0,0,0\n"
+             "cell-circuits-per-decile 0\n",OP_EQ, s);
+
+ done:
+  tor_free(s);
+}
+
+/** Run unit tests the mtbf stats code. */
+static void
+test_rephist_mtbf(void *arg)
+{
+  (void)arg;
+
+  time_t now = 1572500000; /* 2010-10-31 05:33:20 UTC */
+  time_t far_future = MAX(now, time(NULL)) + 365*24*60*60;
+  int r;
+
+  /* Make a temporary datadir for these tests */
+  char *ddir_fname = tor_strdup(get_fname_rnd("datadir_mtbf"));
+  tor_free(get_options_mutable()->DataDirectory);
+  get_options_mutable()->DataDirectory = tor_strdup(ddir_fname);
+  check_private_dir(ddir_fname, CPD_CREATE, NULL);
+
+  rep_history_clean(far_future);
+
+  /* No data */
+
+  r = rep_hist_load_mtbf_data(now);
+  tt_int_op(r, OP_EQ, -1);
+  rep_history_clean(far_future);
+
+  /* Blank data */
+
+  r = rep_hist_record_mtbf_data(now, 0);
+  tt_int_op(r, OP_EQ, 0);
+  r = rep_hist_load_mtbf_data(now);
+  tt_int_op(r, OP_EQ, 0);
+  rep_history_clean(far_future);
+
+  r = rep_hist_record_mtbf_data(now, 1);
+  tt_int_op(r, OP_EQ, 0);
+  r = rep_hist_load_mtbf_data(now);
+  tt_int_op(r, OP_EQ, 0);
+  rep_history_clean(far_future);
+
+ done:
+  rep_history_clean(far_future);
+  tor_free(ddir_fname);
+}
+
+#define ENT(name)                                                       \
+  { #name, test_ ## name , 0, NULL, NULL }
+#define FORK(name)                                                      \
+  { #name, test_ ## name , TT_FORK, NULL, NULL }
+
+struct testcase_t stats_tests[] = {
+  FORK(stats),
+  ENT(rephist_mtbf),
+
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_status.c b/src/test/test_status.c
index 9c47469975..82afe0fd2a 100644
--- a/src/test/test_status.c
+++ b/src/test/test_status.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014-2019, The Tor Project, Inc. */
+/* Copyright (c) 2014-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define STATUS_PRIVATE
@@ -33,10 +33,6 @@
 
 #include "test/test.h"
 
-#define NS_MODULE status
-
-#define NS_SUBMODULE count_circuits
-
 /*
  * Test that count_circuits() is correctly counting the number of
  * global circuits.
@@ -44,10 +40,10 @@
 
 static smartlist_t * mock_global_circuitlist = NULL;
 
-NS_DECL(smartlist_t *, circuit_get_global_list, (void));
+static smartlist_t * status_count_circuits_circuit_get_global_list(void);
 
 static void
-NS(test_main)(void *arg)
+test_status_count_circuits(void *arg)
 {
   /* Choose origin_circuit_t wlog. */
   origin_circuit_t *mock_circuit1, *mock_circuit2;
@@ -61,7 +57,8 @@ NS(test_main)(void *arg)
   smartlist_add(mock_global_circuitlist, TO_CIRCUIT(mock_circuit1));
   smartlist_add(mock_global_circuitlist, TO_CIRCUIT(mock_circuit2));
 
-  NS_MOCK(circuit_get_global_list);
+  MOCK(circuit_get_global_list,
+       status_count_circuits_circuit_get_global_list);
 
   actual_circuits = count_circuits();
 
@@ -72,25 +69,22 @@ NS(test_main)(void *arg)
   tor_free(mock_circuit2);
   smartlist_free(mock_global_circuitlist);
   mock_global_circuitlist = NULL;
-  NS_UNMOCK(circuit_get_global_list);
+  UNMOCK(circuit_get_global_list);
 }
 
 static smartlist_t *
-NS(circuit_get_global_list)(void)
+status_count_circuits_circuit_get_global_list(void)
 {
   return mock_global_circuitlist;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE secs_to_uptime
-
 /*
  * Test that secs_to_uptime() is converting the number of seconds that
  * Tor is up for into the appropriate string form containing hours and minutes.
  */
 
 static void
-NS(test_main)(void *arg)
+test_status_secs_to_uptime(void *arg)
 {
   const char *expected;
   char *actual;
@@ -161,9 +155,6 @@ NS(test_main)(void *arg)
       tor_free(actual);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE bytes_to_usage
-
 /*
  * Test that bytes_to_usage() is correctly converting the number of bytes that
  * Tor has read/written into the appropriate string form containing kilobytes,
@@ -171,7 +162,7 @@ NS(test_main)(void *arg)
  */
 
 static void
-NS(test_main)(void *arg)
+test_status_bytes_to_usage(void *arg)
 {
   const char *expected;
   char *actual;
@@ -242,29 +233,30 @@ NS(test_main)(void *arg)
       tor_free(actual);
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, fails)
-
 /*
  * Tests that log_heartbeat() fails when in the public server mode,
  * not hibernating, and we couldn't get the current routerinfo.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
+static double status_hb_fails_tls_get_write_overhead_ratio(void);
+static int status_hb_fails_we_are_hibernating(void);
+static int status_hb_fails_public_server_mode(const or_options_t *options);
+static const routerinfo_t * status_hb_fails_router_get_my_routerinfo(void);
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_fails(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(router_get_my_routerinfo);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_fails_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_fails_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_fails_public_server_mode);
+  MOCK(router_get_my_routerinfo,
+       status_hb_fails_router_get_my_routerinfo);
 
   expected = -1;
   actual = log_heartbeat(0);
@@ -272,26 +264,26 @@ NS(test_main)(void *arg)
   tt_int_op(actual, OP_EQ, expected);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(router_get_my_routerinfo);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(router_get_my_routerinfo);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_fails_tls_get_write_overhead_ratio(void)
 {
   return 2.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_fails_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_fails_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -299,43 +291,51 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+status_hb_fails_router_get_my_routerinfo(void)
 {
   return NULL;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, not_in_consensus)
-
 /*
  * Tests that log_heartbeat() logs appropriately if we are not in the cached
  * consensus.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(const routerinfo_t *, router_get_my_routerinfo, (void));
-NS_DECL(const node_t *, node_get_by_id, (const char *identity_digest));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
+static double status_hb_not_in_consensus_tls_get_write_overhead_ratio(void);
+static int status_hb_not_in_consensus_we_are_hibernating(void);
+static int status_hb_not_in_consensus_public_server_mode(
+             const or_options_t *options);
+static const routerinfo_t *status_hb_not_in_consensus_get_my_routerinfo(void);
+static const node_t * status_hb_not_in_consensus_node_get_by_id(
+             const char *identity_digest);
+static void status_hb_not_in_consensus_logv(
+             int severity, log_domain_mask_t domain, const char *funcname,
+             const char *suffix, const char *format, va_list ap);
+static int status_hb_not_in_consensus_logv_called = 0;
+static int status_hb_not_in_consensus_server_mode(const or_options_t *options);
 
 static routerinfo_t *mock_routerinfo;
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_not_in_consensus(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(router_get_my_routerinfo);
-  NS_MOCK(node_get_by_id);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_not_in_consensus_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_not_in_consensus_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_not_in_consensus_public_server_mode);
+  MOCK(router_get_my_routerinfo,
+       status_hb_not_in_consensus_get_my_routerinfo);
+  MOCK(node_get_by_id,
+       status_hb_not_in_consensus_node_get_by_id);
+  MOCK(logv,
+       status_hb_not_in_consensus_logv);
+  MOCK(server_mode,
+       status_hb_not_in_consensus_server_mode);
 
   log_global_min_severity_ = LOG_DEBUG;
   onion_handshakes_requested[ONION_HANDSHAKE_TYPE_TAP] = 1;
@@ -347,33 +347,33 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 6);
+  tt_int_op(status_hb_not_in_consensus_logv_called, OP_EQ, 6);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(router_get_my_routerinfo);
-    NS_UNMOCK(node_get_by_id);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(router_get_my_routerinfo);
+    UNMOCK(node_get_by_id);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
     tor_free(mock_routerinfo);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_not_in_consensus_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_not_in_consensus_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_not_in_consensus_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -381,7 +381,7 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static const routerinfo_t *
-NS(router_get_my_routerinfo)(void)
+status_hb_not_in_consensus_get_my_routerinfo(void)
 {
   mock_routerinfo = tor_malloc(sizeof(routerinfo_t));
 
@@ -389,7 +389,7 @@ NS(router_get_my_routerinfo)(void)
 }
 
 static const node_t *
-NS(node_get_by_id)(const char *identity_digest)
+status_hb_not_in_consensus_node_get_by_id(const char *identity_digest)
 {
   (void)identity_digest;
 
@@ -397,14 +397,14 @@ NS(node_get_by_id)(const char *identity_digest)
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain,
+status_hb_not_in_consensus_logv(int severity, log_domain_mask_t domain,
   const char *funcname, const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_not_in_consensus_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -412,7 +412,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -429,7 +429,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 3:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_circuit_handshake_stats"),
                 OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
@@ -442,19 +442,20 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 4:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "rep_hist_log_link_protocol_counts"),
                 OP_NE, NULL);
       break;
     case 5:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
-      tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s");
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_str_op(format, OP_EQ, "DoS mitigation since startup:%s%s%s%s%s");
       tt_str_op(va_arg(ap, char *), OP_EQ,
                 " 0 circuits killed with too many cells.");
       tt_str_op(va_arg(ap, char *), OP_EQ, " [cc not enabled]");
       tt_str_op(va_arg(ap, char *), OP_EQ, " [conn not enabled]");
       tt_str_op(va_arg(ap, char *), OP_EQ, "");
+      tt_str_op(va_arg(ap, char *), OP_EQ, " 0 INTRODUCE2 rejected.");
       break;
     default:
       tt_abort_msg("unexpected call to logv()");  // TODO: prettyprint args
@@ -462,51 +463,58 @@ NS(logv)(int severity, log_domain_mask_t domain,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_not_in_consensus_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_not_in_consensus_server_mode(const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, simple)
-
 /*
  * Tests that log_heartbeat() correctly logs heartbeat information
  * normally.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
+static double status_hb_simple_tls_get_write_overhead_ratio(void);
+static int status_hb_simple_we_are_hibernating(void);
+static int status_hb_simple_public_server_mode(const or_options_t *options);
+static long status_hb_simple_get_uptime(void);
+static uint64_t status_hb_simple_get_bytes_read(void);
+static uint64_t status_hb_simple_get_bytes_written(void);
+static void status_hb_simple_logv(int severity, log_domain_mask_t domain,
+                                  const char *funcname, const char *suffix,
+                                  const char *format, va_list ap);
+ATTR_UNUSED static int status_hb_simple_logv_called = 0;
+static int status_hb_simple_server_mode(const or_options_t *options);
 
-static int NS(n_msgs) = 0;
+static int status_hb_simple_n_msgs = 0;
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_simple(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_simple_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_simple_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_simple_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_simple_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_simple_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_simple_get_bytes_written);
+  MOCK(logv,
+       status_hb_simple_logv);
+  MOCK(server_mode,
+       status_hb_simple_server_mode);
 
   log_global_min_severity_ = LOG_DEBUG;
 
@@ -514,33 +522,33 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(NS(n_msgs), OP_EQ, 1);
+  tt_int_op(status_hb_simple_n_msgs, OP_EQ, 1);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_simple_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_simple_we_are_hibernating(void)
 {
   return 1;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_simple_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -548,33 +556,34 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_simple_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_simple_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_simple_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
+status_hb_simple_logv(int severity, log_domain_mask_t domain,
+  const char *funcname,
   const char *suffix, const char *format, va_list ap)
 {
   if (severity == LOG_INFO)
     return;
-  ++NS(n_msgs);
+  ++status_hb_simple_n_msgs;
 
   tt_int_op(severity, OP_EQ, LOG_NOTICE);
-  tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+  tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
   tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
   tt_ptr_op(suffix, OP_EQ, NULL);
   tt_str_op(format, OP_EQ,
@@ -591,54 +600,69 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_simple_server_mode(const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, calls_log_accounting)
-
 /*
  * Tests that log_heartbeat() correctly logs heartbeat information
  * and accounting information when configured.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
-NS_DECL(or_state_t *, get_or_state, (void));
-NS_DECL(int, accounting_is_enabled, (const or_options_t *options));
-NS_DECL(time_t, accounting_get_end_time, (void));
-
-static or_state_t * NS(mock_state) = NULL;
-static or_options_t * NS(mock_options) = NULL;
+static double status_hb_calls_log_accounting_tls_get_write_overhead_ratio(
+                       void);
+static int status_hb_calls_log_accounting_we_are_hibernating(void);
+static int status_hb_calls_log_accounting_public_server_mode(
+                      const or_options_t *options);
+static long status_hb_calls_log_accounting_get_uptime(void);
+static uint64_t status_hb_calls_log_accounting_get_bytes_read(void);
+static uint64_t status_hb_calls_log_accounting_get_bytes_written(void);
+static void status_hb_calls_log_accounting_logv(
+                      int severity, log_domain_mask_t domain,
+                      const char *funcname, const char *suffix,
+                      const char *format, va_list ap);
+static int status_hb_calls_log_accounting_logv_called = 0;
+static int status_hb_calls_log_accounting_server_mode(
+                      const or_options_t *options);
+static or_state_t * status_hb_calls_log_accounting_get_or_state(void);
+static int status_hb_calls_log_accounting_accounting_is_enabled(
+                      const or_options_t *options);
+static time_t status_hb_calls_log_accounting_accounting_get_end_time(void);
+
+static or_state_t * status_hb_calls_log_accounting_mock_state = NULL;
+static or_options_t * status_hb_calls_log_accounting_mock_options = NULL;
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_calls_log_accounting(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
-  NS_MOCK(get_or_state);
-  NS_MOCK(accounting_is_enabled);
-  NS_MOCK(accounting_get_end_time);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_calls_log_accounting_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_calls_log_accounting_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_calls_log_accounting_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_calls_log_accounting_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_calls_log_accounting_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_calls_log_accounting_get_bytes_written);
+  MOCK(logv,
+       status_hb_calls_log_accounting_logv);
+  MOCK(server_mode,
+       status_hb_calls_log_accounting_server_mode);
+  MOCK(get_or_state,
+       status_hb_calls_log_accounting_get_or_state);
+  MOCK(accounting_is_enabled,
+       status_hb_calls_log_accounting_accounting_is_enabled);
+  MOCK(accounting_get_end_time,
+       status_hb_calls_log_accounting_accounting_get_end_time);
 
   log_global_min_severity_ = LOG_DEBUG;
 
@@ -646,37 +670,37 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 3);
+  tt_int_op(status_hb_calls_log_accounting_logv_called, OP_EQ, 3);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
-    NS_UNMOCK(accounting_is_enabled);
-    NS_UNMOCK(accounting_get_end_time);
-    tor_free_(NS(mock_state));
-    tor_free_(NS(mock_options));
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
+    UNMOCK(accounting_is_enabled);
+    UNMOCK(accounting_get_end_time);
+    tor_free_(status_hb_calls_log_accounting_mock_state);
+    tor_free_(status_hb_calls_log_accounting_mock_options);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_calls_log_accounting_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_calls_log_accounting_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_calls_log_accounting_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -684,32 +708,32 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_calls_log_accounting_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_calls_log_accounting_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_calls_log_accounting_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain,
+status_hb_calls_log_accounting_logv(int severity, log_domain_mask_t domain,
   const char *funcname, const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_calls_log_accounting_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -723,7 +747,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_accounting"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -750,11 +774,11 @@ NS(logv)(int severity, log_domain_mask_t domain,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_calls_log_accounting_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_calls_log_accounting_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -762,7 +786,8 @@ NS(server_mode)(const or_options_t *options)
 }
 
 static int
-NS(accounting_is_enabled)(const or_options_t *options)
+status_hb_calls_log_accounting_accounting_is_enabled(
+                                        const or_options_t *options)
 {
   (void)options;
 
@@ -770,55 +795,71 @@ NS(accounting_is_enabled)(const or_options_t *options)
 }
 
 static time_t
-NS(accounting_get_end_time)(void)
+status_hb_calls_log_accounting_accounting_get_end_time(void)
 {
   return 60;
 }
 
 static or_state_t *
-NS(get_or_state)(void)
+status_hb_calls_log_accounting_get_or_state(void)
 {
-  NS(mock_state) = tor_malloc_zero(sizeof(or_state_t));
-  NS(mock_state)->AccountingBytesReadInInterval = 0;
-  NS(mock_state)->AccountingBytesWrittenInInterval = 0;
-
-  return NS(mock_state);
+  status_hb_calls_log_accounting_mock_state =
+    tor_malloc_zero(sizeof(or_state_t));
+  status_hb_calls_log_accounting_mock_state
+    ->AccountingBytesReadInInterval = 0;
+  status_hb_calls_log_accounting_mock_state
+    ->AccountingBytesWrittenInInterval = 0;
+
+  return status_hb_calls_log_accounting_mock_state;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, packaged_cell_fullness)
-
 /*
  * Tests that log_heartbeat() correctly logs packaged cell
  * fullness information.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
-NS_DECL(int, accounting_is_enabled, (const or_options_t *options));
+static double status_hb_packaged_cell_fullness_tls_get_write_overhead_ratio(
+                          void);
+static int status_hb_packaged_cell_fullness_we_are_hibernating(void);
+static int status_hb_packaged_cell_fullness_public_server_mode(
+                          const or_options_t *options);
+static long status_hb_packaged_cell_fullness_get_uptime(void);
+static uint64_t status_hb_packaged_cell_fullness_get_bytes_read(void);
+static uint64_t status_hb_packaged_cell_fullness_get_bytes_written(void);
+static void status_hb_packaged_cell_fullness_logv(
+                          int severity, log_domain_mask_t domain,
+                          const char *funcname, const char *suffix,
+                          const char *format, va_list ap);
+static int status_hb_packaged_cell_fullness_logv_called = 0;
+static int status_hb_packaged_cell_fullness_server_mode(
+                          const or_options_t *options);
+static int status_hb_packaged_cell_fullness_accounting_is_enabled(
+                          const or_options_t *options);
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_packaged_cell_fullness(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
-  NS_MOCK(accounting_is_enabled);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_packaged_cell_fullness_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_packaged_cell_fullness_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_packaged_cell_fullness_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_packaged_cell_fullness_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_packaged_cell_fullness_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_packaged_cell_fullness_get_bytes_written);
+  MOCK(logv,
+       status_hb_packaged_cell_fullness_logv);
+  MOCK(server_mode,
+       status_hb_packaged_cell_fullness_server_mode);
+  MOCK(accounting_is_enabled,
+       status_hb_packaged_cell_fullness_accounting_is_enabled);
   log_global_min_severity_ = LOG_DEBUG;
 
   stats_n_data_bytes_packaged = RELAY_PAYLOAD_SIZE;
@@ -827,36 +868,37 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 2);
+  tt_int_op(status_hb_packaged_cell_fullness_logv_called, OP_EQ, 2);
 
   done:
     stats_n_data_bytes_packaged = 0;
     stats_n_data_cells_packaged = 0;
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
-    NS_UNMOCK(accounting_is_enabled);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
+    UNMOCK(accounting_is_enabled);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_packaged_cell_fullness_tls_get_write_overhead_ratio(void)
 {
   return 1.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_packaged_cell_fullness_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_packaged_cell_fullness_public_server_mode(
+                                     const or_options_t *options)
 {
   (void)options;
 
@@ -864,32 +906,33 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_packaged_cell_fullness_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_packaged_cell_fullness_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_packaged_cell_fullness_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
+status_hb_packaged_cell_fullness_logv(int severity,
+    log_domain_mask_t domain, const char *funcname,
     const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_packaged_cell_fullness_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -903,7 +946,7 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -918,11 +961,11 @@ NS(logv)(int severity, log_domain_mask_t domain, const char *funcname,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_packaged_cell_fullness_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_packaged_cell_fullness_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -930,47 +973,60 @@ NS(server_mode)(const or_options_t *options)
 }
 
 static int
-NS(accounting_is_enabled)(const or_options_t *options)
+status_hb_packaged_cell_fullness_accounting_is_enabled(
+                                             const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-#define NS_SUBMODULE ASPECT(log_heartbeat, tls_write_overhead)
-
 /*
  * Tests that log_heartbeat() correctly logs the TLS write overhead information
  * when the TLS write overhead ratio exceeds 1.
  */
 
-NS_DECL(double, tls_get_write_overhead_ratio, (void));
-NS_DECL(int, we_are_hibernating, (void));
-NS_DECL(int, public_server_mode, (const or_options_t *options));
-NS_DECL(long, get_uptime, (void));
-NS_DECL(uint64_t, get_bytes_read, (void));
-NS_DECL(uint64_t, get_bytes_written, (void));
-NS_DECL(void, logv, (int severity, log_domain_mask_t domain,
-    const char *funcname, const char *suffix, const char *format, va_list ap));
-NS_DECL(int, server_mode, (const or_options_t *options));
-NS_DECL(int, accounting_is_enabled, (const or_options_t *options));
+static double status_hb_tls_write_overhead_tls_get_write_overhead_ratio(void);
+static int status_hb_tls_write_overhead_we_are_hibernating(void);
+static int status_hb_tls_write_overhead_public_server_mode(
+               const or_options_t *options);
+static long status_hb_tls_write_overhead_get_uptime(void);
+static uint64_t status_hb_tls_write_overhead_get_bytes_read(void);
+static uint64_t status_hb_tls_write_overhead_get_bytes_written(void);
+static void status_hb_tls_write_overhead_logv(
+               int severity, log_domain_mask_t domain,
+               const char *funcname, const char *suffix,
+               const char *format, va_list ap);
+static int status_hb_tls_write_overhead_logv_called = 0;
+static int status_hb_tls_write_overhead_server_mode(
+               const or_options_t *options);
+static int status_hb_tls_write_overhead_accounting_is_enabled(
+               const or_options_t *options);
 
 static void
-NS(test_main)(void *arg)
+test_status_hb_tls_write_overhead(void *arg)
 {
   int expected, actual;
   (void)arg;
 
-  NS_MOCK(tls_get_write_overhead_ratio);
-  NS_MOCK(we_are_hibernating);
-  NS_MOCK(public_server_mode);
-  NS_MOCK(get_uptime);
-  NS_MOCK(get_bytes_read);
-  NS_MOCK(get_bytes_written);
-  NS_MOCK(logv);
-  NS_MOCK(server_mode);
-  NS_MOCK(accounting_is_enabled);
+  MOCK(tls_get_write_overhead_ratio,
+       status_hb_tls_write_overhead_tls_get_write_overhead_ratio);
+  MOCK(we_are_hibernating,
+       status_hb_tls_write_overhead_we_are_hibernating);
+  MOCK(public_server_mode,
+       status_hb_tls_write_overhead_public_server_mode);
+  MOCK(get_uptime,
+       status_hb_tls_write_overhead_get_uptime);
+  MOCK(get_bytes_read,
+       status_hb_tls_write_overhead_get_bytes_read);
+  MOCK(get_bytes_written,
+       status_hb_tls_write_overhead_get_bytes_written);
+  MOCK(logv,
+       status_hb_tls_write_overhead_logv);
+  MOCK(server_mode,
+       status_hb_tls_write_overhead_server_mode);
+  MOCK(accounting_is_enabled,
+       status_hb_tls_write_overhead_accounting_is_enabled);
   stats_n_data_cells_packaged = 0;
   log_global_min_severity_ = LOG_DEBUG;
 
@@ -978,34 +1034,34 @@ NS(test_main)(void *arg)
   actual = log_heartbeat(0);
 
   tt_int_op(actual, OP_EQ, expected);
-  tt_int_op(CALLED(logv), OP_EQ, 2);
+  tt_int_op(status_hb_tls_write_overhead_logv_called, OP_EQ, 2);
 
   done:
-    NS_UNMOCK(tls_get_write_overhead_ratio);
-    NS_UNMOCK(we_are_hibernating);
-    NS_UNMOCK(public_server_mode);
-    NS_UNMOCK(get_uptime);
-    NS_UNMOCK(get_bytes_read);
-    NS_UNMOCK(get_bytes_written);
-    NS_UNMOCK(logv);
-    NS_UNMOCK(server_mode);
-    NS_UNMOCK(accounting_is_enabled);
+    UNMOCK(tls_get_write_overhead_ratio);
+    UNMOCK(we_are_hibernating);
+    UNMOCK(public_server_mode);
+    UNMOCK(get_uptime);
+    UNMOCK(get_bytes_read);
+    UNMOCK(get_bytes_written);
+    UNMOCK(logv);
+    UNMOCK(server_mode);
+    UNMOCK(accounting_is_enabled);
 }
 
 static double
-NS(tls_get_write_overhead_ratio)(void)
+status_hb_tls_write_overhead_tls_get_write_overhead_ratio(void)
 {
   return 2.0;
 }
 
 static int
-NS(we_are_hibernating)(void)
+status_hb_tls_write_overhead_we_are_hibernating(void)
 {
   return 0;
 }
 
 static int
-NS(public_server_mode)(const or_options_t *options)
+status_hb_tls_write_overhead_public_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -1013,32 +1069,32 @@ NS(public_server_mode)(const or_options_t *options)
 }
 
 static long
-NS(get_uptime)(void)
+status_hb_tls_write_overhead_get_uptime(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_read)(void)
+status_hb_tls_write_overhead_get_bytes_read(void)
 {
   return 0;
 }
 
 static uint64_t
-NS(get_bytes_written)(void)
+status_hb_tls_write_overhead_get_bytes_written(void)
 {
   return 0;
 }
 
 static void
-NS(logv)(int severity, log_domain_mask_t domain,
+status_hb_tls_write_overhead_logv(int severity, log_domain_mask_t domain,
   const char *funcname, const char *suffix, const char *format, va_list ap)
 {
-  switch (CALLED(logv))
+  switch (status_hb_tls_write_overhead_logv_called)
   {
     case 0:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1052,7 +1108,7 @@ NS(logv)(int severity, log_domain_mask_t domain,
       break;
     case 1:
       tt_int_op(severity, OP_EQ, LOG_NOTICE);
-      tt_int_op(domain, OP_EQ, LD_HEARTBEAT);
+      tt_u64_op(domain, OP_EQ, LD_HEARTBEAT);
       tt_ptr_op(strstr(funcname, "log_heartbeat"), OP_NE, NULL);
       tt_ptr_op(suffix, OP_EQ, NULL);
       tt_str_op(format, OP_EQ,
@@ -1067,11 +1123,11 @@ NS(logv)(int severity, log_domain_mask_t domain,
   }
 
   done:
-    CALLED(logv)++;
+    status_hb_tls_write_overhead_logv_called++;
 }
 
 static int
-NS(server_mode)(const or_options_t *options)
+status_hb_tls_write_overhead_server_mode(const or_options_t *options)
 {
   (void)options;
 
@@ -1079,24 +1135,26 @@ NS(server_mode)(const or_options_t *options)
 }
 
 static int
-NS(accounting_is_enabled)(const or_options_t *options)
+status_hb_tls_write_overhead_accounting_is_enabled(const or_options_t *options)
 {
   (void)options;
 
   return 0;
 }
 
-#undef NS_SUBMODULE
-
 struct testcase_t status_tests[] = {
-  TEST_CASE(count_circuits),
-  TEST_CASE(secs_to_uptime),
-  TEST_CASE(bytes_to_usage),
-  TEST_CASE_ASPECT(log_heartbeat, fails),
-  TEST_CASE_ASPECT(log_heartbeat, simple),
-  TEST_CASE_ASPECT(log_heartbeat, not_in_consensus),
-  TEST_CASE_ASPECT(log_heartbeat, calls_log_accounting),
-  TEST_CASE_ASPECT(log_heartbeat, packaged_cell_fullness),
-  TEST_CASE_ASPECT(log_heartbeat, tls_write_overhead),
+  { "count_circuits", test_status_count_circuits, TT_FORK, NULL, NULL },
+  { "secs_to_uptime", test_status_secs_to_uptime, TT_FORK, NULL, NULL },
+  { "bytes_to_usage", test_status_bytes_to_usage, TT_FORK, NULL, NULL },
+  { "hb_fails", test_status_hb_fails, TT_FORK, NULL, NULL },
+  { "hb_simple", test_status_hb_simple, TT_FORK, NULL, NULL },
+  { "hb_not_in_consensus", test_status_hb_not_in_consensus,
+    TT_FORK, NULL, NULL },
+  { "hb_calls_log_accounting", test_status_hb_calls_log_accounting,
+    TT_FORK, NULL, NULL },
+  { "hb_packaged_cell_fullness", test_status_hb_packaged_cell_fullness,
+    TT_FORK, NULL, NULL },
+  { "hb_tls_write_overhead", test_status_hb_tls_write_overhead,
+    TT_FORK, NULL, NULL },
   END_OF_TESTCASES
 };
diff --git a/src/test/test_storagedir.c b/src/test/test_storagedir.c
index 24e45c7428..eb3779cfee 100644
--- a/src/test/test_storagedir.c
+++ b/src/test/test_storagedir.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2017-2019, The Tor Project, Inc. */
+/* Copyright (c) 2017-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
diff --git a/src/test/test_switch_id.c b/src/test/test_switch_id.c
index baddf8d66e..f97af55d17 100644
--- a/src/test/test_switch_id.c
+++ b/src/test/test_switch_id.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2015-2019, The Tor Project, Inc. */
+/* Copyright (c) 2015-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -32,6 +32,46 @@ static const struct {
 };
 
 #if !defined(_WIN32)
+
+/* Returns the first port that we think we can bind to without special
+ * permissions. Usually this function returns 1024. */
+static uint16_t
+unprivileged_port_range_start(void)
+{
+  uint16_t result = 1024;
+
+#if defined(__linux__)
+  char *content = NULL;
+
+  content = read_file_to_str(
+              "/proc/sys/net/ipv4/ip_unprivileged_port_start",
+              0,
+              NULL);
+
+  if (content != NULL) {
+    int ok = 1;
+    uint16_t tmp_result;
+
+    tmp_result = (uint16_t)tor_parse_long(content, 10, 0, 65535, &ok, NULL);
+
+    if (ok) {
+      result = tmp_result;
+    } else {
+      fprintf(stderr,
+              "Unable to convert ip_unprivileged_port_start to integer: %s\n",
+              content);
+    }
+  }
+
+  tor_free(content);
+#endif /* defined(__linux__) */
+
+  return result;
+}
+
+#define PORT_TEST_RANGE_START 600
+#define PORT_TEST_RANGE_END   1024
+
 /* 0 on no, 1 on yes, -1 on failure. */
 static int
 check_can_bind_low_ports(void)
@@ -41,7 +81,7 @@ check_can_bind_low_ports(void)
   memset(&sin, 0, sizeof(sin));
   sin.sin_family = AF_INET;
 
-  for (port = 600; port < 1024; ++port) {
+  for (port = PORT_TEST_RANGE_START; port < PORT_TEST_RANGE_END; ++port) {
     sin.sin_port = htons(port);
     tor_socket_t fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
     if (! SOCKET_OK(fd)) {
@@ -87,7 +127,7 @@ main(int argc, char **argv)
 
   fprintf(stderr, "This test is not supported on your OS.\n");
   return 77;
-#else /* !(defined(_WIN32)) */
+#else /* !defined(_WIN32) */
   const char *username;
   const char *testname;
   if (argc != 3) {
@@ -149,10 +189,24 @@ main(int argc, char **argv)
       /* Succeed if we can do a setuid with capability retention, and doing so
        * does not make us lose the ability to bind low ports */
     {
-      int keepcaps = (test_id == TEST_SETUID_KEEPCAPS);
+      const int keepcaps = (test_id == TEST_SETUID_KEEPCAPS);
       okay = switch_id(username, keepcaps ? SWITCH_ID_KEEP_BINDLOW : 0) == 0;
+
       if (okay) {
-        okay = check_can_bind_low_ports() == keepcaps;
+        /* Only run this check if there are ports we may not be able to bind
+         * to. */
+        const uint16_t min_port = unprivileged_port_range_start();
+
+        if (min_port >= PORT_TEST_RANGE_START &&
+            min_port < PORT_TEST_RANGE_END) {
+          okay = check_can_bind_low_ports() == keepcaps;
+        } else {
+          fprintf(stderr,
+                  "Skipping check for whether we can bind to any "
+                  "privileged ports as the user system seems to "
+                  "allow us to bind to ports even without any "
+                  "capabilities set.\n");
+        }
       }
       break;
     }
diff --git a/src/test/test_switch_id.sh b/src/test/test_switch_id.sh
index 79c44f2eb1..b13bf7602f 100755
--- a/src/test/test_switch_id.sh
+++ b/src/test/test_switch_id.sh
@@ -1,11 +1,11 @@
 #!/bin/sh
 
-if test "`id -u`" != '0'; then
+if test "$(id -u)" != '0'; then
     echo "This test only works when run as root. Skipping." >&2
     exit 77
 fi
 
-if test "`id -u nobody`" = ""; then
+if test "$(id -u nobody)" = ""; then
     echo "This test requires that your system have a 'nobody' user. Sorry." >&2
     exit 1
 fi
diff --git a/src/test/test_threads.c b/src/test/test_threads.c
index 4a5ecc6fae..d5a1834aef 100644
--- a/src/test/test_threads.c
+++ b/src/test/test_threads.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -155,7 +155,7 @@ test_threads_basic(void *arg)
     tor_mutex_free(thread_test_start2_);
 }
 
-typedef struct cv_testinfo_s {
+typedef struct cv_testinfo_t {
   tor_cond_t *cond;
   tor_mutex_t *mutex;
   int value;
diff --git a/src/test/test_token_bucket.c b/src/test/test_token_bucket.c
new file mode 100644
index 0000000000..cf315f2944
--- /dev/null
+++ b/src/test/test_token_bucket.c
@@ -0,0 +1,152 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+/**
+ * \file test_bwmgt.c
+ * \brief tests for bandwidth management / token bucket functions
+ */
+
+#define TOKEN_BUCKET_PRIVATE
+
+#include "core/or/or.h"
+#include "test/test.h"
+
+#include "lib/evloop/token_bucket.h"
+
+// an imaginary time, in timestamp units. Chosen so it will roll over.
+static const uint32_t START_TS = UINT32_MAX - 1000;
+static const uint32_t RATE = 10;
+static const uint32_t BURST = 50;
+
+static void
+test_token_bucket_ctr_init(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+  tt_uint_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_adjust(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Increase burst. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST * 2);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST * 2);
+
+  /* Decrease burst but still above bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST + 10);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST + 10);
+
+  /* Decrease burst below bucket value. */
+  token_bucket_ctr_adjust(&tb, RATE, BURST - 1);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST - 1);
+
+  /* Change rate. */
+  token_bucket_ctr_adjust(&tb, RATE * 2, BURST);
+  tt_uint_op(tb.cfg.rate, OP_EQ, RATE * 2);
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+  tt_uint_op(tb.cfg.burst, OP_EQ, BURST);
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_dec(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Simple decrement by one. */
+  tt_uint_op(0, OP_EQ, token_bucket_ctr_dec(&tb, 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, BURST - 1);
+
+  /* Down to 0. Becomes empty. */
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST - 1));
+  tt_uint_op(tb.counter.bucket, OP_EQ, 0);
+
+  /* Reset and try to underflow. */
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+  tt_uint_op(true, OP_EQ, token_bucket_ctr_dec(&tb, BURST + 1));
+  tt_int_op(tb.counter.bucket, OP_EQ, -1);
+
+  /* Keep underflowing shouldn't flag the bucket as empty. */
+  tt_uint_op(false, OP_EQ, token_bucket_ctr_dec(&tb, BURST));
+  tt_int_op(tb.counter.bucket, OP_EQ, - (int32_t) (BURST + 1));
+
+ done:
+  ;
+}
+
+static void
+test_token_bucket_ctr_refill(void *arg)
+{
+  (void) arg;
+  token_bucket_ctr_t tb;
+
+  token_bucket_ctr_init(&tb, RATE, BURST, START_TS);
+
+  /* Reduce of half the bucket and let a single second go before refill. */
+  token_bucket_ctr_dec(&tb, BURST / 2);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST / 2);
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* No time change, nothing should move. */
+  token_bucket_ctr_refill(&tb, START_TS + 1);
+  tt_int_op(tb.counter.bucket, OP_EQ, (BURST / 2) + RATE);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 1);
+
+  /* Add 99 seconds, bucket should be back to a full BURST. */
+  token_bucket_ctr_refill(&tb, START_TS + 99);
+  tt_int_op(tb.counter.bucket, OP_EQ, BURST);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 99);
+
+  /* Empty bucket at once. */
+  token_bucket_ctr_dec(&tb, BURST);
+  tt_int_op(tb.counter.bucket, OP_EQ, 0);
+  /* On second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 100);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 100);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE);
+  /* A second second passes. */
+  token_bucket_ctr_refill(&tb, START_TS + 101);
+  tt_int_op(tb.last_refilled_at_timestamp, OP_EQ, START_TS + 101);
+  tt_int_op(tb.counter.bucket, OP_EQ, RATE * 2);
+
+ done:
+  ;
+}
+
+#define TOKEN_BUCKET(name)                                          \
+  { #name, test_token_bucket_ ## name , 0, NULL, NULL }
+
+struct testcase_t token_bucket_tests[] = {
+  TOKEN_BUCKET(ctr_init),
+  TOKEN_BUCKET(ctr_adjust),
+  TOKEN_BUCKET(ctr_dec),
+  TOKEN_BUCKET(ctr_refill),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_tortls.c b/src/test/test_tortls.c
index 853abc4f91..12ba873650 100644
--- a/src/test/test_tortls.c
+++ b/src/test/test_tortls.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
@@ -225,12 +225,12 @@ test_tortls_tor_tls_get_error(void *data)
 
  done:
   UNMOCK(tor_tls_cert_matches_key);
-  NS_UNMOCK(logv);
+  UNMOCK(logv);
   crypto_pk_free(key1);
   crypto_pk_free(key2);
   tor_tls_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_x509_cert_get_id_digests(void *ignored)
@@ -347,7 +347,7 @@ test_tortls_server_got_renegotiate(void *ignored)
  done:
   tor_free(tls);
 }
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
 
 static void
 test_tortls_evaluate_ecgroup_for_tls(void *ignored)
@@ -598,7 +598,7 @@ struct testcase_t tortls_tests[] = {
   LOCAL_TEST_CASE(get_forced_write_size, 0),
   LOCAL_TEST_CASE(used_v1_handshake, TT_FORK),
   LOCAL_TEST_CASE(server_got_renegotiate, 0),
-#endif
+#endif /* defined(ENABLE_OPENSSL) */
   LOCAL_TEST_CASE(evaluate_ecgroup_for_tls, 0),
   LOCAL_TEST_CASE(double_init, TT_FORK),
   LOCAL_TEST_CASE(address, TT_FORK),
diff --git a/src/test/test_tortls.h b/src/test/test_tortls.h
index 1a8b117d0f..21c6fa0a8f 100644
--- a/src/test/test_tortls.h
+++ b/src/test/test_tortls.h
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #ifndef TEST_TORTLS_H
@@ -10,4 +10,4 @@ extern const char *notCompletelyValidCertString;
 extern const char *validCertString;
 extern const char *caCertString;
 
-#endif
+#endif /* !defined(TEST_TORTLS_H) */
diff --git a/src/test/test_tortls_openssl.c b/src/test/test_tortls_openssl.c
index 81c65d7446..c1a87fbb4f 100644
--- a/src/test/test_tortls_openssl.c
+++ b/src/test/test_tortls_openssl.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TORTLS_PRIVATE
@@ -16,7 +16,7 @@
 
 /* Some versions of OpenSSL declare SSL_get_selected_srtp_profile twice in
  * srtp.h. Suppress the GCC warning so we can build with -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/opensslv.h>
 
@@ -29,7 +29,7 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/evp.h>
 #include <openssl/bn.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
+ENABLE_GCC_WARNING("-Wredundant-decls")
 
 #include "core/or/or.h"
 #include "lib/log/log.h"
@@ -46,8 +46,6 @@ ENABLE_GCC_WARNING(redundant-decls)
 #include "test/log_test_helpers.h"
 #include "test/test_tortls.h"
 
-#define NS_MODULE tortls
-
 #ifndef HAVE_SSL_STATE
 #define OPENSSL_OPAQUE
 #endif
@@ -123,8 +121,6 @@ test_tortls_tor_tls_new(void *data)
   tor_tls_free_all();
 }
 
-#define NS_MODULE tortls
-
 static void
 library_init(void)
 {
@@ -133,7 +129,7 @@ library_init(void)
 #else
   SSL_library_init();
   SSL_load_error_strings();
-#endif
+#endif /* defined(OPENSSL_1_1_API) */
 }
 
 static void
@@ -476,7 +472,7 @@ fake_x509_free(X509 *cert)
     tor_free(cert);
   }
 }
-#endif
+#endif /* !defined(OPENSSL_OPAQUE) */
 
 #ifndef OPENSSL_OPAQUE
 static void
diff --git a/src/test/test_util.c b/src/test/test_util.c
index 7f07370998..00626c7ec0 100644
--- a/src/test/test_util.c
+++ b/src/test/test_util.c
@@ -1,27 +1,26 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
-#define COMPAT_PRIVATE
 #define COMPAT_TIME_PRIVATE
-#define CONTROL_PRIVATE
-#define UTIL_PRIVATE
 #define UTIL_MALLOC_PRIVATE
-#define SOCKET_PRIVATE
-#define SUBPROCESS_PRIVATE
+#define PROCESS_WIN32_PRIVATE
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "feature/control/control.h"
+#include "feature/control/control_proto.h"
 #include "feature/client/transports.h"
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/crypt_ops/crypto_rand.h"
+#include "lib/defs/time.h"
 #include "test/test.h"
 #include "lib/memarea/memarea.h"
 #include "lib/process/waitpid.h"
+#include "lib/process/process_win32.h"
 #include "test/log_test_helpers.h"
 #include "lib/compress/compress.h"
 #include "lib/compress/compress_zstd.h"
@@ -30,8 +29,8 @@
 #include "lib/fs/winlib.h"
 #include "lib/process/env.h"
 #include "lib/process/pidfile.h"
-#include "lib/process/subprocess.h"
 #include "lib/intmath/weakrng.h"
+#include "lib/intmath/muldiv.h"
 #include "lib/thread/numcpus.h"
 #include "lib/math/fp.h"
 #include "lib/math/laplace.h"
@@ -39,6 +38,7 @@
 #include "lib/time/tvdiff.h"
 #include "lib/encoding/confline.h"
 #include "lib/net/socketpair.h"
+#include "lib/malloc/map_anon.h"
 
 #ifdef HAVE_PWD_H
 #include <pwd.h>
@@ -58,6 +58,12 @@
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
+#ifdef HAVE_SYS_MMAN_H
+#include <sys/mman.h>
+#endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
 
 #ifdef _WIN32
 #include <tchar.h>
@@ -66,9 +72,36 @@
 #include <ctype.h>
 #include <float.h>
 
+/* These platforms don't have meaningful pwdb or homedirs. */
+#if defined(_WIN32) || defined(__ANDROID__)
+#define DISABLE_PWDB_TESTS
+#endif
+
 #define INFINITY_DBL ((double)INFINITY)
 #define NAN_DBL ((double)NAN)
 
+/** Test the tor_isinf() wrapper */
+static void
+test_tor_isinf(void *arg)
+{
+  (void) arg;
+
+  tt_assert(tor_isinf(INFINITY_DBL));
+
+  tt_assert(!tor_isinf(NAN_DBL));
+  tt_assert(!tor_isinf(DBL_EPSILON));
+  tt_assert(!tor_isinf(DBL_MAX));
+  tt_assert(!tor_isinf(DBL_MIN));
+
+  tt_assert(!tor_isinf(0.0));
+  tt_assert(!tor_isinf(0.1));
+  tt_assert(!tor_isinf(3));
+  tt_assert(!tor_isinf(3.14));
+
+ done:
+  ;
+}
+
 /* XXXX this is a minimal wrapper to make the unit tests compile with the
  * changed tor_timegm interface. */
 static time_t
@@ -322,6 +355,7 @@ test_util_write_chunks_to_file(void *arg)
   tor_free(temp_str);
 }
 
+#ifndef COCCI
 #define _TFE(a, b, f)  tt_int_op((a).f, OP_EQ, (b).f)
 /** test the minimum set of struct tm fields needed for a unique epoch value
  * this is also the set we use to test tor_timegm */
@@ -334,6 +368,7 @@ test_util_write_chunks_to_file(void *arg)
             _TFE(a, b, tm_min ); \
             _TFE(a, b, tm_sec ); \
           TT_STMT_END
+#endif /* !defined(COCCI) */
 
 static void
 test_util_time(void *arg)
@@ -452,7 +487,6 @@ test_util_time(void *arg)
 
 /* Assume tv_usec is an unsigned integer until proven otherwise */
 #define TV_USEC_MAX UINT_MAX
-#define TOR_USEC_PER_SEC 1000000
 
   /* Overflows in the result type */
 
@@ -1864,7 +1898,57 @@ test_util_config_line_crlf(void *arg)
   tor_free(k); tor_free(v);
 }
 
-#ifndef _WIN32
+static void
+test_util_config_line_partition(void *arg)
+{
+  (void)arg;
+  config_line_t *lines = NULL, *orig, *rest = NULL;
+
+  config_line_append(&lines, "Header", "X");
+  config_line_append(&lines, "Item", "Y");
+  config_line_append(&lines, "Thing", "Z");
+
+  config_line_append(&lines, "HEADER", "X2");
+
+  config_line_append(&lines, "header", "X3");
+  config_line_append(&lines, "Item3", "Foob");
+
+  /* set up h2 and h3 to point to the places where we hope the headers will
+     be. */
+  config_line_t *h2 = lines->next->next->next;
+  config_line_t *h3 = h2->next;
+  tt_str_op(h2->key, OP_EQ, "HEADER");
+  tt_str_op(h3->key, OP_EQ, "header");
+
+  orig = lines;
+  rest = config_lines_partition(lines, "Header");
+  tt_ptr_op(lines, OP_EQ, orig);
+  tt_ptr_op(rest, OP_EQ, h2);
+  tt_str_op(lines->next->key, OP_EQ, "Item");
+  tt_str_op(lines->next->next->key, OP_EQ, "Thing");
+  tt_ptr_op(lines->next->next->next, OP_EQ, NULL);
+  config_free_lines(lines);
+
+  orig = lines = rest;
+  rest = config_lines_partition(lines, "Header");
+  tt_ptr_op(lines, OP_EQ, orig);
+  tt_ptr_op(rest, OP_EQ, h3);
+  tt_ptr_op(lines->next, OP_EQ, NULL);
+  config_free_lines(lines);
+
+  orig = lines = rest;
+  rest = config_lines_partition(lines, "Header");
+  tt_ptr_op(lines, OP_EQ, orig);
+  tt_ptr_op(rest, OP_EQ, NULL);
+  tt_str_op(lines->next->key, OP_EQ, "Item3");
+  tt_ptr_op(lines->next->next, OP_EQ, NULL);
+
+ done:
+  config_free_lines(lines);
+  config_free_lines(rest);
+}
+
+#ifndef DISABLE_PWDB_TESTS
 static void
 test_util_expand_filename(void *arg)
 {
@@ -1961,7 +2045,7 @@ test_util_expand_filename(void *arg)
  done:
   tor_free(str);
 }
-#endif /* !defined(_WIN32) */
+#endif /* !defined(DISABLE_PWDB_TESTS) */
 
 /** Test tor_escape_str_for_pt_args(). */
 static void
@@ -2106,14 +2190,14 @@ test_util_strmisc(void *arg)
   /* Test mem_is_zero */
   memset(buf,0,128);
   buf[128] = 'x';
-  tt_assert(tor_mem_is_zero(buf, 10));
-  tt_assert(tor_mem_is_zero(buf, 20));
-  tt_assert(tor_mem_is_zero(buf, 128));
-  tt_assert(!tor_mem_is_zero(buf, 129));
+  tt_assert(fast_mem_is_zero(buf, 10));
+  tt_assert(fast_mem_is_zero(buf, 20));
+  tt_assert(fast_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 129));
   buf[60] = (char)255;
-  tt_assert(!tor_mem_is_zero(buf, 128));
+  tt_assert(!fast_mem_is_zero(buf, 128));
   buf[0] = (char)1;
-  tt_assert(!tor_mem_is_zero(buf, 10));
+  tt_assert(!fast_mem_is_zero(buf, 10));
 
   /* Test 'escaped' */
   tt_ptr_op(escaped(NULL), OP_EQ, NULL);
@@ -2227,15 +2311,6 @@ test_util_strmisc(void *arg)
   tt_int_op(strcmp_opt(NULL,  "foo"), OP_LT, 0);
   tt_int_op(strcmp_opt("foo", NULL),  OP_GT, 0);
 
-  /* Test strcmp_len */
-  tt_int_op(strcmp_len("foo", "bar", 3),   OP_GT, 0);
-  tt_int_op(strcmp_len("foo", "bar", 2),   OP_LT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 4), OP_GT, 0);
-  tt_int_op(strcmp_len("foo2", "foo1", 3), OP_LT, 0); /* Really stop at len */
-  tt_int_op(strcmp_len("foo2", "foo", 3), OP_EQ, 0);  /* Really stop at len */
-  tt_int_op(strcmp_len("blah", "", 4),     OP_GT, 0);
-  tt_int_op(strcmp_len("blah", "", 0),    OP_EQ, 0);
-
  done:
   tor_free(cp_tmp);
 }
@@ -3817,7 +3892,7 @@ test_util_memarea(void *arg)
   tt_int_op(((uintptr_t)p3) % sizeof(void*),OP_EQ, 0);
   tt_assert(!memarea_owns_ptr(area, p3+8192));
   tt_assert(!memarea_owns_ptr(area, p3+30));
-  tt_assert(tor_mem_is_zero(p2, 52));
+  tt_assert(fast_mem_is_zero(p2, 52));
   /* Make sure we don't overalign. */
   p1 = memarea_alloc(area, 1);
   p2 = memarea_alloc(area, 1);
@@ -4098,6 +4173,13 @@ test_util_string_is_utf8(void *ptr)
   tt_int_op(1, OP_EQ, string_is_utf8("ascii\x7f\n", 7));
   tt_int_op(1, OP_EQ, string_is_utf8("Risqu\u00e9=1", 9));
 
+  /* Test the utf8_no_bom function */
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFF", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFFFE", 3));
+  tt_int_op(0, OP_EQ, string_is_utf8_no_bom("\uFEFFlove", 7));
+  tt_int_op(1, OP_EQ, string_is_utf8_no_bom("loveandrespect",
+                                            strlen("loveandrespect")));
+
   // Validate exactly 'len' bytes.
   tt_int_op(0, OP_EQ, string_is_utf8("\0\x80", 2));
   tt_int_op(0, OP_EQ, string_is_utf8("Risqu\u00e9=1", 6));
@@ -4125,10 +4207,43 @@ test_util_string_is_utf8(void *ptr)
   tt_int_op(0, OP_EQ, string_is_utf8("\xed\xbf\xbf", 3));
   tt_int_op(1, OP_EQ, string_is_utf8("\xee\x80\x80", 3));
 
-  // The maximum legal codepoint, 10FFFF.
+  // The minimum legal codepoint, 0x00.
+  tt_int_op(1, OP_EQ, string_is_utf8("\0", 1));
+
+  // The maximum legal codepoint, 0x10FFFF.
   tt_int_op(1, OP_EQ, string_is_utf8("\xf4\x8f\xbf\xbf", 4));
   tt_int_op(0, OP_EQ, string_is_utf8("\xf4\x90\x80\x80", 4));
 
+  /* Test cases that vary between programming languages /
+   * UTF-8 implementations.
+   * Source: POC||GTFO 19, page 43
+   * https://www.alchemistowl.org/pocorgtfo/
+   */
+
+  // Invalid (in most implementations)
+  // surrogate
+  tt_int_op(0, OP_EQ, string_is_utf8("\xed\xa0\x81", 3));
+  // nullsurrog
+  tt_int_op(0, OP_EQ, string_is_utf8("\x30\x00\xed\xa0\x81", 5));
+  // threehigh
+  tt_int_op(0, OP_EQ, string_is_utf8("\xed\xbf\xbf", 3));
+  // fourhigh
+  tt_int_op(0, OP_EQ, string_is_utf8("\xf4\x90\xbf\xbf", 4));
+  // fivebyte
+  tt_int_op(0, OP_EQ, string_is_utf8("\xfb\x80\x80\x80\x80", 5));
+  // sixbyte
+  tt_int_op(0, OP_EQ, string_is_utf8("\xfd\x80\x80\x80\x80", 5));
+  // sixhigh
+  tt_int_op(0, OP_EQ, string_is_utf8("\xfd\xbf\xbf\xbf\xbf", 5));
+
+  // Valid (in most implementations)
+  // fourbyte
+  tt_int_op(1, OP_EQ, string_is_utf8("\xf0\x90\x8d\x88", 4));
+  // fourbyte2
+  tt_int_op(1, OP_EQ, string_is_utf8("\xf0\xbf\xbf\xbf", 4));
+  // nullbyte
+  tt_int_op(1, OP_EQ, string_is_utf8("\x30\x31\x32\x00\x33", 5));
+
  done:
   ;
 }
@@ -4368,204 +4483,6 @@ test_util_load_win_lib(void *ptr)
 }
 #endif /* defined(_WIN32) */
 
-#ifndef _WIN32
-static void
-clear_hex_errno(char *hex_errno)
-{
-  memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1);
-}
-
-static void
-test_util_exit_status(void *ptr)
-{
-  /* Leave an extra byte for a \0 so we can do string comparison */
-  char hex_errno[HEX_ERRNO_SIZE + 1];
-  int n;
-
-  (void)ptr;
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0, hex_errno);
-  tt_str_op("0/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-#if SIZEOF_INT == 4
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x80000000, hex_errno);
-  tt_str_op("FF/-80000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#elif SIZEOF_INT == 8
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno);
-  tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno);
-  tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-  tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE);
-
-#endif /* SIZEOF_INT == 4 || ... */
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x7F, 0, hex_errno);
-  tt_str_op("7F/0\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  n = format_helper_exit_status(0x08, -0x242, hex_errno);
-  tt_str_op("8/-242\n",OP_EQ, hex_errno);
-  tt_int_op(n,OP_EQ, strlen(hex_errno));
-
-  clear_hex_errno(hex_errno);
-  tt_str_op("",OP_EQ, hex_errno);
-
- done:
-  ;
-}
-#endif /* !defined(_WIN32) */
-
-#ifndef _WIN32
-static void
-test_util_string_from_pipe(void *ptr)
-{
-  int test_pipe[2] = {-1, -1};
-  int retval = 0;
-  enum stream_status status = IO_STREAM_TERM;
-  ssize_t retlen;
-  char buf[4] = { 0 };
-
-  (void)ptr;
-
-  errno = 0;
-
-  /* Set up a pipe to test on */
-  retval = pipe(test_pipe);
-  tt_int_op(retval, OP_EQ, 0);
-
-  /* Send in a string. */
-  retlen = write(test_pipe[1], "ABC", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "ABC");
-  errno = 0;
-
-  /* Send in a string that contains a nul. */
-  retlen = write(test_pipe[1], "AB\0", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a nul only. */
-  retlen = write(test_pipe[1], "\0", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string that contains a trailing newline. */
-  retlen = write(test_pipe[1], "AB\n", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Send in a string that contains a newline only. */
-  retlen = write(test_pipe[1], "\n", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "");
-  errno = 0;
-
-  /* Send in a string and check that we nul terminate return values. */
-  retlen = write(test_pipe[1], "AAA", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AAA");
-  tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf));
-  errno = 0;
-
-  retlen = write(test_pipe[1], "B", 1);
-  tt_int_op(retlen, OP_EQ, 1);
-
-  memset(buf, '\xff', sizeof(buf));
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "B");
-  tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf));
-  errno = 0;
-
-  /* Send in multiple lines. */
-  retlen = write(test_pipe[1], "A\nB", 3);
-  tt_int_op(retlen, OP_EQ, 3);
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "A\nB");
-  errno = 0;
-
-  /* Send in a line and close */
-  retlen = write(test_pipe[1], "AB", 2);
-  tt_int_op(retlen, OP_EQ, 2);
-  retval = close(test_pipe[1]);
-  tt_int_op(retval, OP_EQ, 0);
-  test_pipe[1] = -1;
-
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_OKAY);
-  tt_str_op(buf, OP_EQ, "AB");
-  errno = 0;
-
-  /* Check for EOF */
-  status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1);
-  tt_int_op(errno, OP_EQ, 0);
-  tt_int_op(status, OP_EQ, IO_STREAM_CLOSED);
-  errno = 0;
-
- done:
-  if (test_pipe[0] != -1)
-    close(test_pipe[0]);
-  if (test_pipe[1] != -1)
-    close(test_pipe[1]);
-}
-
-#endif /* !defined(_WIN32) */
-
 /**
  * Test for format_hex_number_sigsafe()
  */
@@ -4660,57 +4577,6 @@ test_util_format_dec_number(void *ptr)
   return;
 }
 
-/**
- * Test that we can properly format a Windows command line
- */
-static void
-test_util_join_win_cmdline(void *ptr)
-{
-  /* Based on some test cases from "Parsing C++ Command-Line Arguments" in
-   * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline
-   * will try to only generate simple cases for the child process to parse;
-   * i.e. we never embed quoted strings in arguments. */
-
-  const char *argvs[][4] = {
-    {"a", "bb", "CCC", NULL}, // Normal
-    {NULL, NULL, NULL, NULL}, // Empty argument list
-    {"", NULL, NULL, NULL}, // Empty argument
-    {"\"a", "b\"b", "CCC\"", NULL}, // Quotes
-    {"a\tbc", "dd  dd", "E", NULL}, // Whitespace
-    {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes
-    {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote
-    {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote
-    { NULL } // Terminator
-  };
-
-  const char *cmdlines[] = {
-    "a bb CCC",
-    "",
-    "\"\"",
-    "\\\"a b\\\"b CCC\\\"",
-    "\"a\tbc\" \"dd  dd\" E",
-    "a\\\\\\b \"de fg\" H",
-    "a\\\\\\\"b \\c D\\",
-    "\"a\\\\b c\" d E",
-    NULL // Terminator
-  };
-
-  int i;
-  char *joined_argv = NULL;
-
-  (void)ptr;
-
-  for (i=0; cmdlines[i]!=NULL; i++) {
-    log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]);
-    joined_argv = tor_join_win_cmdline(argvs[i]);
-    tt_str_op(cmdlines[i],OP_EQ, joined_argv);
-    tor_free(joined_argv);
-  }
-
- done:
-  tor_free(joined_argv);
-}
-
 #define MAX_SPLIT_LINE_COUNT 4
 struct split_lines_test_t {
   const char *orig_line; // Line to be split (may contain \0's)
@@ -4718,67 +4584,6 @@ struct split_lines_test_t {
   const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines
 };
 
-/**
- * Test that we properly split a buffer into lines
- */
-static void
-test_util_split_lines(void *ptr)
-{
-  /* Test cases. orig_line of last test case must be NULL.
-   * The last element of split_line[i] must be NULL. */
-  struct split_lines_test_t tests[] = {
-    {"", 0, {NULL}},
-    {"foo", 3, {"foo", NULL}},
-    {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}},
-    {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}},
-    {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}},
-    {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}},
-    {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}},
-    {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}},
-    {NULL, 0, { NULL }}
-  };
-
-  int i, j;
-  char *orig_line=NULL;
-  smartlist_t *sl=NULL;
-
-  (void)ptr;
-
-  for (i=0; tests[i].orig_line; i++) {
-    sl = smartlist_new();
-    /* Allocate space for string and trailing NULL */
-    orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1);
-    tor_split_lines(sl, orig_line, tests[i].orig_length);
-
-    j = 0;
-    log_info(LD_GENERAL, "Splitting test %d of length %d",
-             i, tests[i].orig_length);
-    SMARTLIST_FOREACH_BEGIN(sl, const char *, line) {
-      /* Check we have not got too many lines */
-      tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j);
-      /* Check that there actually should be a line here */
-      tt_ptr_op(tests[i].split_line[j], OP_NE, NULL);
-      log_info(LD_GENERAL, "Line %d of test %d, should be <%s>",
-               j, i, tests[i].split_line[j]);
-      /* Check that the line is as expected */
-      tt_str_op(line,OP_EQ, tests[i].split_line[j]);
-      j++;
-    } SMARTLIST_FOREACH_END(line);
-    /* Check that we didn't miss some lines */
-    tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]);
-    tor_free(orig_line);
-    smartlist_free(sl);
-    sl = NULL;
-  }
-
- done:
-  tor_free(orig_line);
-  smartlist_free(sl);
-}
-
 static void
 test_util_di_ops(void *arg)
 {
@@ -4865,6 +4670,35 @@ test_util_di_ops(void *arg)
 }
 
 static void
+test_util_memcpy_iftrue_timei(void *arg)
+{
+  (void)arg;
+  char buf1[25];
+  char buf2[25];
+  char buf3[25];
+
+  for (int i = 0; i < 100; ++i) {
+    crypto_rand(buf1, sizeof(buf1));
+    crypto_rand(buf2, sizeof(buf2));
+    memcpy(buf3, buf1, sizeof(buf1));
+
+    /* We just copied buf1 into buf3.  Now we're going to copy buf2 into buf2,
+       iff our coin flip comes up heads. */
+    bool coinflip = crypto_rand_int(2) == 0;
+
+    memcpy_if_true_timei(coinflip, buf3, buf2, sizeof(buf3));
+
+    if (coinflip) {
+      tt_mem_op(buf3, OP_EQ, buf2, sizeof(buf2));
+    } else {
+      tt_mem_op(buf3, OP_EQ, buf1, sizeof(buf1));
+    }
+  }
+ done:
+  ;
+}
+
+static void
 test_util_di_map(void *arg)
 {
   (void)arg;
@@ -5730,6 +5564,13 @@ test_util_socketpair(void *arg)
     tt_skip();
   }
 #endif /* defined(__FreeBSD__) */
+#ifdef ENETUNREACH
+  if (ersatz && socketpair_result == -ENETUNREACH) {
+    /* We can also fail with -ENETUNREACH if we have no network stack at
+     * all. */
+    tt_skip();
+  }
+#endif /* defined(ENETUNREACH) */
   tt_int_op(0, OP_EQ, socketpair_result);
 
   tt_assert(SOCKET_OK(fds[0]));
@@ -5858,7 +5699,7 @@ test_util_hostname_validation(void *arg)
   tt_assert(string_is_valid_nonrfc_hostname("luck.y13."));
 
   // We allow punycode TLDs. For examples, see
-  // http://data.iana.org/TLD/tlds-alpha-by-domain.txt
+  // https://data.iana.org/TLD/tlds-alpha-by-domain.txt
   tt_assert(string_is_valid_nonrfc_hostname("example.xn--l1acc"));
 
   done:
@@ -5882,6 +5723,18 @@ test_util_ipv4_validation(void *arg)
 }
 
 static void
+test_util_ipv6_validation(void *arg)
+{
+  (void)arg;
+
+  tt_assert(string_is_valid_ipv6_address("2a00:1450:401b:800::200e"));
+  tt_assert(!string_is_valid_ipv6_address("11:22::33:44:"));
+
+  done:
+  return;
+}
+
+static void
 test_util_writepid(void *arg)
 {
   (void) arg;
@@ -5965,7 +5818,7 @@ test_util_touch_file(void *arg)
   ;
 }
 
-#ifndef _WIN32
+#ifndef DISABLE_PWDB_TESTS
 static void
 test_util_pwdb(void *arg)
 {
@@ -6037,7 +5890,7 @@ test_util_pwdb(void *arg)
   tor_free(dir);
   teardown_capture_of_logs();
 }
-#endif /* !defined(_WIN32) */
+#endif /* !defined(DISABLE_PWDB_TESTS) */
 
 static void
 test_util_calloc_check(void *arg)
@@ -6285,6 +6138,14 @@ test_util_nowrap_math(void *arg)
   tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(2, UINT32_MAX-1));
   tt_u64_op(UINT32_MAX, OP_EQ, tor_add_u32_nowrap(UINT32_MAX, UINT32_MAX));
 
+  tt_u64_op(0, OP_EQ, tor_mul_u64_nowrap(0, 0));
+  tt_u64_op(1, OP_EQ, tor_mul_u64_nowrap(1, 1));
+  tt_u64_op(2, OP_EQ, tor_mul_u64_nowrap(2, 1));
+  tt_u64_op(4, OP_EQ, tor_mul_u64_nowrap(2, 2));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_mul_u64_nowrap(UINT64_MAX, 1));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_mul_u64_nowrap(2, UINT64_MAX));
+  tt_u64_op(UINT64_MAX, OP_EQ, tor_mul_u64_nowrap(UINT64_MAX, UINT64_MAX));
+
  done:
   ;
 }
@@ -6394,40 +6255,168 @@ test_util_get_unquoted_path(void *arg)
   tor_free(r);
 }
 
+static void
+test_util_map_anon(void *arg)
+{
+  (void)arg;
+  char *ptr = NULL;
+  size_t sz = 16384;
+  unsigned inherit=0;
+
+  /* Basic checks. */
+  ptr = tor_mmap_anonymous(sz, 0, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 3;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz-2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 3);
+
+  /* Try again, with a private (non-swappable) mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_PRIVATE, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+  /* Now let's test a drop-on-fork mapping. */
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  ptr[sz-1] = 10;
+  tt_int_op(ptr[0], OP_EQ, 0);
+  tt_int_op(ptr[sz/2], OP_EQ, 0);
+  tt_int_op(ptr[sz-1], OP_EQ, 10);
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+}
+
+static void
+test_util_map_anon_nofork(void *arg)
+{
+  (void)arg;
+#ifdef _WIN32
+  /* The operating system doesn't support forking. */
+  tt_skip();
+ done:
+  ;
+#else /* !defined(_WIN32) */
+  /* We have the right OS support.  We're going to try marking the buffer as
+   * either zero-on-fork or as drop-on-fork, whichever is supported.  Then we
+   * will fork and send a byte back to the parent process.  This will either
+   * crash, or send zero. */
+
+  char *ptr = NULL;
+  const char TEST_VALUE = 0xd0;
+  size_t sz = 16384;
+  int pipefd[2] = {-1, -1};
+  unsigned inherit=0;
+
+  tor_munmap_anonymous(ptr, sz);
+  ptr = tor_mmap_anonymous(sz, ANONMAP_NOINHERIT, &inherit);
+  tt_ptr_op(ptr, OP_NE, 0);
+  memset(ptr, (uint8_t)TEST_VALUE, sz);
+
+  tt_int_op(0, OP_EQ, pipe(pipefd));
+  pid_t child = fork();
+  if (child == 0) {
+    /* We're in the child. */
+    close(pipefd[0]);
+    ssize_t r = write(pipefd[1], &ptr[sz-1], 1); /* This may crash. */
+    close(pipefd[1]);
+    if (r < 0)
+      exit(1);
+    exit(0);
+  }
+  tt_int_op(child, OP_GT, 0);
+  /* In the parent. */
+  close(pipefd[1]);
+  pipefd[1] = -1;
+  char buf[1];
+  ssize_t r = read(pipefd[0], buf, 1);
+
+  if (inherit == INHERIT_RES_ZERO) {
+    // We should be seeing clear-on-fork behavior.
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, 0); // that byte should be zero.
+  } else if (inherit == INHERIT_RES_DROP) {
+    // We should be seeing noinherit behavior.
+    tt_int_op(r, OP_LE, 0); // child said nothing; it should have crashed.
+  } else {
+    // noinherit isn't implemented.
+    tt_int_op(inherit, OP_EQ, INHERIT_RES_KEEP);
+    tt_int_op((int)r, OP_EQ, 1); // child should send us a byte.
+    tt_int_op(buf[0], OP_EQ, TEST_VALUE); // that byte should be TEST_VALUE.
+  }
+
+  int ws;
+  waitpid(child, &ws, 0);
+
+#ifndef NOINHERIT_CAN_FAIL
+  /* Only if NOINHERIT_CAN_FAIL should it be possible for us to get
+   * INHERIT_KEEP behavior in this case. */
+  tt_int_op(inherit, OP_NE, INHERIT_RES_KEEP);
+#else
+  if (inherit == INHERIT_RES_KEEP) {
+    /* Call this test "skipped", not "passed", since noinherit wasn't
+     * implemented. */
+    tt_skip();
+  }
+#endif /* !defined(NOINHERIT_CAN_FAIL) */
+
+ done:
+  tor_munmap_anonymous(ptr, sz);
+  if (pipefd[0] >= 0) {
+    close(pipefd[0]);
+  }
+  if (pipefd[1] >= 0) {
+    close(pipefd[1]);
+  }
+#endif /* defined(_WIN32) */
+}
+
+#ifndef COCCI
 #define UTIL_LEGACY(name)                                               \
-  { #name, test_util_ ## name , 0, NULL, NULL }
+  { (#name), test_util_ ## name , 0, NULL, NULL }
 
 #define UTIL_TEST(name, flags)                          \
-  { #name, test_util_ ## name, flags, NULL, NULL }
+  { (#name), test_util_ ## name, flags, NULL, NULL }
 
 #define COMPRESS(name, identifier)              \
-  { "compress/" #name, test_util_compress, 0, &compress_setup,          \
+  { ("compress/" #name), test_util_compress, 0, &compress_setup,        \
     (char*)(identifier) }
 
 #define COMPRESS_CONCAT(name, identifier)                               \
-  { "compress_concat/" #name, test_util_decompress_concatenated, 0,     \
+  { ("compress_concat/" #name), test_util_decompress_concatenated, 0,   \
     &compress_setup,                                                    \
     (char*)(identifier) }
 
 #define COMPRESS_JUNK(name, identifier)                                 \
-  { "compress_junk/" #name, test_util_decompress_junk, 0,               \
+  { ("compress_junk/" #name), test_util_decompress_junk, 0,             \
     &compress_setup,                                                    \
     (char*)(identifier) }
 
 #define COMPRESS_DOS(name, identifier)                                  \
-  { "compress_dos/" #name, test_util_decompress_dos, 0,                 \
+  { ("compress_dos/" #name), test_util_decompress_dos, 0,               \
     &compress_setup,                                                    \
     (char*)(identifier) }
 
 #ifdef _WIN32
-#define UTIL_TEST_NO_WIN(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
 #define UTIL_TEST_WIN_ONLY(n, f) UTIL_TEST(n, (f))
-#define UTIL_LEGACY_NO_WIN(n) UTIL_TEST_NO_WIN(n, 0)
 #else
-#define UTIL_TEST_NO_WIN(n, f) UTIL_TEST(n, (f))
-#define UTIL_TEST_WIN_ONLY(n, f) { #n, NULL, TT_SKIP, NULL, NULL }
-#define UTIL_LEGACY_NO_WIN(n) UTIL_LEGACY(n)
-#endif /* defined(_WIN32) */
+#define UTIL_TEST_WIN_ONLY(n, f) { (#n), NULL, TT_SKIP, NULL, NULL }
+#endif
+
+#ifdef DISABLE_PWDB_TESTS
+#define UTIL_TEST_PWDB(n, f) { (#n), NULL, TT_SKIP, NULL, NULL }
+#else
+#define UTIL_TEST_PWDB(n, f) UTIL_TEST(n, (f))
+#endif
+#endif /* !defined(COCCI) */
 
 struct testcase_t util_tests[] = {
   UTIL_LEGACY(time),
@@ -6437,7 +6426,8 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(config_line_comment_character),
   UTIL_LEGACY(config_line_escaped_content),
   UTIL_LEGACY(config_line_crlf),
-  UTIL_LEGACY_NO_WIN(expand_filename),
+  UTIL_TEST(config_line_partition, 0),
+  UTIL_TEST_PWDB(expand_filename, 0),
   UTIL_LEGACY(escape_string_socks),
   UTIL_LEGACY(string_is_key_value),
   UTIL_LEGACY(strmisc),
@@ -6473,6 +6463,7 @@ struct testcase_t util_tests[] = {
   UTIL_LEGACY(path_is_relative),
   UTIL_LEGACY(strtok),
   UTIL_LEGACY(di_ops),
+  UTIL_TEST(memcpy_iftrue_timei, 0),
   UTIL_TEST(di_map, 0),
   UTIL_TEST(round_to_next_multiple_of, 0),
   UTIL_TEST(laplace, 0),
@@ -6487,12 +6478,8 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(nowrap_math, 0),
   UTIL_TEST(num_cpus, 0),
   UTIL_TEST_WIN_ONLY(load_win_lib, 0),
-  UTIL_TEST_NO_WIN(exit_status, 0),
-  UTIL_TEST_NO_WIN(string_from_pipe, 0),
   UTIL_TEST(format_hex_number, 0),
   UTIL_TEST(format_dec_number, 0),
-  UTIL_TEST(join_win_cmdline, 0),
-  UTIL_TEST(split_lines, 0),
   UTIL_TEST(n_bits_set, 0),
   UTIL_TEST(eat_whitespace, 0),
   UTIL_TEST(sl_new_from_text_lines, 0),
@@ -6510,6 +6497,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(mathlog, 0),
   UTIL_TEST(fraction, 0),
   UTIL_TEST(weak_random, 0),
+  { "tor_isinf", test_tor_isinf, TT_FORK, NULL, NULL },
   { "socket_ipv4", test_util_socket, TT_FORK, &passthrough_setup,
     (void*)"4" },
   { "socket_ipv6", test_util_socket, TT_FORK,
@@ -6522,10 +6510,11 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(hostname_validation, 0),
   UTIL_TEST(dest_validation_edgecase, 0),
   UTIL_TEST(ipv4_validation, 0),
+  UTIL_TEST(ipv6_validation, 0),
   UTIL_TEST(writepid, 0),
   UTIL_TEST(get_avail_disk_space, 0),
   UTIL_TEST(touch_file, 0),
-  UTIL_TEST_NO_WIN(pwdb, TT_FORK),
+  UTIL_TEST_PWDB(pwdb, TT_FORK),
   UTIL_TEST(calloc_check, 0),
   UTIL_TEST(monotonic_time, 0),
   UTIL_TEST(monotonic_time_ratchet, TT_FORK),
@@ -6533,5 +6522,7 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(monotonic_time_add_msec, 0),
   UTIL_TEST(htonll, 0),
   UTIL_TEST(get_unquoted_path, 0),
+  UTIL_TEST(map_anon, 0),
+  UTIL_TEST(map_anon_nofork, 0),
   END_OF_TESTCASES
 };
diff --git a/src/test/test_util_format.c b/src/test/test_util_format.c
index d344d0e95c..726e8e7427 100644
--- a/src/test/test_util_format.c
+++ b/src/test/test_util_format.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -7,11 +7,8 @@
 #include "test/test.h"
 
 #include "lib/crypt_ops/crypto_rand.h"
-#define UTIL_FORMAT_PRIVATE
 #include "lib/encoding/binascii.h"
 
-#define NS_MODULE util_format
-
 static void
 test_util_format_unaligned_accessors(void *ignored)
 {
@@ -346,7 +343,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswqmjs";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 10);
     tt_str_op(expected, OP_EQ, dst);
   }
 
@@ -357,7 +354,7 @@ test_util_format_base32_decode(void *arg)
     const char *src = "mjwgc2dcnrswq";
 
     ret = base32_decode(dst, strlen(expected), src, strlen(src));
-    tt_int_op(ret, OP_EQ, 0);
+    tt_int_op(ret, OP_EQ, 8);
     tt_mem_op(expected, OP_EQ, dst, strlen(expected));
   }
 
@@ -367,7 +364,7 @@ test_util_format_base32_decode(void *arg)
     ret = base32_decode(dst, real_dstlen, "#abcde", 6);
     tt_int_op(ret, OP_EQ, -1);
     /* Make sure the destination buffer has been zeroed even on error. */
-    tt_int_op(tor_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
+    tt_int_op(fast_mem_is_zero(dst, real_dstlen), OP_EQ, 1);
   }
 
  done:
@@ -392,10 +389,13 @@ test_util_format_encoded_size(void *arg)
 
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i, 0);
     tt_int_op(strlen(outbuf), OP_EQ, base64_encode_size(i, 0));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
+
     base64_encode(outbuf, sizeof(outbuf), (char *)inbuf, i,
                   BASE64_ENCODE_MULTILINE);
     tt_int_op(strlen(outbuf), OP_EQ,
               base64_encode_size(i, BASE64_ENCODE_MULTILINE));
+    tt_int_op(i, OP_LE, base64_decode_maxsize(strlen(outbuf)));
   }
 
  done:
@@ -417,4 +417,3 @@ struct testcase_t util_format_tests[] = {
   { "encoded_size", test_util_format_encoded_size, 0, NULL, NULL },
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_process.c b/src/test/test_util_process.c
index 4d04eb6dfc..fc79fe9b1f 100644
--- a/src/test/test_util_process.c
+++ b/src/test/test_util_process.c
@@ -1,7 +1,6 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
-#define UTIL_PROCESS_PRIVATE
 #include "orconfig.h"
 #include "core/or/or.h"
 
@@ -12,7 +11,6 @@
 #include "test/log_test_helpers.h"
 
 #ifndef _WIN32
-#define NS_MODULE util_process
 
 static void
 temp_callback(int r, void *s)
@@ -69,15 +67,16 @@ test_util_process_clear_waitpid_callback(void *ignored)
 }
 #endif /* !defined(_WIN32) */
 
+#ifndef COCCI
 #ifndef _WIN32
-#define TEST(name) { #name, test_util_process_##name, 0, NULL, NULL }
+#define TEST(name) { (#name), test_util_process_##name, 0, NULL, NULL }
 #else
-#define TEST(name) { #name, NULL, TT_SKIP, NULL, NULL }
+#define TEST(name) { (#name), NULL, TT_SKIP, NULL, NULL }
 #endif
+#endif /* !defined(COCCI) */
 
 struct testcase_t util_process_tests[] = {
   TEST(set_waitpid_callback),
   TEST(clear_waitpid_callback),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c
deleted file mode 100644
index 29e30eaa11..0000000000
--- a/src/test/test_util_slow.c
+++ /dev/null
@@ -1,396 +0,0 @@
-/* Copyright (c) 2001-2004, Roger Dingledine.
- * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
-/* See LICENSE for licensing information */
-
-#include "orconfig.h"
-#define UTIL_PRIVATE
-#define SUBPROCESS_PRIVATE
-#include "lib/crypt_ops/crypto_cipher.h"
-#include "lib/log/log.h"
-#include "lib/process/subprocess.h"
-#include "lib/process/waitpid.h"
-#include "lib/string/printf.h"
-#include "lib/time/compat_time.h"
-#include "test/test.h"
-
-#include <errno.h>
-#include <string.h>
-
-#ifndef BUILDDIR
-#define BUILDDIR "."
-#endif
-
-#ifdef _WIN32
-#define notify_pending_waitpid_callbacks() STMT_NIL
-#define TEST_CHILD "test-child.exe"
-#define EOL "\r\n"
-#else
-#define TEST_CHILD (BUILDDIR "/src/test/test-child")
-#define EOL "\n"
-#endif /* defined(_WIN32) */
-
-#ifdef _WIN32
-/* I've assumed Windows doesn't have the gap between fork and exec
- * that causes the race condition on unix-like platforms */
-#define MATCH_PROCESS_STATUS(s1,s2)         ((s1) == (s2))
-
-#else /* !(defined(_WIN32)) */
-/* work around a race condition of the timing of SIGCHLD handler updates
- * to the process_handle's fields, and checks of those fields
- *
- * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to
- * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */
-#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING  (PROCESS_STATUS_RUNNING+1)
-#define IS_RUNNING_OR_NOTRUNNING(s)           \
-  ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING)
-/* well, this is ugly */
-#define MATCH_PROCESS_STATUS(s1,s2)           \
-  (  (s1) == (s2)                                         \
-     ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s2))                  \
-     ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING      \
-        && IS_RUNNING_OR_NOTRUNNING(s1)))
-
-#endif /* defined(_WIN32) */
-
-/** Helper function for testing tor_spawn_background */
-static void
-run_util_spawn_background(const char *argv[], const char *expected_out,
-                          const char *expected_err, int expected_exit,
-                          int expected_status)
-{
-  int retval, exit_code;
-  ssize_t pos;
-  process_handle_t *process_handle=NULL;
-  char stdout_buf[100], stderr_buf[100];
-  int status;
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  notify_pending_waitpid_callbacks();
-
-  /* the race condition doesn't affect status,
-   * because status isn't updated by the SIGCHLD handler,
-   * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, status));
-  if (status == PROCESS_STATUS_ERROR) {
-    tt_ptr_op(process_handle, OP_EQ, NULL);
-    return;
-  }
-
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* When a spawned process forks, fails, then exits very quickly,
-   * (this typically occurs when exec fails)
-   * there is a race condition between the SIGCHLD handler
-   * updating the process_handle's fields, and this test
-   * checking the process status in those fields.
-   * The SIGCHLD update can occur before or after the code below executes.
-   * This causes intermittent failures in spawn_background_fail(),
-   * typically when the machine is under load.
-   * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */
-
-  /* the race condition affects the change in
-   * process_handle->status from RUNNING to NOTRUNNING */
-  tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status));
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  /* the race condition affects the change in
-   * process_handle->waitpid_cb to NULL,
-   * so we skip the check if expected_status is ambiguous,
-   * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */
-  tt_assert(process_handle->waitpid_cb != NULL
-              || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING);
-#endif /* !defined(_WIN32) */
-
-#ifdef _WIN32
-  tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE);
-  tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE);
-#else
-  tt_assert(process_handle->stdout_pipe >= 0);
-  tt_assert(process_handle->stderr_pipe >= 0);
-  tt_assert(process_handle->stdin_pipe >= 0);
-#endif /* defined(_WIN32) */
-
-  /* Check stdout */
-  pos = tor_read_all_from_process_stdout(process_handle, stdout_buf,
-                                         sizeof(stdout_buf) - 1);
-  tt_assert(pos >= 0);
-  stdout_buf[pos] = '\0';
-  tt_int_op(strlen(expected_out),OP_EQ, pos);
-  tt_str_op(expected_out,OP_EQ, stdout_buf);
-
-  notify_pending_waitpid_callbacks();
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-  // TODO: Make test-child exit with something other than 0
-
-#ifndef _WIN32
-  notify_pending_waitpid_callbacks();
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
-  notify_pending_waitpid_callbacks();
-
- done:
-  if (process_handle)
-    tor_process_handle_destroy(process_handle, 1);
-}
-
-/** Check that we can launch a process and read the output */
-static void
-test_util_spawn_background_ok(void *ptr)
-{
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL;
-  const char *expected_err = "ERR"EOL;
-
-  (void)ptr;
-
-  run_util_spawn_background(argv, expected_out, expected_err, 0,
-      PROCESS_STATUS_RUNNING);
-}
-
-/** Check that failing to find the executable works as expected */
-static void
-test_util_spawn_background_fail(void *ptr)
-{
-  const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL};
-  const char *expected_err = "";
-  char expected_out[1024];
-  char code[32];
-#ifdef _WIN32
-  const int expected_status = PROCESS_STATUS_ERROR;
-#else
-  /* TODO: Once we can signal failure to exec, set this to be
-   * PROCESS_STATUS_RUNNING_OR_ERROR */
-  const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING;
-#endif /* defined(_WIN32) */
-
-  memset(expected_out, 0xf0, sizeof(expected_out));
-  memset(code, 0xf0, sizeof(code));
-
-  (void)ptr;
-
-  tor_snprintf(code, sizeof(code), "%x/%x",
-    9 /* CHILD_STATE_FAILEXEC */ , ENOENT);
-  tor_snprintf(expected_out, sizeof(expected_out),
-    "ERR: Failed to spawn background process - code %s\n", code);
-
-  run_util_spawn_background(argv, expected_out, expected_err, 255,
-      expected_status);
-}
-
-/** Test that reading from a handle returns a partial read rather than
- * blocking */
-static void
-test_util_spawn_background_partial_read_impl(int exit_early)
-{
-  const int expected_exit = 0;
-  const int expected_status = PROCESS_STATUS_RUNNING;
-
-  int retval, exit_code;
-  ssize_t pos = -1;
-  process_handle_t *process_handle=NULL;
-  int status;
-  char stdout_buf[100], stderr_buf[100];
-
-  const char *argv[] = {TEST_CHILD, "--test", NULL};
-  const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL,
-                                 "DONE" EOL,
-                                 NULL };
-  const char *expected_err = "ERR" EOL;
-
-#ifndef _WIN32
-  int eof = 0;
-#endif
-  int expected_out_ctr;
-
-  if (exit_early) {
-    argv[1] = "--hang";
-    expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL;
-  }
-
-  /* Start the program */
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-  tt_int_op(expected_status,OP_EQ, status);
-  tt_assert(process_handle);
-  tt_int_op(expected_status,OP_EQ, process_handle->status);
-
-  /* Check stdout */
-  for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) {
-#ifdef _WIN32
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL);
-#else
-    /* Check that we didn't read the end of file last time */
-    tt_assert(!eof);
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1, NULL, &eof);
-#endif /* defined(_WIN32) */
-    log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos);
-
-    /* We would have blocked, keep on trying */
-    if (0 == pos)
-      continue;
-
-    tt_assert(pos > 0);
-    stdout_buf[pos] = '\0';
-    tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf);
-    tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos);
-    expected_out_ctr++;
-  }
-
-  if (exit_early) {
-    tor_process_handle_destroy(process_handle, 1);
-    process_handle = NULL;
-    goto done;
-  }
-
-  /* The process should have exited without writing more */
-#ifdef _WIN32
-  pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                            sizeof(stdout_buf) - 1,
-                            process_handle);
-  tt_int_op(0,OP_EQ, pos);
-#else /* !(defined(_WIN32)) */
-  if (!eof) {
-    /* We should have got all the data, but maybe not the EOF flag */
-    pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf,
-                              sizeof(stdout_buf) - 1,
-                              process_handle, &eof);
-    tt_int_op(0,OP_EQ, pos);
-    tt_assert(eof);
-  }
-  /* Otherwise, we got the EOF on the last read */
-#endif /* defined(_WIN32) */
-
-  /* Check it terminated correctly */
-  retval = tor_get_exit_code(process_handle, 1, &exit_code);
-  tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval);
-  tt_int_op(expected_exit,OP_EQ, exit_code);
-
-  // TODO: Make test-child exit with something other than 0
-
-  /* Check stderr */
-  pos = tor_read_all_from_process_stderr(process_handle, stderr_buf,
-                                         sizeof(stderr_buf) - 1);
-  tt_assert(pos >= 0);
-  stderr_buf[pos] = '\0';
-  tt_str_op(expected_err,OP_EQ, stderr_buf);
-  tt_int_op(strlen(expected_err),OP_EQ, pos);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-static void
-test_util_spawn_background_partial_read(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(0);
-}
-
-static void
-test_util_spawn_background_exit_early(void *arg)
-{
-  (void)arg;
-  test_util_spawn_background_partial_read_impl(1);
-}
-
-static void
-test_util_spawn_background_waitpid_notify(void *arg)
-{
-  int retval, exit_code;
-  process_handle_t *process_handle=NULL;
-  int status;
-  int ms_timer;
-
-  const char *argv[] = {TEST_CHILD, "--fast", NULL};
-
-  (void) arg;
-
-#ifdef _WIN32
-  status = tor_spawn_background(NULL, argv, NULL, &process_handle);
-#else
-  status = tor_spawn_background(argv[0], argv, NULL, &process_handle);
-#endif
-
-  tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING);
-  tt_ptr_op(process_handle, OP_NE, NULL);
-
-  /* We're not going to look at the stdout/stderr output this time. Instead,
-   * we're testing whether notify_pending_waitpid_calbacks() can report the
-   * process exit (on unix) and/or whether tor_get_exit_code() can notice it
-   * (on windows) */
-
-#ifndef _WIN32
-  ms_timer = 30*1000;
-  tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL);
-  while (process_handle->waitpid_cb && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-    notify_pending_waitpid_callbacks();
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-  tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL);
-#endif /* !defined(_WIN32) */
-
-  ms_timer = 30*1000;
-  while (((retval = tor_get_exit_code(process_handle, 0, &exit_code))
-                == PROCESS_EXIT_RUNNING) && ms_timer > 0) {
-    tor_sleep_msec(100);
-    ms_timer -= 100;
-  }
-  tt_int_op(ms_timer, OP_GT, 0);
-
-  tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED);
-
- done:
-  tor_process_handle_destroy(process_handle, 1);
-}
-
-#undef TEST_CHILD
-#undef EOL
-
-#undef MATCH_PROCESS_STATUS
-
-#ifndef _WIN32
-#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING
-#undef IS_RUNNING_OR_NOTRUNNING
-#endif
-
-#define UTIL_TEST(name, flags)                          \
-  { #name, test_util_ ## name, flags, NULL, NULL }
-
-struct testcase_t slow_util_tests[] = {
-  UTIL_TEST(spawn_background_ok, 0),
-  UTIL_TEST(spawn_background_fail, 0),
-  UTIL_TEST(spawn_background_partial_read, 0),
-  UTIL_TEST(spawn_background_exit_early, 0),
-  UTIL_TEST(spawn_background_waitpid_notify, 0),
-  END_OF_TESTCASES
-};
diff --git a/src/test/test_voting_flags.c b/src/test/test_voting_flags.c
new file mode 100644
index 0000000000..ae89e43889
--- /dev/null
+++ b/src/test/test_voting_flags.c
@@ -0,0 +1,193 @@
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
+/* See LICENSE for licensing information */
+
+#include "orconfig.h"
+
+#define VOTEFLAGS_PRIVATE
+
+#include "core/or/or.h"
+
+#include "feature/dirauth/voteflags.h"
+#include "feature/dirauth/dirauth_options_st.h"
+#include "feature/nodelist/node_st.h"
+#include "feature/nodelist/routerstatus_st.h"
+#include "feature/nodelist/routerinfo_st.h"
+
+#include "app/config/config.h"
+
+#include "test/test.h"
+#include "test/opts_test_helpers.h"
+
+typedef struct {
+  time_t now;
+  routerinfo_t ri;
+  node_t node;
+
+  routerstatus_t expected;
+} flag_vote_test_cfg_t;
+
+static void
+setup_cfg(flag_vote_test_cfg_t *c)
+{
+  memset(c, 0, sizeof(*c));
+
+  c->now = approx_time();
+
+  c->ri.nickname = (char *) "testing100";
+  strlcpy(c->expected.nickname, "testing100", sizeof(c->expected.nickname));
+
+  memset(c->ri.cache_info.identity_digest, 0xff, DIGEST_LEN);
+  memset(c->ri.cache_info.signed_descriptor_digest, 0xee, DIGEST_LEN);
+
+  c->ri.cache_info.published_on = c->now - 100;
+  c->expected.published_on = c->now - 100;
+
+  c->ri.addr = 0x7f010105;
+  c->expected.addr = 0x7f010105;
+  c->ri.or_port = 9090;
+  c->expected.or_port = 9090;
+
+  tor_addr_make_null(&c->ri.ipv6_addr, AF_INET6);
+  tor_addr_make_null(&c->expected.ipv6_addr, AF_INET6);
+
+  // By default we have no loaded information about stability or speed,
+  // so we'll default to voting "yeah sure." on these two.
+  c->expected.is_fast = 1;
+  c->expected.is_stable = 1;
+}
+
+static bool
+check_result(flag_vote_test_cfg_t *c)
+{
+  bool result = false;
+  routerstatus_t rs;
+  memset(&rs, 0, sizeof(rs));
+  dirauth_set_routerstatus_from_routerinfo(&rs, &c->node, &c->ri, c->now, 0);
+
+  tt_i64_op(rs.published_on, OP_EQ, c->expected.published_on);
+  tt_str_op(rs.nickname, OP_EQ, c->expected.nickname);
+
+  // identity_digest and descriptor_digest are not set here.
+
+  tt_uint_op(rs.addr, OP_EQ, c->expected.addr);
+  tt_uint_op(rs.or_port, OP_EQ, c->expected.or_port);
+  tt_uint_op(rs.dir_port, OP_EQ, c->expected.dir_port);
+
+  tt_assert(tor_addr_eq(&rs.ipv6_addr, &c->expected.ipv6_addr));
+  tt_uint_op(rs.ipv6_orport, OP_EQ, c->expected.ipv6_orport);
+
+#define FLAG(flagname) \
+  tt_uint_op(rs.flagname, OP_EQ, c->expected.flagname)
+
+  FLAG(is_authority);
+  FLAG(is_exit);
+  FLAG(is_stable);
+  FLAG(is_fast);
+  FLAG(is_flagged_running);
+  FLAG(is_named);
+  FLAG(is_unnamed);
+  FLAG(is_valid);
+  FLAG(is_possible_guard);
+  FLAG(is_bad_exit);
+  FLAG(is_hs_dir);
+  FLAG(is_v2_dir);
+  FLAG(is_staledesc);
+  FLAG(has_bandwidth);
+  FLAG(has_exitsummary);
+  FLAG(bw_is_unmeasured);
+
+  result = true;
+
+ done:
+  return result;
+}
+
+static void
+test_voting_flags_minimal(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  (void) check_result(cfg);
+}
+
+static void
+test_voting_flags_ipv6(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+
+  tt_assert(tor_addr_parse(&cfg->ri.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->ri.ipv6_orport = 9091;
+  // no change in expected results, since we aren't set up with ipv6
+  // connectivity.
+  if (!check_result(cfg))
+    goto done;
+
+  get_dirauth_options(get_options_mutable())->AuthDirHasIPv6Connectivity = 1;
+  // no change in expected results, since last_reachable6 won't be set.
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->node.last_reachable6 = cfg->now - 10;
+  // now that lastreachable6 is set, we expect to see the result.
+  tt_assert(tor_addr_parse(&cfg->expected.ipv6_addr, "f00::b42") == AF_INET6);
+  cfg->expected.ipv6_orport = 9091;
+  if (!check_result(cfg))
+    goto done;
+ done:
+  ;
+}
+
+static void
+test_voting_flags_staledesc(void *arg)
+{
+  flag_vote_test_cfg_t *cfg = arg;
+  time_t now = cfg->now;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL + 10;
+  // no change in expectations for is_staledesc
+  if (!check_result(cfg))
+    goto done;
+
+  cfg->ri.cache_info.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.published_on = now - DESC_IS_STALE_INTERVAL - 10;
+  cfg->expected.is_staledesc = 1;
+  if (!check_result(cfg))
+    goto done;
+
+ done:
+  ;
+}
+
+static void *
+setup_voting_flags_test(const struct testcase_t *testcase)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = tor_malloc_zero(sizeof(*cfg));
+  setup_cfg(cfg);
+  return cfg;
+}
+
+static int
+teardown_voting_flags_test(const struct testcase_t *testcase, void *arg)
+{
+  (void)testcase;
+  flag_vote_test_cfg_t *cfg = arg;
+  tor_free(cfg);
+  return 1;
+}
+
+static const struct testcase_setup_t voting_flags_setup = {
+  .setup_fn = setup_voting_flags_test,
+  .cleanup_fn = teardown_voting_flags_test,
+};
+
+#define T(name,flags)                                   \
+  { #name, test_voting_flags_##name, (flags), &voting_flags_setup, NULL }
+
+struct testcase_t voting_flags_tests[] = {
+  T(minimal, 0),
+  T(ipv6, TT_FORK),
+  // TODO: Add more of these tests.
+  T(staledesc, TT_FORK),
+  END_OF_TESTCASES
+};
diff --git a/src/test/test_voting_schedule.c b/src/test/test_voting_schedule.c
index ba4d53a4ae..df64b79167 100644
--- a/src/test/test_voting_schedule.c
+++ b/src/test/test_voting_schedule.c
@@ -1,17 +1,18 @@
-/* Copyright (c) 2018-2019, The Tor Project, Inc. */
+/* Copyright (c) 2018-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
 
 #include "core/or/or.h"
-#include "feature/dircommon/voting_schedule.h"
+#include "feature/dirauth/voting_schedule.h"
+#include "feature/nodelist/networkstatus.h"
 
 #include "test/test.h"
 
 static void
 test_voting_schedule_interval_start(void *arg)
 {
-#define next_interval voting_schedule_get_start_of_next_interval
+#define next_interval voting_sched_get_start_of_interval_after
   (void)arg;
   char buf[ISO_TIME_LEN+1];
 
@@ -61,4 +62,3 @@ struct testcase_t voting_schedule_tests[] = {
   VS(interval_start, 0),
   END_OF_TESTCASES
 };
-
diff --git a/src/test/test_workqueue.c b/src/test/test_workqueue.c
index c58634da5c..3734c08e48 100644
--- a/src/test/test_workqueue.c
+++ b/src/test/test_workqueue.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
@@ -32,7 +32,7 @@ int handled_len;
 bitarray_t *handled;
 #endif
 
-typedef struct state_s {
+typedef struct state_t {
   int magic;
   int n_handled;
   crypto_pk_t *rsa;
@@ -40,13 +40,13 @@ typedef struct state_s {
   int is_shutdown;
 } state_t;
 
-typedef struct rsa_work_s {
+typedef struct rsa_work_t {
   int serial;
   uint8_t msg[128];
   uint8_t msglen;
 } rsa_work_t;
 
-typedef struct ecdh_work_s {
+typedef struct ecdh_work_t {
   int serial;
   union {
     curve25519_public_key_t pk;
@@ -63,7 +63,7 @@ mark_handled(int serial)
   tor_assert(! bitarray_is_set(handled, serial));
   bitarray_set(handled, serial);
   tor_mutex_release(&bitmap_mutex);
-#else /* !(defined(TRACK_RESPONSES)) */
+#else /* !defined(TRACK_RESPONSES) */
   (void)serial;
 #endif /* defined(TRACK_RESPONSES) */
 }
@@ -339,7 +339,7 @@ main(int argc, char **argv)
   replyqueue_t *rq;
   threadpool_t *tp;
   int i;
-  tor_libevent_cfg evcfg;
+  tor_libevent_cfg_t evcfg;
   uint32_t as_flags = 0;
 
   for (i = 1; i < argc; ++i) {
diff --git a/src/test/test_workqueue_cancel.sh b/src/test/test_workqueue_cancel.sh
index f7c663171e..e50b884f26 100755
--- a/src/test/test_workqueue_cancel.sh
+++ b/src/test/test_workqueue_cancel.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue -C 1
+"${builddir:-.}/src/test/test_workqueue" -C 1
 
diff --git a/src/test/test_workqueue_efd.sh b/src/test/test_workqueue_efd.sh
index 4d89396819..592841fc91 100755
--- a/src/test/test_workqueue_efd.sh
+++ b/src/test/test_workqueue_efd.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_efd2.sh b/src/test/test_workqueue_efd2.sh
index 7cfff45ff3..4cf1b76cbe 100755
--- a/src/test/test_workqueue_efd2.sh
+++ b/src/test/test_workqueue_efd2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd --no-pipe2 --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_pipe.sh b/src/test/test_workqueue_pipe.sh
index afcef87853..fc3ef34c6c 100755
--- a/src/test/test_workqueue_pipe.sh
+++ b/src/test/test_workqueue_pipe.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-socketpair
diff --git a/src/test/test_workqueue_pipe2.sh b/src/test/test_workqueue_pipe2.sh
index a20a1427e0..7f19ea880d 100755
--- a/src/test/test_workqueue_pipe2.sh
+++ b/src/test/test_workqueue_pipe2.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe --no-socketpair
diff --git a/src/test/test_workqueue_socketpair.sh b/src/test/test_workqueue_socketpair.sh
index 76af79746d..1ee1776447 100755
--- a/src/test/test_workqueue_socketpair.sh
+++ b/src/test/test_workqueue_socketpair.sh
@@ -1,4 +1,4 @@
 #!/bin/sh
 
-${builddir:-.}/src/test/test_workqueue \
+"${builddir:-.}/src/test/test_workqueue" \
 	   --no-eventfd2 --no-eventfd --no-pipe2 --no-pipe
diff --git a/src/test/test_x509.c b/src/test/test_x509.c
index 792849ae4b..94e7db33de 100644
--- a/src/test/test_x509.c
+++ b/src/test/test_x509.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2010-2019, The Tor Project, Inc. */
+/* Copyright (c) 2010-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #define TOR_X509_PRIVATE
diff --git a/src/test/test_zero_length_keys.sh b/src/test/test_zero_length_keys.sh
index 84ca513b0a..b944d9bf3f 100755
--- a/src/test/test_zero_length_keys.sh
+++ b/src/test/test_zero_length_keys.sh
@@ -1,10 +1,44 @@
 #!/bin/sh
 # Check that tor regenerates keys when key files are zero-length
 
+umask 077
+set -e
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "$TOR_BINARY" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
+
 exitcode=0
 
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -z || exitcode=1
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -d || exitcode=1
-"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "${builddir:-.}/src/app/tor" -e || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "$TOR_BINARY" -z || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "$TOR_BINARY" -d || exitcode=1
+"${SHELL:-sh}" "${abs_top_srcdir:-.}/src/test/zero_length_keys.sh" "$TOR_BINARY" -e || exitcode=1
 
 exit ${exitcode}
diff --git a/src/test/testing_common.c b/src/test/testing_common.c
index daa7aa524a..d68dfa4047 100644
--- a/src/test/testing_common.c
+++ b/src/test/testing_common.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
@@ -12,6 +12,7 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 #include "feature/control/control.h"
+#include "feature/control/control_events.h"
 #include "app/config/config.h"
 #include "lib/crypt_ops/crypto_dh.h"
 #include "lib/crypt_ops/crypto_ed25519.h"
@@ -25,6 +26,8 @@
 #include "lib/compress/compress.h"
 #include "lib/evloop/compat_libevent.h"
 #include "lib/crypt_ops/crypto_init.h"
+#include "lib/version/torversion.h"
+#include "app/main/subsysmgr.h"
 
 #include <stdio.h>
 #ifdef HAVE_FCNTL_H
@@ -86,7 +89,18 @@ setup_directory(void)
                  (int)getpid(), rnd32);
     r = mkdir(temp_dir);
   }
-#else /* !(defined(_WIN32)) */
+#elif defined(__ANDROID__)
+  /* tor might not like the default perms, so create a subdir */
+  tor_snprintf(temp_dir, sizeof(temp_dir),
+               "/data/local/tmp/tor_%d_%d_%s",
+               (int) getuid(), (int) getpid(), rnd32);
+  r = mkdir(temp_dir, 0700);
+  if (r) {
+    fprintf(stderr, "Can't create directory %s:", temp_dir);
+    perror("");
+    exit(1);
+  }
+#else /* !defined(_WIN32) */
   tor_snprintf(temp_dir, sizeof(temp_dir), "/tmp/tor_test_%d_%s",
                (int) getpid(), rnd32);
   r = mkdir(temp_dir, 0700);
@@ -94,7 +108,7 @@ setup_directory(void)
     /* undo sticky bit so tests don't get confused. */
     r = chown(temp_dir, getuid(), getgid());
   }
-#endif /* defined(_WIN32) */
+#endif /* defined(_WIN32) || ... */
   if (r) {
     fprintf(stderr, "Can't create directory %s:", temp_dir);
     perror("");
@@ -230,17 +244,17 @@ void
 tinytest_prefork(void)
 {
   free_pregenerated_keys();
-  crypto_prefork();
+  subsystems_prefork();
 }
 void
 tinytest_postfork(void)
 {
-  crypto_postfork();
+  subsystems_postfork();
   init_pregenerated_keys();
 }
 
 static void
-log_callback_failure(int severity, uint32_t domain, const char *msg)
+log_callback_failure(int severity, log_domain_mask_t domain, const char *msg)
 {
   (void)msg;
   if (severity == LOG_ERR || (domain & LD_BUG)) {
@@ -259,24 +273,18 @@ main(int c, const char **v)
   int loglevel = LOG_ERR;
   int accel_crypto = 0;
 
-  /* We must initialise logs before we call tor_assert() */
-  init_logging(1);
+  subsystems_init();
 
-  update_approx_time(time(NULL));
   options = options_new();
-  tor_threads_init();
-  tor_compress_init();
-
-  network_init();
-
-  monotime_init();
 
-  struct tor_libevent_cfg cfg;
+  struct tor_libevent_cfg_t cfg;
   memset(&cfg, 0, sizeof(cfg));
   tor_libevent_initialize(&cfg);
 
   control_initialize_event_queue();
-  configure_backtrace_handler(get_version());
+
+  /* Don't add default logs; the tests manage their own. */
+  quiet_level = QUIET_SILENT;
 
   unsigned num=1, den=1;
 
@@ -316,7 +324,7 @@ main(int c, const char **v)
     memset(&s, 0, sizeof(s));
     set_log_severity_config(loglevel, LOG_ERR, &s);
     /* ALWAYS log bug warnings. */
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_stream_log(&s, "", fileno(stdout));
   }
   {
@@ -324,9 +332,10 @@ main(int c, const char **v)
     log_severity_list_t s;
     memset(&s, 0, sizeof(s));
     set_log_severity_config(LOG_ERR, LOG_ERR, &s);
-    s.masks[LOG_WARN-LOG_ERR] |= LD_BUG;
+    s.masks[SEVERITY_MASK_IDX(LOG_WARN)] |= LD_BUG;
     add_callback_log(&s, log_callback_failure);
   }
+  flush_log_messages_from_startup();
   init_protocol_warning_severity_level();
 
   options->command = CMD_RUN_UNITTESTS;
@@ -343,6 +352,7 @@ main(int c, const char **v)
   initialize_mainloop_events();
   options_init(options);
   options->DataDirectory = tor_strdup(temp_dir);
+  options->DataDirectory_option = tor_strdup(temp_dir);
   tor_asprintf(&options->KeyDirectory, "%s"PATH_SEPARATOR"keys",
                options->DataDirectory);
   options->CacheDirectory = tor_strdup(temp_dir);
@@ -409,8 +419,6 @@ main(int c, const char **v)
 
   free_pregenerated_keys();
 
-  crypto_global_cleanup();
-
   if (have_failed)
     return 1;
   else
diff --git a/src/test/testing_rsakeys.c b/src/test/testing_rsakeys.c
index 0f22d4e01b..e058f72d01 100644
--- a/src/test/testing_rsakeys.c
+++ b/src/test/testing_rsakeys.c
@@ -1,6 +1,6 @@
 /* Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "lib/crypt_ops/crypto_rand.h"
@@ -448,7 +448,8 @@ static int next_key_idx_2048;
 static crypto_pk_t *
 pk_generate_internal(int bits)
 {
-  tor_assert(bits == 2048 || bits == 1024);
+  tor_assertf(bits == 2048 || bits == 1024,
+             "Wrong key size: %d", bits);
 
 #ifdef USE_PREGENERATED_RSA_KEYS
   int *idxp;
@@ -467,7 +468,7 @@ pk_generate_internal(int bits)
   *idxp += crypto_rand_int_range(1,3);
   *idxp %= n_pregen;
   return crypto_pk_dup_key(pregen_array[*idxp]);
-#else /* !(defined(USE_PREGENERATED_RSA_KEYS)) */
+#else /* !defined(USE_PREGENERATED_RSA_KEYS) */
   crypto_pk_t *result;
   int res;
   result = crypto_pk_new();
diff --git a/src/test/zero_length_keys.sh b/src/test/zero_length_keys.sh
index 5635bdfd89..1702d11245 100755
--- a/src/test/zero_length_keys.sh
+++ b/src/test/zero_length_keys.sh
@@ -19,7 +19,7 @@
 #   3: a command failed - the test could not be completed
 #
 
-if [ $# -eq 0 ] || [ ! -f ${1} ] || [ ! -x ${1} ]; then
+if [ $# -eq 0 ] || [ ! -f "${1}" ] || [ ! -x "${1}" ]; then
   echo "Usage: ${0} PATH_TO_TOR [-z|-d|-e]"
   exit 1
 elif [ $# -eq 1 ]; then
@@ -31,7 +31,7 @@ else #[$# -gt 1 ]; then
   shift
 fi
 
-DATA_DIR=`mktemp -d -t tor_zero_length_keys.XXXXXX`
+DATA_DIR=$(mktemp -d -t tor_zero_length_keys.XXXXXX)
 if [ -z "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation returned empty string" >&2
   exit 3
@@ -40,7 +40,7 @@ if [ ! -d "$DATA_DIR" ]; then
   echo "Failure: mktemp invocation result doesn't point to directory" >&2
   exit 3
 fi
-trap "rm -rf '$DATA_DIR'" 0
+trap 'rm -rf "$DATA_DIR"' 0
 
 touch "$DATA_DIR"/empty_torrc
 touch "$DATA_DIR"/empty_defaults_torrc
diff --git a/src/tools/.may_include b/src/tools/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/tools/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/tools/include.am b/src/tools/include.am
index f7aa7e0d1e..72dfe6017c 100644
--- a/src/tools/include.am
+++ b/src/tools/include.am
@@ -5,9 +5,11 @@ noinst_PROGRAMS+= src/tools/tor-cov-resolve
 endif
 
 src_tools_tor_resolve_SOURCES = src/tools/tor-resolve.c
-src_tools_tor_resolve_LDFLAGS =
+src_tools_tor_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_LIBS) \
+        $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@
 
@@ -15,8 +17,11 @@ if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
 src_tools_tor_cov_resolve_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
 src_tools_tor_cov_resolve_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+src_tools_tor_cov_resolve_LDFLAGS = @TOR_LDFLAGS_openssl@
 src_tools_tor_cov_resolve_LDADD = \
+	src/trunnel/libor-trunnel.a \
 	$(TOR_UTIL_TESTING_LIBS) \
+	$(TOR_CRYPTO_TESTING_LIBS) $(TOR_LIBS_CRYPTLIB) \
 	@TOR_LIB_MATH@ @TOR_LIB_WS32@
 endif
 
diff --git a/src/tools/tools.md b/src/tools/tools.md
new file mode 100644
index 0000000000..1f00381679
--- /dev/null
+++ b/src/tools/tools.md
@@ -0,0 +1,6 @@
+@dir /tools
+@brief tools: other command-line tools for use with Tor.
+
+The "tools" directory has a few other programs that use Tor, but are not part
+of the main Tor binary.
+
diff --git a/src/tools/tor-gencert.c b/src/tools/tor-gencert.c
index 25113420df..e4f6530b46 100644
--- a/src/tools/tor-gencert.c
+++ b/src/tools/tor-gencert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include "orconfig.h"
@@ -21,7 +21,7 @@
 /* Some versions of OpenSSL declare X509_STORE_CTX_set_verify_cb twice in
  * x509.h and x509_vfy.h. Suppress the GCC warning so we can build with
  * -Wredundant-decl. */
-DISABLE_GCC_WARNING(redundant-decls)
+DISABLE_GCC_WARNING("-Wredundant-decls")
 
 #include <openssl/evp.h>
 #include <openssl/pem.h>
@@ -30,8 +30,8 @@ DISABLE_GCC_WARNING(redundant-decls)
 #include <openssl/obj_mac.h>
 #include <openssl/err.h>
 
-ENABLE_GCC_WARNING(redundant-decls)
-#endif
+ENABLE_GCC_WARNING("-Wredundant-decls")
+#endif /* defined(ENABLE_OPENSSL) */
 
 #include <errno.h>
 
diff --git a/src/tools/tor-print-ed-signing-cert.c b/src/tools/tor-print-ed-signing-cert.c
index 1f1a01ab5c..7836293df4 100644
--- a/src/tools/tor-print-ed-signing-cert.c
+++ b/src/tools/tor-print-ed-signing-cert.c
@@ -1,4 +1,4 @@
-/* Copyright (c) 2007-2019, The Tor Project, Inc. */
+/* Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 #include <errno.h>
@@ -10,11 +10,13 @@
 #include "lib/cc/torint.h"  /* TOR_PRIdSZ */
 #include "lib/crypt_ops/crypto_format.h"
 #include "lib/malloc/malloc.h"
+#include "lib/encoding/time_fmt.h"
 
 int
 main(int argc, char **argv)
 {
   ed25519_cert_t *cert = NULL;
+  char rfc1123_buf[RFC1123_TIME_LEN+1] = "";
 
   if (argc != 2) {
     fprintf(stderr, "Usage:\n");
@@ -59,6 +61,11 @@ main(int argc, char **argv)
 
   printf("Expires at: %s", ctime(&expires_at));
 
+  format_rfc1123_time(rfc1123_buf, expires_at);
+  printf("RFC 1123 timestamp: %s\n", rfc1123_buf);
+
+  printf("UNIX timestamp: %ld\n", (long int)expires_at);
+
   ed25519_cert_free(cert);
 
   return 0;
diff --git a/src/tools/tor-resolve.c b/src/tools/tor-resolve.c
index 6a84abe557..e6d6bddcdb 100644
--- a/src/tools/tor-resolve.c
+++ b/src/tools/tor-resolve.c
@@ -1,5 +1,5 @@
 /* Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson
- * Copyright (c) 2007-2019, The Tor Project, Inc.
+ * Copyright (c) 2007-2020, The Tor Project, Inc.
  */
 /* See LICENSE for licensing information */
 
@@ -15,6 +15,7 @@
 #include "lib/string/util_string.h"
 
 #include "lib/net/socks5_status.h"
+#include "trunnel/socks5.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -44,73 +45,211 @@
 
 #define RESPONSE_LEN_4 8
 #define log_sock_error(act, _s)                                         \
-  STMT_BEGIN log_fn(LOG_ERR, LD_NET, "Error while %s: %s", act,         \
-              tor_socket_strerror(tor_socket_errno(_s))); STMT_END
+  STMT_BEGIN                                                            \
+    log_fn(LOG_ERR, LD_NET, "Error while %s: %s", act,                  \
+    tor_socket_strerror(tor_socket_errno(_s)));                         \
+  STMT_END
 
 static void usage(void) ATTR_NORETURN;
 
+/**
+ * Set <b>out</b> to a pointer to newly allocated buffer containing
+ * SOCKS4a RESOLVE request with <b>username</b> and <b>hostname</b>.
+ * Return number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks4a_resolve_request(uint8_t **out,
+                              const char *username,
+                              const char *hostname)
+{
+  tor_assert(out);
+  tor_assert(username);
+  tor_assert(hostname);
+
+  const char *errmsg = NULL;
+  uint8_t *output = NULL;
+  socks4_client_request_t *rq = socks4_client_request_new();
+
+  socks4_client_request_set_version(rq, 4);
+  socks4_client_request_set_command(rq, CMD_RESOLVE);
+  socks4_client_request_set_port(rq, 0);
+  socks4_client_request_set_addr(rq, 0x00000001u);
+  socks4_client_request_set_username(rq, username);
+  socks4_client_request_set_socks4a_addr_hostname(rq, hostname);
+
+  errmsg = socks4_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks4_client_request_encoded_len(rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encoded_len failed";
+    goto cleanup;
+  }
+
+  output = tor_malloc(encoded_len);
+  memset(output, 0, encoded_len);
+
+  encoded_len = socks4_client_request_encode(output, encoded_len, rq);
+  if (encoded_len <= 0) {
+    errmsg = "socks4_client_request_encode failed";
+    goto cleanup;
+  }
+
+  *out = output;
+
+ cleanup:
+  socks4_client_request_free(rq);
+  if (errmsg) {
+    log_err(LD_NET, "build_socks4a_resolve_request failed: %s", errmsg);
+    *out = NULL;
+    tor_free(output);
+  }
+  return errmsg ? -1 : encoded_len;
+}
+
+#define SOCKS5_ATYPE_HOSTNAME 0x03
+#define SOCKS5_ATYPE_IPV4     0x01
+#define SOCKS5_ATYPE_IPV6     0x04
+
+/**
+ * Set <b>out</b> to pointer to newly allocated buffer containing
+ * SOCKS5 RESOLVE/RESOLVE_PTR request with given <b>hostname<b>.
+ * Generate a reverse request if <b>reverse</b> is true.
+ * Return the number of bytes in the buffer if succeeded or -1 if failed.
+ */
+static ssize_t
+build_socks5_resolve_request(uint8_t **out,
+                             const char *hostname,
+                             int reverse)
+{
+  const char *errmsg = NULL;
+  uint8_t *outbuf = NULL;
+  int is_ip_address;
+  tor_addr_t addr;
+  size_t addrlen;
+  int ipv6;
+  is_ip_address = tor_addr_parse(&addr, hostname) != -1;
+  if (!is_ip_address && reverse) {
+    log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
+    return -1;
+  }
+  ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
+  addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
+  if (addrlen > UINT8_MAX) {
+    log_err(LD_GENERAL, "Hostname is too long!");
+    return -1;
+  }
+
+  socks5_client_request_t *rq = socks5_client_request_new();
+
+  socks5_client_request_set_version(rq, 5);
+  /* RESOLVE_PTR or RESOLVE */
+  socks5_client_request_set_command(rq, reverse ? CMD_RESOLVE_PTR :
+                                                  CMD_RESOLVE);
+  socks5_client_request_set_reserved(rq, 0);
+
+  uint8_t atype = SOCKS5_ATYPE_HOSTNAME;
+  if (reverse)
+    atype = ipv6 ? SOCKS5_ATYPE_IPV6 : SOCKS5_ATYPE_IPV4;
+
+  socks5_client_request_set_atype(rq, atype);
+
+  switch (atype) {
+    case SOCKS5_ATYPE_IPV4: {
+      socks5_client_request_set_dest_addr_ipv4(rq,
+                                        tor_addr_to_ipv4h(&addr));
+    } break;
+    case SOCKS5_ATYPE_IPV6: {
+      uint8_t *ipv6_array =
+        socks5_client_request_getarray_dest_addr_ipv6(rq);
+
+      tor_assert(ipv6_array);
+
+      memcpy(ipv6_array, tor_addr_to_in6_addr8(&addr), 16);
+    } break;
+
+    case SOCKS5_ATYPE_HOSTNAME: {
+      domainname_t *dn = domainname_new();
+      domainname_set_len(dn, addrlen - 1);
+      domainname_setlen_name(dn, addrlen - 1);
+      char *dn_buf = domainname_getarray_name(dn);
+      memcpy(dn_buf, hostname, addrlen - 1);
+
+      errmsg = domainname_check(dn);
+
+      if (errmsg) {
+        domainname_free(dn);
+        goto cleanup;
+      } else {
+        socks5_client_request_set_dest_addr_domainname(rq, dn);
+      }
+    } break;
+    default:
+      tor_assert_unreached();
+      break;
+  }
+
+  socks5_client_request_set_dest_port(rq, 0);
+
+  errmsg = socks5_client_request_check(rq);
+  if (errmsg) {
+    goto cleanup;
+  }
+
+  ssize_t encoded_len = socks5_client_request_encoded_len(rq);
+  if (encoded_len < 0) {
+    errmsg = "Cannot predict encoded length";
+    goto cleanup;
+  }
+
+  outbuf = tor_malloc(encoded_len);
+  memset(outbuf, 0, encoded_len);
+
+  encoded_len = socks5_client_request_encode(outbuf, encoded_len, rq);
+  if (encoded_len < 0) {
+    errmsg = "encoding failed";
+    goto cleanup;
+  }
+
+  *out = outbuf;
+
+ cleanup:
+  socks5_client_request_free(rq);
+  if (errmsg) {
+    tor_free(outbuf);
+    log_err(LD_NET, "build_socks5_resolve_request failed with error: %s",
+            errmsg);
+  }
+
+  return errmsg ? -1 : encoded_len;
+}
+
 /** Set *<b>out</b> to a newly allocated SOCKS4a resolve request with
  * <b>username</b> and <b>hostname</b> as provided.  Return the number
  * of bytes in the request. */
 static ssize_t
-build_socks_resolve_request(char **out,
+build_socks_resolve_request(uint8_t **out,
                             const char *username,
                             const char *hostname,
                             int reverse,
                             int version)
 {
-  size_t len = 0;
   tor_assert(out);
   tor_assert(username);
   tor_assert(hostname);
 
+  tor_assert(version == 4 || version == 5);
+
   if (version == 4) {
-    len = 8 + strlen(username) + 1 + strlen(hostname) + 1;
-    *out = tor_malloc(len);
-    (*out)[0] = 4;      /* SOCKS version 4 */
-    (*out)[1] = '\xF0'; /* Command: resolve. */
-    set_uint16((*out)+2, htons(0)); /* port: 0. */
-    set_uint32((*out)+4, htonl(0x00000001u)); /* addr: 0.0.0.1 */
-    memcpy((*out)+8, username, strlen(username)+1);
-    memcpy((*out)+8+strlen(username)+1, hostname, strlen(hostname)+1);
+    return build_socks4a_resolve_request(out, username, hostname);
   } else if (version == 5) {
-    int is_ip_address;
-    tor_addr_t addr;
-    size_t addrlen;
-    int ipv6;
-    is_ip_address = tor_addr_parse(&addr, hostname) != -1;
-    if (!is_ip_address && reverse) {
-      log_err(LD_GENERAL, "Tried to do a reverse lookup on a non-IP!");
-      return -1;
-    }
-    ipv6 = reverse && tor_addr_family(&addr) == AF_INET6;
-    addrlen = reverse ? (ipv6 ? 16 : 4) : 1 + strlen(hostname);
-    if (addrlen > UINT8_MAX) {
-      log_err(LD_GENERAL, "Hostname is too long!");
-      return -1;
-    }
-    len = 6 + addrlen;
-    *out = tor_malloc(len);
-    (*out)[0] = 5; /* SOCKS version 5 */
-    (*out)[1] = reverse ? '\xF1' : '\xF0'; /* RESOLVE_PTR or RESOLVE */
-    (*out)[2] = 0; /* reserved. */
-    if (reverse) {
-      (*out)[3] = ipv6 ? 4 : 1;
-      if (ipv6)
-        memcpy((*out)+4, tor_addr_to_in6_addr8(&addr), 16);
-      else
-        set_uint32((*out)+4, tor_addr_to_ipv4n(&addr));
-    } else {
-      (*out)[3] = 3;
-      (*out)[4] = (char)(uint8_t)(addrlen - 1);
-      memcpy((*out)+5, hostname, addrlen - 1);
-    }
-    set_uint16((*out)+4+addrlen, 0); /* port */
-  } else {
-    tor_assert(0);
+    return build_socks5_resolve_request(out, hostname, reverse);
   }
 
-  return len;
+  tor_assert_unreached();
+  return -1;
 }
 
 static void
@@ -134,34 +273,50 @@ parse_socks4a_resolve_response(const char *hostname,
                                const char *response, size_t len,
                                tor_addr_t *addr_out)
 {
+  int result = 0;
   uint8_t status;
   tor_assert(response);
   tor_assert(addr_out);
 
-  if (len < RESPONSE_LEN_4) {
+  socks4_server_reply_t *reply;
+
+  ssize_t parsed = socks4_server_reply_parse(&reply,
+                                             (const uint8_t *)response,
+                                             len);
+
+  if (parsed == -1) {
+    log_warn(LD_PROTOCOL, "Failed parsing SOCKS4a response");
+    result = -1; goto cleanup;
+  }
+
+  if (parsed == -2) {
     log_warn(LD_PROTOCOL,"Truncated socks response.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  if (((uint8_t)response[0])!=0) { /* version: 0 */
+
+  if (socks4_server_reply_get_version(reply) != 0) { /* version: 0 */
     log_warn(LD_PROTOCOL,"Nonzero version in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
-  status = (uint8_t)response[1];
-  if (get_uint16(response+2)!=0) { /* port: 0 */
+  if (socks4_server_reply_get_port(reply) != 0) { /* port: 0 */
     log_warn(LD_PROTOCOL,"Nonzero port in socks response: bad format.");
-    return -1;
+    result = -1; goto cleanup;
   }
+  status = socks4_server_reply_get_status(reply);
   if (status != 90) {
     log_warn(LD_NET,"Got status response '%d': socks request failed.", status);
     if (!strcasecmpend(hostname, ".onion")) {
       onion_warning(hostname);
-      return -1;
+      result = -1; goto cleanup;
     }
-    return -1;
+    result = -1; goto cleanup;
   }
 
-  tor_addr_from_ipv4n(addr_out, get_uint32(response+4));
-  return 0;
+  tor_addr_from_ipv4h(addr_out, socks4_server_reply_get_addr(reply));
+
+ cleanup:
+  socks4_server_reply_free(reply);
+  return result;
 }
 
 /* It would be nice to let someone know what SOCKS5 issue a user may have */
@@ -205,7 +360,7 @@ do_resolve(const char *hostname,
   int s = -1;
   struct sockaddr_storage ss;
   socklen_t socklen;
-  char *req = NULL;
+  uint8_t *req = NULL;
   ssize_t len = 0;
 
   tor_assert(hostname);
@@ -230,23 +385,58 @@ do_resolve(const char *hostname,
   }
 
   if (version == 5) {
-    char method_buf[2];
-    if (write_all_to_socket(s, "\x05\x01\x00", 3) != 3) {
+    socks5_client_version_t *v = socks5_client_version_new();
+
+    socks5_client_version_set_version(v, 5);
+    socks5_client_version_set_n_methods(v, 1);
+    socks5_client_version_setlen_methods(v, 1);
+    socks5_client_version_set_methods(v, 0, 0x00);
+
+    tor_assert(!socks5_client_version_check(v));
+    ssize_t encoded_len = socks5_client_version_encoded_len(v);
+    tor_assert(encoded_len > 0);
+
+    uint8_t *buf = tor_malloc(encoded_len);
+    encoded_len = socks5_client_version_encode(buf, encoded_len, v);
+    tor_assert(encoded_len > 0);
+
+    socks5_client_version_free(v);
+
+    if (write_all_to_socket(s, (const char *)buf,
+                            encoded_len) != encoded_len) {
       log_err(LD_NET, "Error sending SOCKS5 method list.");
+      tor_free(buf);
+
       goto err;
     }
-    if (read_all_from_socket(s, method_buf, 2) != 2) {
+
+    tor_free(buf);
+
+    uint8_t method_buf[2];
+
+    if (read_all_from_socket(s, (char *)method_buf, 2) != 2) {
       log_err(LD_NET, "Error reading SOCKS5 methods.");
       goto err;
     }
-    if (method_buf[0] != '\x05') {
-      log_err(LD_NET, "Unrecognized socks version: %u",
-              (unsigned)method_buf[0]);
+
+    socks5_server_method_t *m;
+    ssize_t parsed = socks5_server_method_parse(&m, method_buf,
+                                                sizeof(method_buf));
+
+    if (parsed < 2) {
+      log_err(LD_NET, "Failed to parse SOCKS5 method selection "
+                      "message");
+      socks5_server_method_free(m);
       goto err;
     }
-    if (method_buf[1] != '\x00') {
+
+    uint8_t method = socks5_server_method_get_method(m);
+
+    socks5_server_method_free(m);
+
+    if (method != 0x00) {
       log_err(LD_NET, "Unrecognized socks authentication method: %u",
-              (unsigned)method_buf[1]);
+              (unsigned)method);
       goto err;
     }
   }
@@ -257,7 +447,7 @@ do_resolve(const char *hostname,
     tor_assert(!req);
     goto err;
   }
-  if (write_all_to_socket(s, req, len) != len) {
+  if (write_all_to_socket(s, (const char *)req, len) != len) {
     log_sock_error("sending SOCKS request", s);
     tor_free(req);
     goto err;
@@ -276,55 +466,59 @@ do_resolve(const char *hostname,
       goto err;
     }
   } else {
-    char reply_buf[16];
-    if (read_all_from_socket(s, reply_buf, 4) != 4) {
-      log_err(LD_NET, "Error reading SOCKS5 response.");
+    uint8_t reply_buf[512];
+
+    len = read_all_from_socket(s, (char *)reply_buf,
+                               sizeof(reply_buf));
+
+    socks5_server_reply_t *reply;
+
+    ssize_t parsed = socks5_server_reply_parse(&reply,
+                                               reply_buf,
+                                               len);
+    if (parsed == -1) {
+      log_err(LD_NET, "Failed parsing SOCKS5 response");
       goto err;
     }
-    if (reply_buf[0] != 5) {
-      log_err(LD_NET, "Bad SOCKS5 reply version.");
+
+    if (parsed == -2) {
+      log_err(LD_NET, "Truncated SOCKS5 response");
       goto err;
     }
+
     /* Give a user some useful feedback about SOCKS5 errors */
-    if (reply_buf[1] != 0) {
+    uint8_t reply_field = socks5_server_reply_get_reply(reply);
+    if (reply_field != 0) {
       log_warn(LD_NET,"Got SOCKS5 status response '%u': %s",
-               (unsigned)reply_buf[1],
-               socks5_reason_to_string(reply_buf[1]));
-      if (reply_buf[1] == 4 && !strcasecmpend(hostname, ".onion")) {
+               (unsigned)reply_field,
+               socks5_reason_to_string(reply_field));
+      if (reply_field == 4 && !strcasecmpend(hostname, ".onion")) {
         onion_warning(hostname);
       }
+
+      socks5_server_reply_free(reply);
       goto err;
     }
-    if (reply_buf[3] == 1) {
+
+    uint8_t atype = socks5_server_reply_get_atype(reply);
+
+    if (atype == SOCKS5_ATYPE_IPV4) {
       /* IPv4 address */
-      if (read_all_from_socket(s, reply_buf, 4) != 4) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv4n(result_addr, get_uint32(reply_buf));
-    } else if (reply_buf[3] == 4) {
+      tor_addr_from_ipv4h(result_addr,
+        socks5_server_reply_get_bind_addr_ipv4(reply));
+    } else if (atype == SOCKS5_ATYPE_IPV6) {
       /* IPv6 address */
-      if (read_all_from_socket(s, reply_buf, 16) != 16) {
-        log_err(LD_NET, "Error reading address in socks5 response.");
-        goto err;
-      }
-      tor_addr_from_ipv6_bytes(result_addr, reply_buf);
-    } else if (reply_buf[3] == 3) {
+      tor_addr_from_ipv6_bytes(result_addr,
+        socks5_server_reply_getarray_bind_addr_ipv6(reply));
+    } else if (atype == SOCKS5_ATYPE_HOSTNAME) {
       /* Domain name */
-      size_t result_len;
-      if (read_all_from_socket(s, reply_buf, 1) != 1) {
-        log_err(LD_NET, "Error reading address_length in socks5 response.");
-        goto err;
-      }
-      result_len = *(uint8_t*)(reply_buf);
-      *result_hostname = tor_malloc(result_len+1);
-      if (read_all_from_socket(s, *result_hostname, result_len)
-          != (int) result_len) {
-        log_err(LD_NET, "Error reading hostname in socks5 response.");
-        goto err;
-      }
-      (*result_hostname)[result_len] = '\0';
+      domainname_t *dn =
+        socks5_server_reply_get_bind_addr_domainname(reply);
+
+      *result_hostname = tor_strdup(domainname_getstr_name(dn));
     }
+
+    socks5_server_reply_free(reply);
   }
 
   tor_close_socket(s);
diff --git a/src/tools/tor_runner.c b/src/tools/tor_runner.c
index 3c6ade91d9..83f1a495cd 100644
--- a/src/tools/tor_runner.c
+++ b/src/tools/tor_runner.c
@@ -1,7 +1,7 @@
 /* Copyright (c) 2001 Matej Pfajfar.
  * Copyright (c) 2001-2004, Roger Dingledine.
  * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson.
- * Copyright (c) 2007-2019, The Tor Project, Inc. */
+ * Copyright (c) 2007-2020, The Tor Project, Inc. */
 /* See LICENSE for licensing information */
 
 /**
diff --git a/src/trunnel/.may_include b/src/trunnel/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/trunnel/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/trunnel/channelpadding_negotiation.c b/src/trunnel/channelpadding_negotiation.c
index 59e6b38384..d96496e90c 100644
--- a/src/trunnel/channelpadding_negotiation.c
+++ b/src/trunnel/channelpadding_negotiation.c
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.c -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/channelpadding_negotiation.h b/src/trunnel/channelpadding_negotiation.h
index fcfc232fea..3f96174f68 100644
--- a/src/trunnel/channelpadding_negotiation.h
+++ b/src/trunnel/channelpadding_negotiation.h
@@ -1,4 +1,4 @@
-/* channelpadding_negotiation.h -- generated by Trunnel v1.5.2.
+/* channelpadding_negotiation.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/circpad_negotiation.c b/src/trunnel/circpad_negotiation.c
new file mode 100644
index 0000000000..547818f2ec
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.c
@@ -0,0 +1,549 @@
+/* circpad_negotiation.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "circpad_negotiation.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int circpadnegotiation_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || circpadnegotiation_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+circpad_negotiate_t *
+circpad_negotiate_new(void)
+{
+  circpad_negotiate_t *val = trunnel_calloc(1, sizeof(circpad_negotiate_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiate_clear(circpad_negotiate_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiate_free(circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiate_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiate_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiate_get_version(const circpad_negotiate_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_command(const circpad_negotiate_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+uint8_t
+circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp)
+{
+  return inp->echo_request;
+}
+int
+circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->echo_request = val;
+  return 0;
+}
+const char *
+circpad_negotiate_check(const circpad_negotiate_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiate_encoded_len(const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiate_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+
+  /* Length of u8 echo_request IN [0, 1] */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiate_clear_errors(circpad_negotiate_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiate_encode(uint8_t *output, const size_t avail, const circpad_negotiate_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiate_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiate_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 echo_request IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->echo_request));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiate_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiate_parse_into(circpad_negotiate_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 echo_request IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->echo_request = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->echo_request == 0 || obj->echo_request == 1))
+    goto fail;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiate_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiate_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiate_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+circpad_negotiated_t *
+circpad_negotiated_new(void)
+{
+  circpad_negotiated_t *val = trunnel_calloc(1, sizeof(circpad_negotiated_t));
+  if (NULL == val)
+    return NULL;
+  val->command = CIRCPAD_COMMAND_START;
+  val->response = CIRCPAD_RESPONSE_ERR;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+circpad_negotiated_clear(circpad_negotiated_t *obj)
+{
+  (void) obj;
+}
+
+void
+circpad_negotiated_free(circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return;
+  circpad_negotiated_clear(obj);
+  trunnel_memwipe(obj, sizeof(circpad_negotiated_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+circpad_negotiated_get_version(const circpad_negotiated_t *inp)
+{
+  return inp->version;
+}
+int
+circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == 0))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_command(const circpad_negotiated_t *inp)
+{
+  return inp->command;
+}
+int
+circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_COMMAND_START || val == CIRCPAD_COMMAND_STOP))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->command = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_response(const circpad_negotiated_t *inp)
+{
+  return inp->response;
+}
+int
+circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val)
+{
+  if (! ((val == CIRCPAD_RESPONSE_ERR || val == CIRCPAD_RESPONSE_OK))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->response = val;
+  return 0;
+}
+uint8_t
+circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp)
+{
+  return inp->machine_type;
+}
+int
+circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val)
+{
+  inp->machine_type = val;
+  return 0;
+}
+const char *
+circpad_negotiated_check(const circpad_negotiated_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0))
+    return "Integer out of bounds";
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    return "Integer out of bounds";
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    return "Integer out of bounds";
+  return NULL;
+}
+
+ssize_t
+circpad_negotiated_encoded_len(const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != circpad_negotiated_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0] */
+  result += 1;
+
+  /* Length of u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  result += 1;
+
+  /* Length of u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  result += 1;
+
+  /* Length of u8 machine_type */
+  result += 1;
+  return result;
+}
+int
+circpad_negotiated_clear_errors(circpad_negotiated_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+circpad_negotiated_encode(uint8_t *output, const size_t avail, const circpad_negotiated_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = circpad_negotiated_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = circpad_negotiated_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->command));
+  written += 1; ptr += 1;
+
+  /* Encode u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->response));
+  written += 1; ptr += 1;
+
+  /* Encode u8 machine_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->machine_type));
+  written += 1; ptr += 1;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As circpad_negotiated_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+circpad_negotiated_parse_into(circpad_negotiated_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0))
+    goto fail;
+
+  /* Parse u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP] */
+  CHECK_REMAINING(1, truncated);
+  obj->command = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->command == CIRCPAD_COMMAND_START || obj->command == CIRCPAD_COMMAND_STOP))
+    goto fail;
+
+  /* Parse u8 response IN [CIRCPAD_RESPONSE_ERR, CIRCPAD_RESPONSE_OK] */
+  CHECK_REMAINING(1, truncated);
+  obj->response = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->response == CIRCPAD_RESPONSE_ERR || obj->response == CIRCPAD_RESPONSE_OK))
+    goto fail;
+
+  /* Parse u8 machine_type */
+  CHECK_REMAINING(1, truncated);
+  obj->machine_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = circpad_negotiated_new();
+  if (NULL == *output)
+    return -1;
+  result = circpad_negotiated_parse_into(*output, input, len_in);
+  if (result < 0) {
+    circpad_negotiated_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/circpad_negotiation.h b/src/trunnel/circpad_negotiation.h
new file mode 100644
index 0000000000..ba9155019e
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.h
@@ -0,0 +1,195 @@
+/* circpad_negotiation.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_CIRCPAD_NEGOTIATION_H
+#define TRUNNEL_CIRCPAD_NEGOTIATION_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define CIRCPAD_COMMAND_STOP 1
+#define CIRCPAD_COMMAND_START 2
+#define CIRCPAD_RESPONSE_OK 1
+#define CIRCPAD_RESPONSE_ERR 2
+#define CIRCPAD_MACHINE_CIRC_SETUP 1
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATE)
+struct circpad_negotiate_st {
+  uint8_t version;
+  uint8_t command;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  /** If true, send a relay_drop reply.. */
+  uint8_t echo_request;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiate_st circpad_negotiate_t;
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_CIRCPAD_NEGOTIATED)
+struct circpad_negotiated_st {
+  uint8_t version;
+  uint8_t command;
+  uint8_t response;
+  /** Machine type is left unbounded because we can specify
+     * new machines in the consensus */
+  uint8_t machine_type;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct circpad_negotiated_st circpad_negotiated_t;
+/** Return a newly allocated circpad_negotiate with all elements set
+ * to zero.
+ */
+circpad_negotiate_t *circpad_negotiate_new(void);
+/** Release all storage held by the circpad_negotiate in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void circpad_negotiate_free(circpad_negotiate_t *victim);
+/** Try to parse a circpad_negotiate from the buffer in 'input', using
+ * up to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * circpad_negotiate_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiate_parse(circpad_negotiate_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiate in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiate_encoded_len(const circpad_negotiate_t *obj);
+/** Try to encode the circpad_negotiate from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiate_encode(uint8_t *output, size_t avail, const circpad_negotiate_t *input);
+/** Check whether the internal state of the circpad_negotiate in 'obj'
+ * is consistent. Return NULL if it is, and a short message if it is
+ * not.
+ */
+const char *circpad_negotiate_check(const circpad_negotiate_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiate_clear_errors(circpad_negotiate_t *obj);
+/** Return the value of the version field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_version(const circpad_negotiate_t *inp);
+/** Set the value of the version field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_version(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiate_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiate_get_command(const circpad_negotiate_t *inp);
+/** Set the value of the command field of the circpad_negotiate_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_command(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_machine_type(const circpad_negotiate_t *inp);
+/** Set the value of the machine_type field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_machine_type(circpad_negotiate_t *inp, uint8_t val);
+/** Return the value of the echo_request field of the
+ * circpad_negotiate_t in 'inp'
+ */
+uint8_t circpad_negotiate_get_echo_request(const circpad_negotiate_t *inp);
+/** Set the value of the echo_request field of the circpad_negotiate_t
+ * in 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiate_set_echo_request(circpad_negotiate_t *inp, uint8_t val);
+/** Return a newly allocated circpad_negotiated with all elements set
+ * to zero.
+ */
+circpad_negotiated_t *circpad_negotiated_new(void);
+/** Release all storage held by the circpad_negotiated in 'victim'.
+ * (Do nothing if 'victim' is NULL.)
+ */
+void circpad_negotiated_free(circpad_negotiated_t *victim);
+/** Try to parse a circpad_negotiated from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated circpad_negotiated_t. On failure, return -2 if the input
+ * appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t circpad_negotiated_parse(circpad_negotiated_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * circpad_negotiated in 'obj'. On failure, return a negative value.
+ * Note that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t circpad_negotiated_encoded_len(const circpad_negotiated_t *obj);
+/** Try to encode the circpad_negotiated from 'input' into the buffer
+ * at 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t circpad_negotiated_encode(uint8_t *output, size_t avail, const circpad_negotiated_t *input);
+/** Check whether the internal state of the circpad_negotiated in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *circpad_negotiated_check(const circpad_negotiated_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int circpad_negotiated_clear_errors(circpad_negotiated_t *obj);
+/** Return the value of the version field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_version(const circpad_negotiated_t *inp);
+/** Set the value of the version field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_version(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the command field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_command(const circpad_negotiated_t *inp);
+/** Set the value of the command field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_command(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the response field of the circpad_negotiated_t
+ * in 'inp'
+ */
+uint8_t circpad_negotiated_get_response(const circpad_negotiated_t *inp);
+/** Set the value of the response field of the circpad_negotiated_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int circpad_negotiated_set_response(circpad_negotiated_t *inp, uint8_t val);
+/** Return the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp'
+ */
+uint8_t circpad_negotiated_get_machine_type(const circpad_negotiated_t *inp);
+/** Set the value of the machine_type field of the
+ * circpad_negotiated_t in 'inp' to 'val'. Return 0 on success; return
+ * -1 and set the error code on 'inp' on failure.
+ */
+int circpad_negotiated_set_machine_type(circpad_negotiated_t *inp, uint8_t val);
+
+
+#endif
diff --git a/src/trunnel/circpad_negotiation.trunnel b/src/trunnel/circpad_negotiation.trunnel
new file mode 100644
index 0000000000..abbc929cc5
--- /dev/null
+++ b/src/trunnel/circpad_negotiation.trunnel
@@ -0,0 +1,44 @@
+/* These are the padding negotiation commands */
+const CIRCPAD_COMMAND_STOP = 1;
+const CIRCPAD_COMMAND_START = 2;
+
+/* Responses to commands */
+const CIRCPAD_RESPONSE_OK = 1;
+const CIRCPAD_RESPONSE_ERR = 2;
+
+/* Built-in machine types */
+
+/* 1) Machine that obscures circuit setup */
+const CIRCPAD_MACHINE_CIRC_SETUP = 1;
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiate {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+
+  /** If true, send a relay_drop reply.. */
+  // FIXME-MP-AP: Maybe we just say to transition to the first state
+  // here instead.. Also what about delay before responding?
+  u8 echo_request IN [0,1];
+};
+
+/**
+ * This command tells the relay to alter its min and max netflow
+ * timeout range values, and send padding at that rate (resuming
+ * if stopped). */
+struct circpad_negotiated {
+  u8 version IN [0];
+  u8 command IN [CIRCPAD_COMMAND_START, CIRCPAD_COMMAND_STOP];
+  u8 response IN [CIRCPAD_RESPONSE_OK, CIRCPAD_RESPONSE_ERR];
+
+  /** Machine type is left unbounded because we can specify
+   * new machines in the consensus */
+  u8 machine_type;
+};
diff --git a/src/trunnel/ed25519_cert.c b/src/trunnel/ed25519_cert.c
index 1276c7a505..86b79ef9b6 100644
--- a/src/trunnel/ed25519_cert.c
+++ b/src/trunnel/ed25519_cert.c
@@ -1,4 +1,4 @@
-/* ed25519_cert.c -- generated by Trunnel v1.5.2.
+/* ed25519_cert.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.h b/src/trunnel/ed25519_cert.h
index e086c6fced..bd91ce1055 100644
--- a/src/trunnel/ed25519_cert.h
+++ b/src/trunnel/ed25519_cert.h
@@ -1,4 +1,4 @@
-/* ed25519_cert.h -- generated by Trunnel v1.5.2.
+/* ed25519_cert.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/ed25519_cert.trunnel b/src/trunnel/ed25519_cert.trunnel
index 8d6483d558..e424ce5464 100644
--- a/src/trunnel/ed25519_cert.trunnel
+++ b/src/trunnel/ed25519_cert.trunnel
@@ -28,12 +28,6 @@ const LS_IPV6 = 0x01;
 const LS_LEGACY_ID = 0x02;
 const LS_ED25519_ID = 0x03;
 
-// XXX hs_link_specifier_dup() violates the opaqueness of link_specifier_t by
-//  taking its sizeof(). If we ever want to turn on TRUNNEL_OPAQUE, or
-//  if we ever make link_specifier contain other types, we will
-//  need to refactor that function to do the copy by encoding and decoding the
-//  object.
-
 // amended from tor.trunnel
 struct link_specifier {
   u8 ls_type;
diff --git a/src/trunnel/hs/.may_include b/src/trunnel/hs/.may_include
new file mode 100644
index 0000000000..424c745c12
--- /dev/null
+++ b/src/trunnel/hs/.may_include
@@ -0,0 +1 @@
+*.h
diff --git a/src/trunnel/hs/cell_common.c b/src/trunnel/hs/cell_common.c
index af223560c1..1f50961d69 100644
--- a/src/trunnel/hs/cell_common.c
+++ b/src/trunnel/hs/cell_common.c
@@ -1,4 +1,4 @@
-/* cell_common.c -- generated by Trunnel v1.5.2.
+/* cell_common.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -28,10 +28,10 @@ int cellcommon_deadcode_dummy__ = 0;
     }                                                            \
   } while (0)
 
-trn_cell_extension_fields_t *
-trn_cell_extension_fields_new(void)
+trn_cell_extension_field_t *
+trn_cell_extension_field_new(void)
 {
-  trn_cell_extension_fields_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_field_t));
   if (NULL == val)
     return NULL;
   return val;
@@ -40,7 +40,7 @@ trn_cell_extension_fields_new(void)
 /** Release all storage held inside 'obj', but do not free 'obj'.
  */
 static void
-trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear(trn_cell_extension_field_t *obj)
 {
   (void) obj;
   TRUNNEL_DYNARRAY_WIPE(&obj->field);
@@ -48,62 +48,62 @@ trn_cell_extension_fields_clear(trn_cell_extension_fields_t *obj)
 }
 
 void
-trn_cell_extension_fields_free(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_free(trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return;
-  trn_cell_extension_fields_clear(obj);
-  trunnel_memwipe(obj, sizeof(trn_cell_extension_fields_t));
+  trn_cell_extension_field_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_field_t));
   trunnel_free_(obj);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp)
 {
   return inp->field_type;
 }
 int
-trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_type = val;
   return 0;
 }
 uint8_t
-trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp)
 {
   return inp->field_len;
 }
 int
-trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val)
+trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val)
 {
   inp->field_len = val;
   return 0;
 }
 size_t
-trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp)
 {
   return TRUNNEL_DYNARRAY_LEN(&inp->field);
 }
 
 uint8_t
-trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->field, idx);
 }
 
 uint8_t
-trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx)
+trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx)
 {
-  return trn_cell_extension_fields_get_field((trn_cell_extension_fields_t*)inp, idx);
+  return trn_cell_extension_field_get_field((trn_cell_extension_field_t*)inp, idx);
 }
 int
-trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt)
+trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->field, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt)
+trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->field.n_ == UINT8_MAX)
@@ -117,17 +117,17 @@ trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t el
 }
 
 uint8_t *
-trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp)
 {
   return inp->field.elts_;
 }
 const uint8_t  *
-trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp)
+trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp)
 {
-  return (const uint8_t  *)trn_cell_extension_fields_getarray_field((trn_cell_extension_fields_t*)inp);
+  return (const uint8_t  *)trn_cell_extension_field_getarray_field((trn_cell_extension_field_t*)inp);
 }
 int
-trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen)
+trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen)
 {
   uint8_t *newptr;
 #if UINT8_MAX < SIZE_MAX
@@ -147,7 +147,7 @@ trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t
   return -1;
 }
 const char *
-trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_check(const trn_cell_extension_field_t *obj)
 {
   if (obj == NULL)
     return "Object was NULL";
@@ -159,11 +159,11 @@ trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj)
 }
 
 ssize_t
-trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
 
-  if (NULL != trn_cell_extension_fields_check(obj))
+  if (NULL != trn_cell_extension_field_check(obj))
      return -1;
 
 
@@ -178,24 +178,24 @@ trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj)
   return result;
 }
 int
-trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj)
 {
   int r = obj->trunnel_error_code_;
   obj->trunnel_error_code_ = 0;
   return r;
 }
 ssize_t
-trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_cell_extension_fields_t *obj)
+trn_cell_extension_field_encode(uint8_t *output, const size_t avail, const trn_cell_extension_field_t *obj)
 {
   ssize_t result = 0;
   size_t written = 0;
   uint8_t *ptr = output;
   const char *msg;
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
-  const ssize_t encoded_len = trn_cell_extension_fields_encoded_len(obj);
+  const ssize_t encoded_len = trn_cell_extension_field_encoded_len(obj);
 #endif
 
-  if (NULL != (msg = trn_cell_extension_fields_check(obj)))
+  if (NULL != (msg = trn_cell_extension_field_check(obj)))
     goto check_failed;
 
 #ifdef TRUNNEL_CHECK_ENCODED_LEN
@@ -252,11 +252,11 @@ trn_cell_extension_fields_encode(uint8_t *output, const size_t avail, const trn_
   return result;
 }
 
-/** As trn_cell_extension_fields_parse(), but do not allocate the
+/** As trn_cell_extension_field_parse(), but do not allocate the
  * output object.
  */
 static ssize_t
-trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse_into(trn_cell_extension_field_t *obj, const uint8_t *input, const size_t len_in)
 {
   const uint8_t *ptr = input;
   size_t remaining = len_in;
@@ -290,15 +290,15 @@ trn_cell_extension_fields_parse_into(trn_cell_extension_fields_t *obj, const uin
 }
 
 ssize_t
-trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in)
+trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in)
 {
   ssize_t result;
-  *output = trn_cell_extension_fields_new();
+  *output = trn_cell_extension_field_new();
   if (NULL == *output)
     return -1;
-  result = trn_cell_extension_fields_parse_into(*output, input, len_in);
+  result = trn_cell_extension_field_parse_into(*output, input, len_in);
   if (result < 0) {
-    trn_cell_extension_fields_free(*output);
+    trn_cell_extension_field_free(*output);
     *output = NULL;
   }
   return result;
@@ -322,7 +322,7 @@ trn_cell_extension_clear(trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      trn_cell_extension_fields_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      trn_cell_extension_field_free(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   TRUNNEL_DYNARRAY_WIPE(&obj->fields);
@@ -356,66 +356,66 @@ trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp)
   return TRUNNEL_DYNARRAY_LEN(&inp->fields);
 }
 
-struct trn_cell_extension_fields_st *
+struct trn_cell_extension_field_st *
 trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx)
 {
   return TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
 }
 
- const struct trn_cell_extension_fields_st *
+ const struct trn_cell_extension_field_st *
 trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx)
 {
   return trn_cell_extension_get_fields((trn_cell_extension_t*)inp, idx);
 }
 int
-trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
-  trn_cell_extension_fields_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
+  trn_cell_extension_field_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->fields, idx);
   if (oldval && oldval != elt)
-    trn_cell_extension_fields_free(oldval);
+    trn_cell_extension_field_free(oldval);
   return trn_cell_extension_set0_fields(inp, idx, elt);
 }
 int
-trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt)
 {
   TRUNNEL_DYNARRAY_SET(&inp->fields, idx, elt);
   return 0;
 }
 int
-trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt)
+trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt)
 {
 #if SIZE_MAX >= UINT8_MAX
   if (inp->fields.n_ == UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
-  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_fields_st *, &inp->fields, elt, {});
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_field_st *, &inp->fields, elt, {});
   return 0;
  trunnel_alloc_failed:
   TRUNNEL_SET_ERROR_CODE(inp);
   return -1;
 }
 
-struct trn_cell_extension_fields_st * *
+struct trn_cell_extension_field_st * *
 trn_cell_extension_getarray_fields(trn_cell_extension_t *inp)
 {
   return inp->fields.elts_;
 }
-const struct trn_cell_extension_fields_st *  const  *
+const struct trn_cell_extension_field_st *  const  *
 trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp)
 {
-  return (const struct trn_cell_extension_fields_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
+  return (const struct trn_cell_extension_field_st *  const  *)trn_cell_extension_getarray_fields((trn_cell_extension_t*)inp);
 }
 int
 trn_cell_extension_setlen_fields(trn_cell_extension_t *inp, size_t newlen)
 {
-  struct trn_cell_extension_fields_st * *newptr;
+  struct trn_cell_extension_field_st * *newptr;
 #if UINT8_MAX < SIZE_MAX
   if (newlen > UINT8_MAX)
     goto trunnel_alloc_failed;
 #endif
   newptr = trunnel_dynarray_setlen(&inp->fields.allocated_,
                  &inp->fields.n_, inp->fields.elts_, newlen,
-                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_fields_free,
+                 sizeof(inp->fields.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_field_free,
                  &inp->trunnel_error_code_);
   if (newlen != 0 && newptr == NULL)
     goto trunnel_alloc_failed;
@@ -437,7 +437,7 @@ trn_cell_extension_check(const trn_cell_extension_t *obj)
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      if (NULL != (msg = trn_cell_extension_fields_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
+      if (NULL != (msg = trn_cell_extension_field_check(TRUNNEL_DYNARRAY_GET(&obj->fields, idx))))
         return msg;
     }
   }
@@ -458,12 +458,12 @@ trn_cell_extension_encoded_len(const trn_cell_extension_t *obj)
   /* Length of u8 num */
   result += 1;
 
-  /* Length of struct trn_cell_extension_fields fields[num] */
+  /* Length of struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
-      result += trn_cell_extension_fields_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result += trn_cell_extension_field_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
     }
   }
   return result;
@@ -500,13 +500,13 @@ trn_cell_extension_encode(uint8_t *output, const size_t avail, const trn_cell_ex
   trunnel_set_uint8(ptr, (obj->num));
   written += 1; ptr += 1;
 
-  /* Encode struct trn_cell_extension_fields fields[num] */
+  /* Encode struct trn_cell_extension_field fields[num] */
   {
 
     unsigned idx;
     for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->fields); ++idx) {
       trunnel_assert(written <= avail);
-      result = trn_cell_extension_fields_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
+      result = trn_cell_extension_field_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->fields, idx));
       if (result < 0)
         goto fail; /* XXXXXXX !*/
       written += result; ptr += result;
@@ -553,18 +553,18 @@ trn_cell_extension_parse_into(trn_cell_extension_t *obj, const uint8_t *input, c
   obj->num = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
 
-  /* Parse struct trn_cell_extension_fields fields[num] */
-  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_fields_t *, &obj->fields, obj->num, {});
+  /* Parse struct trn_cell_extension_field fields[num] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_field_t *, &obj->fields, obj->num, {});
   {
-    trn_cell_extension_fields_t * elt;
+    trn_cell_extension_field_t * elt;
     unsigned idx;
     for (idx = 0; idx < obj->num; ++idx) {
-      result = trn_cell_extension_fields_parse(&elt, ptr, remaining);
+      result = trn_cell_extension_field_parse(&elt, ptr, remaining);
       if (result < 0)
         goto relay_fail;
       trunnel_assert((size_t)result <= remaining);
       remaining -= result; ptr += result;
-      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_fields_t *, &obj->fields, elt, {trn_cell_extension_fields_free(elt);});
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_field_t *, &obj->fields, elt, {trn_cell_extension_field_free(elt);});
     }
   }
   trunnel_assert(ptr + remaining == input + len_in);
diff --git a/src/trunnel/hs/cell_common.h b/src/trunnel/hs/cell_common.h
index e08eedfdb3..beb65e015f 100644
--- a/src/trunnel/hs/cell_common.h
+++ b/src/trunnel/hs/cell_common.h
@@ -1,4 +1,4 @@
-/* cell_common.h -- generated by Trunnel v1.5.2.
+/* cell_common.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -8,112 +8,112 @@
 #include <stdint.h>
 #include "trunnel.h"
 
-#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELDS)
-struct trn_cell_extension_fields_st {
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_FIELD)
+struct trn_cell_extension_field_st {
   uint8_t field_type;
   uint8_t field_len;
   TRUNNEL_DYNARRAY_HEAD(, uint8_t) field;
   uint8_t trunnel_error_code_;
 };
 #endif
-typedef struct trn_cell_extension_fields_st trn_cell_extension_fields_t;
+typedef struct trn_cell_extension_field_st trn_cell_extension_field_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION)
 struct trn_cell_extension_st {
   uint8_t num;
-  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_fields_st *) fields;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_field_st *) fields;
   uint8_t trunnel_error_code_;
 };
 #endif
 typedef struct trn_cell_extension_st trn_cell_extension_t;
-/** Return a newly allocated trn_cell_extension_fields with all
+/** Return a newly allocated trn_cell_extension_field with all
  * elements set to zero.
  */
-trn_cell_extension_fields_t *trn_cell_extension_fields_new(void);
-/** Release all storage held by the trn_cell_extension_fields in
+trn_cell_extension_field_t *trn_cell_extension_field_new(void);
+/** Release all storage held by the trn_cell_extension_field in
  * 'victim'. (Do nothing if 'victim' is NULL.)
  */
-void trn_cell_extension_fields_free(trn_cell_extension_fields_t *victim);
-/** Try to parse a trn_cell_extension_fields from the buffer in
+void trn_cell_extension_field_free(trn_cell_extension_field_t *victim);
+/** Try to parse a trn_cell_extension_field from the buffer in
  * 'input', using up to 'len_in' bytes from the input buffer. On
  * success, return the number of bytes consumed and set *output to the
- * newly allocated trn_cell_extension_fields_t. On failure, return -2
+ * newly allocated trn_cell_extension_field_t. On failure, return -2
  * if the input appears truncated, and -1 if the input is otherwise
  * invalid.
  */
-ssize_t trn_cell_extension_fields_parse(trn_cell_extension_fields_t **output, const uint8_t *input, const size_t len_in);
+ssize_t trn_cell_extension_field_parse(trn_cell_extension_field_t **output, const uint8_t *input, const size_t len_in);
 /** Return the number of bytes we expect to need to encode the
- * trn_cell_extension_fields in 'obj'. On failure, return a negative
+ * trn_cell_extension_field in 'obj'. On failure, return a negative
  * value. Note that this value may be an overestimate, and can even be
  * an underestimate for certain unencodeable objects.
  */
-ssize_t trn_cell_extension_fields_encoded_len(const trn_cell_extension_fields_t *obj);
-/** Try to encode the trn_cell_extension_fields from 'input' into the
+ssize_t trn_cell_extension_field_encoded_len(const trn_cell_extension_field_t *obj);
+/** Try to encode the trn_cell_extension_field from 'input' into the
  * buffer at 'output', using up to 'avail' bytes of the output buffer.
  * On success, return the number of bytes used. On failure, return -2
  * if the buffer was not long enough, and -1 if the input was invalid.
  */
-ssize_t trn_cell_extension_fields_encode(uint8_t *output, size_t avail, const trn_cell_extension_fields_t *input);
-/** Check whether the internal state of the trn_cell_extension_fields
+ssize_t trn_cell_extension_field_encode(uint8_t *output, size_t avail, const trn_cell_extension_field_t *input);
+/** Check whether the internal state of the trn_cell_extension_field
  * in 'obj' is consistent. Return NULL if it is, and a short message
  * if it is not.
  */
-const char *trn_cell_extension_fields_check(const trn_cell_extension_fields_t *obj);
+const char *trn_cell_extension_field_check(const trn_cell_extension_field_t *obj);
 /** Clear any errors that were set on the object 'obj' by its setter
  * functions. Return true iff errors were cleared.
  */
-int trn_cell_extension_fields_clear_errors(trn_cell_extension_fields_t *obj);
+int trn_cell_extension_field_clear_errors(trn_cell_extension_field_t *obj);
 /** Return the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_type(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_type(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_type field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_type(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_type(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp'
+ * trn_cell_extension_field_t in 'inp'
  */
-uint8_t trn_cell_extension_fields_get_field_len(const trn_cell_extension_fields_t *inp);
+uint8_t trn_cell_extension_field_get_field_len(const trn_cell_extension_field_t *inp);
 /** Set the value of the field_len field of the
- * trn_cell_extension_fields_t in 'inp' to 'val'. Return 0 on success;
+ * trn_cell_extension_field_t in 'inp' to 'val'. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_set_field_len(trn_cell_extension_fields_t *inp, uint8_t val);
+int trn_cell_extension_field_set_field_len(trn_cell_extension_field_t *inp, uint8_t val);
 /** Return the length of the dynamic array holding the field field of
- * the trn_cell_extension_fields_t in 'inp'.
+ * the trn_cell_extension_field_t in 'inp'.
  */
-size_t trn_cell_extension_fields_getlen_field(const trn_cell_extension_fields_t *inp);
+size_t trn_cell_extension_field_getlen_field(const trn_cell_extension_field_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp'.
+ * field of the trn_cell_extension_field_t in 'inp'.
  */
-uint8_t trn_cell_extension_fields_get_field(trn_cell_extension_fields_t *inp, size_t idx);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t trn_cell_extension_field_get_field(trn_cell_extension_field_t *inp, size_t idx);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-uint8_t trn_cell_extension_fields_getconst_field(const trn_cell_extension_fields_t *inp, size_t idx);
+uint8_t trn_cell_extension_field_getconst_field(const trn_cell_extension_field_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
- * field of the trn_cell_extension_fields_t in 'inp', so that it will
+ * field of the trn_cell_extension_field_t in 'inp', so that it will
  * hold the value 'elt'.
  */
-int trn_cell_extension_fields_set_field(trn_cell_extension_fields_t *inp, size_t idx, uint8_t elt);
+int trn_cell_extension_field_set_field(trn_cell_extension_field_t *inp, size_t idx, uint8_t elt);
 /** Append a new element 'elt' to the dynamic array field field of the
- * trn_cell_extension_fields_t in 'inp'.
+ * trn_cell_extension_field_t in 'inp'.
  */
-int trn_cell_extension_fields_add_field(trn_cell_extension_fields_t *inp, uint8_t elt);
+int trn_cell_extension_field_add_field(trn_cell_extension_field_t *inp, uint8_t elt);
 /** Return a pointer to the variable-length array field field of
  * 'inp'.
  */
-uint8_t * trn_cell_extension_fields_getarray_field(trn_cell_extension_fields_t *inp);
-/** As trn_cell_extension_fields_get_field, but take and return a
- * const pointer
+uint8_t * trn_cell_extension_field_getarray_field(trn_cell_extension_field_t *inp);
+/** As trn_cell_extension_field_get_field, but take and return a const
+ * pointer
  */
-const uint8_t  * trn_cell_extension_fields_getconstarray_field(const trn_cell_extension_fields_t *inp);
+const uint8_t  * trn_cell_extension_field_getconstarray_field(const trn_cell_extension_field_t *inp);
 /** Change the length of the variable-length array field field of
  * 'inp' to 'newlen'.Fill extra elements with 0. Return 0 on success;
  * return -1 and set the error code on 'inp' on failure.
  */
-int trn_cell_extension_fields_setlen_field(trn_cell_extension_fields_t *inp, size_t newlen);
+int trn_cell_extension_field_setlen_field(trn_cell_extension_field_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_extension with all elements set
  * to zero.
  */
@@ -166,32 +166,32 @@ size_t trn_cell_extension_getlen_fields(const trn_cell_extension_t *inp);
 /** Return the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp'.
  */
-struct trn_cell_extension_fields_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
+struct trn_cell_extension_field_st * trn_cell_extension_get_fields(trn_cell_extension_t *inp, size_t idx);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
- const struct trn_cell_extension_fields_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
+ const struct trn_cell_extension_field_st * trn_cell_extension_getconst_fields(const trn_cell_extension_t *inp, size_t idx);
 /** Change the element at position 'idx' of the dynamic array field
  * fields of the trn_cell_extension_t in 'inp', so that it will hold
  * the value 'elt'. Free the previous value, if any.
  */
-int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** As trn_cell_extension_set_fields, but does not free the previous
  * value.
  */
-int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_set0_fields(trn_cell_extension_t *inp, size_t idx, struct trn_cell_extension_field_st * elt);
 /** Append a new element 'elt' to the dynamic array field fields of
  * the trn_cell_extension_t in 'inp'.
  */
-int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_fields_st * elt);
+int trn_cell_extension_add_fields(trn_cell_extension_t *inp, struct trn_cell_extension_field_st * elt);
 /** Return a pointer to the variable-length array field fields of
  * 'inp'.
  */
-struct trn_cell_extension_fields_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
+struct trn_cell_extension_field_st * * trn_cell_extension_getarray_fields(trn_cell_extension_t *inp);
 /** As trn_cell_extension_get_fields, but take and return a const
  * pointer
  */
-const struct trn_cell_extension_fields_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
+const struct trn_cell_extension_field_st *  const  * trn_cell_extension_getconstarray_fields(const trn_cell_extension_t *inp);
 /** Change the length of the variable-length array field fields of
  * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
  * elements. Return 0 on success; return -1 and set the error code on
diff --git a/src/trunnel/hs/cell_common.trunnel b/src/trunnel/hs/cell_common.trunnel
index 1aa6999de7..7e99cbfa66 100644
--- a/src/trunnel/hs/cell_common.trunnel
+++ b/src/trunnel/hs/cell_common.trunnel
@@ -1,6 +1,6 @@
 /* This file contains common data structure that cells use. */
 
-struct trn_cell_extension_fields {
+struct trn_cell_extension_field {
   u8 field_type;
   u8 field_len;
   u8 field[field_len];
@@ -8,5 +8,5 @@ struct trn_cell_extension_fields {
 
 struct trn_cell_extension {
   u8 num;
-  struct trn_cell_extension_fields fields[num];
+  struct trn_cell_extension_field fields[num];
 };
diff --git a/src/trunnel/hs/cell_establish_intro.c b/src/trunnel/hs/cell_establish_intro.c
index ae3b7b1bc8..f31404c55f 100644
--- a/src/trunnel/hs/cell_establish_intro.c
+++ b/src/trunnel/hs/cell_establish_intro.c
@@ -1,4 +1,4 @@
-/* cell_establish_intro.c -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -36,6 +36,185 @@ ssize_t trn_cell_extension_encoded_len(const trn_cell_extension_t *obj);
 ssize_t trn_cell_extension_encode(uint8_t *output, size_t avail, const trn_cell_extension_t *input);
 const char *trn_cell_extension_check(const trn_cell_extension_t *obj);
 int trn_cell_extension_clear_errors(trn_cell_extension_t *obj);
+trn_cell_extension_dos_param_t *
+trn_cell_extension_dos_param_new(void)
+{
+  trn_cell_extension_dos_param_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_param_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_param_clear(trn_cell_extension_dos_param_t *obj)
+{
+  (void) obj;
+}
+
+void
+trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_param_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_param_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->type;
+}
+int
+trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val)
+{
+  inp->type = val;
+  return 0;
+}
+uint64_t
+trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp)
+{
+  return inp->value;
+}
+int
+trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val)
+{
+  inp->value = val;
+  return 0;
+}
+const char *
+trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_param_check(obj))
+     return -1;
+
+
+  /* Length of u8 type */
+  result += 1;
+
+  /* Length of u64 value */
+  result += 8;
+  return result;
+}
+int
+trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_param_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_param_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_param_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_param_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->type));
+  written += 1; ptr += 1;
+
+  /* Encode u64 value */
+  trunnel_assert(written <= avail);
+  if (avail - written < 8)
+    goto truncated;
+  trunnel_set_uint64(ptr, trunnel_htonll(obj->value));
+  written += 8; ptr += 8;
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_param_parse(), but do not allocate the
+ * output object.
+ */
+static ssize_t
+trn_cell_extension_dos_param_parse_into(trn_cell_extension_dos_param_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 type */
+  CHECK_REMAINING(1, truncated);
+  obj->type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u64 value */
+  CHECK_REMAINING(8, truncated);
+  obj->value = trunnel_ntohll(trunnel_get_uint64(ptr));
+  remaining -= 8; ptr += 8;
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+}
+
+ssize_t
+trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_param_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_param_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_param_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_establish_intro_t *
 trn_cell_establish_intro_new(void)
 {
@@ -561,6 +740,296 @@ trn_cell_establish_intro_parse(trn_cell_establish_intro_t **output, const uint8_
   }
   return result;
 }
+trn_cell_extension_dos_t *
+trn_cell_extension_dos_new(void)
+{
+  trn_cell_extension_dos_t *val = trunnel_calloc(1, sizeof(trn_cell_extension_dos_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+trn_cell_extension_dos_clear(trn_cell_extension_dos_t *obj)
+{
+  (void) obj;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trn_cell_extension_dos_param_free(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->params);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->params);
+}
+
+void
+trn_cell_extension_dos_free(trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return;
+  trn_cell_extension_dos_clear(obj);
+  trunnel_memwipe(obj, sizeof(trn_cell_extension_dos_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp)
+{
+  return inp->n_params;
+}
+int
+trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val)
+{
+  inp->n_params = val;
+  return 0;
+}
+size_t
+trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->params);
+}
+
+struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+}
+
+ const struct trn_cell_extension_dos_param_st *
+trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx)
+{
+  return trn_cell_extension_dos_get_params((trn_cell_extension_dos_t*)inp, idx);
+}
+int
+trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  trn_cell_extension_dos_param_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->params, idx);
+  if (oldval && oldval != elt)
+    trn_cell_extension_dos_param_free(oldval);
+  return trn_cell_extension_dos_set0_params(inp, idx, elt);
+}
+int
+trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->params, idx, elt);
+  return 0;
+}
+int
+trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->params.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct trn_cell_extension_dos_param_st *, &inp->params, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct trn_cell_extension_dos_param_st * *
+trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp)
+{
+  return inp->params.elts_;
+}
+const struct trn_cell_extension_dos_param_st *  const  *
+trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp)
+{
+  return (const struct trn_cell_extension_dos_param_st *  const  *)trn_cell_extension_dos_getarray_params((trn_cell_extension_dos_t*)inp);
+}
+int
+trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen)
+{
+  struct trn_cell_extension_dos_param_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->params.allocated_,
+                 &inp->params.n_, inp->params.elts_, newlen,
+                 sizeof(inp->params.elts_[0]), (trunnel_free_fn_t) trn_cell_extension_dos_param_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->params.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      if (NULL != (msg = trn_cell_extension_dos_param_check(TRUNNEL_DYNARRAY_GET(&obj->params, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->params) != obj->n_params)
+    return "Length mismatch for params";
+  return NULL;
+}
+
+ssize_t
+trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != trn_cell_extension_dos_check(obj))
+     return -1;
+
+
+  /* Length of u8 n_params */
+  result += 1;
+
+  /* Length of struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      result += trn_cell_extension_dos_param_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+    }
+  }
+  return result;
+}
+int
+trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+trn_cell_extension_dos_encode(uint8_t *output, const size_t avail, const trn_cell_extension_dos_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = trn_cell_extension_dos_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = trn_cell_extension_dos_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 n_params */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_params));
+  written += 1; ptr += 1;
+
+  /* Encode struct trn_cell_extension_dos_param params[n_params] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->params); ++idx) {
+      trunnel_assert(written <= avail);
+      result = trn_cell_extension_dos_param_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->params, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As trn_cell_extension_dos_parse(), but do not allocate the output
+ * object.
+ */
+static ssize_t
+trn_cell_extension_dos_parse_into(trn_cell_extension_dos_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 n_params */
+  CHECK_REMAINING(1, truncated);
+  obj->n_params = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct trn_cell_extension_dos_param params[n_params] */
+  TRUNNEL_DYNARRAY_EXPAND(trn_cell_extension_dos_param_t *, &obj->params, obj->n_params, {});
+  {
+    trn_cell_extension_dos_param_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_params; ++idx) {
+      result = trn_cell_extension_dos_param_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(trn_cell_extension_dos_param_t *, &obj->params, elt, {trn_cell_extension_dos_param_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = trn_cell_extension_dos_new();
+  if (NULL == *output)
+    return -1;
+  result = trn_cell_extension_dos_parse_into(*output, input, len_in);
+  if (result < 0) {
+    trn_cell_extension_dos_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
 trn_cell_intro_established_t *
 trn_cell_intro_established_new(void)
 {
diff --git a/src/trunnel/hs/cell_establish_intro.h b/src/trunnel/hs/cell_establish_intro.h
index ccaef5488c..1924d9cab6 100644
--- a/src/trunnel/hs/cell_establish_intro.h
+++ b/src/trunnel/hs/cell_establish_intro.h
@@ -1,4 +1,4 @@
-/* cell_establish_intro.h -- generated by Trunnel v1.5.2.
+/* cell_establish_intro.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -10,6 +10,17 @@
 
 struct trn_cell_extension_st;
 #define TRUNNEL_SHA3_256_LEN 32
+#define TRUNNEL_CELL_EXTENSION_TYPE_DOS 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC 1
+#define TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC 2
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS_PARAM)
+struct trn_cell_extension_dos_param_st {
+  uint8_t type;
+  uint64_t value;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_param_st trn_cell_extension_dos_param_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_ESTABLISH_INTRO)
 struct trn_cell_establish_intro_st {
   const uint8_t *start_cell;
@@ -26,6 +37,14 @@ struct trn_cell_establish_intro_st {
 };
 #endif
 typedef struct trn_cell_establish_intro_st trn_cell_establish_intro_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_EXTENSION_DOS)
+struct trn_cell_extension_dos_st {
+  uint8_t n_params;
+  TRUNNEL_DYNARRAY_HEAD(, struct trn_cell_extension_dos_param_st *) params;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct trn_cell_extension_dos_st trn_cell_extension_dos_t;
 #if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_TRN_CELL_INTRO_ESTABLISHED)
 struct trn_cell_intro_established_st {
   struct trn_cell_extension_st *extensions;
@@ -33,6 +52,62 @@ struct trn_cell_intro_established_st {
 };
 #endif
 typedef struct trn_cell_intro_established_st trn_cell_intro_established_t;
+/** Return a newly allocated trn_cell_extension_dos_param with all
+ * elements set to zero.
+ */
+trn_cell_extension_dos_param_t *trn_cell_extension_dos_param_new(void);
+/** Release all storage held by the trn_cell_extension_dos_param in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_param_free(trn_cell_extension_dos_param_t *victim);
+/** Try to parse a trn_cell_extension_dos_param from the buffer in
+ * 'input', using up to 'len_in' bytes from the input buffer. On
+ * success, return the number of bytes consumed and set *output to the
+ * newly allocated trn_cell_extension_dos_param_t. On failure, return
+ * -2 if the input appears truncated, and -1 if the input is otherwise
+ * invalid.
+ */
+ssize_t trn_cell_extension_dos_param_parse(trn_cell_extension_dos_param_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos_param in 'obj'. On failure, return a
+ * negative value. Note that this value may be an overestimate, and
+ * can even be an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_param_encoded_len(const trn_cell_extension_dos_param_t *obj);
+/** Try to encode the trn_cell_extension_dos_param from 'input' into
+ * the buffer at 'output', using up to 'avail' bytes of the output
+ * buffer. On success, return the number of bytes used. On failure,
+ * return -2 if the buffer was not long enough, and -1 if the input
+ * was invalid.
+ */
+ssize_t trn_cell_extension_dos_param_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_param_t *input);
+/** Check whether the internal state of the
+ * trn_cell_extension_dos_param in 'obj' is consistent. Return NULL if
+ * it is, and a short message if it is not.
+ */
+const char *trn_cell_extension_dos_param_check(const trn_cell_extension_dos_param_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_param_clear_errors(trn_cell_extension_dos_param_t *obj);
+/** Return the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_param_get_type(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the type field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_type(trn_cell_extension_dos_param_t *inp, uint8_t val);
+/** Return the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp'
+ */
+uint64_t trn_cell_extension_dos_param_get_value(const trn_cell_extension_dos_param_t *inp);
+/** Set the value of the value field of the
+ * trn_cell_extension_dos_param_t in 'inp' to 'val'. Return 0 on
+ * success; return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_param_set_value(trn_cell_extension_dos_param_t *inp, uint64_t val);
 /** Return a newly allocated trn_cell_establish_intro with all
  * elements set to zero.
  */
@@ -216,6 +291,90 @@ const uint8_t  * trn_cell_establish_intro_getconstarray_sig(const trn_cell_estab
  * -1 and set the error code on 'inp' on failure.
  */
 int trn_cell_establish_intro_setlen_sig(trn_cell_establish_intro_t *inp, size_t newlen);
+/** Return a newly allocated trn_cell_extension_dos with all elements
+ * set to zero.
+ */
+trn_cell_extension_dos_t *trn_cell_extension_dos_new(void);
+/** Release all storage held by the trn_cell_extension_dos in
+ * 'victim'. (Do nothing if 'victim' is NULL.)
+ */
+void trn_cell_extension_dos_free(trn_cell_extension_dos_t *victim);
+/** Try to parse a trn_cell_extension_dos from the buffer in 'input',
+ * using up to 'len_in' bytes from the input buffer. On success,
+ * return the number of bytes consumed and set *output to the newly
+ * allocated trn_cell_extension_dos_t. On failure, return -2 if the
+ * input appears truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t trn_cell_extension_dos_parse(trn_cell_extension_dos_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * trn_cell_extension_dos in 'obj'. On failure, return a negative
+ * value. Note that this value may be an overestimate, and can even be
+ * an underestimate for certain unencodeable objects.
+ */
+ssize_t trn_cell_extension_dos_encoded_len(const trn_cell_extension_dos_t *obj);
+/** Try to encode the trn_cell_extension_dos from 'input' into the
+ * buffer at 'output', using up to 'avail' bytes of the output buffer.
+ * On success, return the number of bytes used. On failure, return -2
+ * if the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t trn_cell_extension_dos_encode(uint8_t *output, size_t avail, const trn_cell_extension_dos_t *input);
+/** Check whether the internal state of the trn_cell_extension_dos in
+ * 'obj' is consistent. Return NULL if it is, and a short message if
+ * it is not.
+ */
+const char *trn_cell_extension_dos_check(const trn_cell_extension_dos_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int trn_cell_extension_dos_clear_errors(trn_cell_extension_dos_t *obj);
+/** Return the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp'
+ */
+uint8_t trn_cell_extension_dos_get_n_params(const trn_cell_extension_dos_t *inp);
+/** Set the value of the n_params field of the
+ * trn_cell_extension_dos_t in 'inp' to 'val'. Return 0 on success;
+ * return -1 and set the error code on 'inp' on failure.
+ */
+int trn_cell_extension_dos_set_n_params(trn_cell_extension_dos_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the params field of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+size_t trn_cell_extension_dos_getlen_params(const trn_cell_extension_dos_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_get_params(trn_cell_extension_dos_t *inp, size_t idx);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+ const struct trn_cell_extension_dos_param_st * trn_cell_extension_dos_getconst_params(const trn_cell_extension_dos_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * params of the trn_cell_extension_dos_t in 'inp', so that it will
+ * hold the value 'elt'. Free the previous value, if any.
+ */
+int trn_cell_extension_dos_set_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** As trn_cell_extension_dos_set_params, but does not free the
+ * previous value.
+ */
+int trn_cell_extension_dos_set0_params(trn_cell_extension_dos_t *inp, size_t idx, struct trn_cell_extension_dos_param_st * elt);
+/** Append a new element 'elt' to the dynamic array field params of
+ * the trn_cell_extension_dos_t in 'inp'.
+ */
+int trn_cell_extension_dos_add_params(trn_cell_extension_dos_t *inp, struct trn_cell_extension_dos_param_st * elt);
+/** Return a pointer to the variable-length array field params of
+ * 'inp'.
+ */
+struct trn_cell_extension_dos_param_st * * trn_cell_extension_dos_getarray_params(trn_cell_extension_dos_t *inp);
+/** As trn_cell_extension_dos_get_params, but take and return a const
+ * pointer
+ */
+const struct trn_cell_extension_dos_param_st *  const  * trn_cell_extension_dos_getconstarray_params(const trn_cell_extension_dos_t *inp);
+/** Change the length of the variable-length array field params of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int trn_cell_extension_dos_setlen_params(trn_cell_extension_dos_t *inp, size_t newlen);
 /** Return a newly allocated trn_cell_intro_established with all
  * elements set to zero.
  */
diff --git a/src/trunnel/hs/cell_establish_intro.trunnel b/src/trunnel/hs/cell_establish_intro.trunnel
index 011ee62a15..e30938f6c2 100644
--- a/src/trunnel/hs/cell_establish_intro.trunnel
+++ b/src/trunnel/hs/cell_establish_intro.trunnel
@@ -39,3 +39,26 @@ struct trn_cell_intro_established {
   /* Extension(s). Reserved fields. */
   struct trn_cell_extension extensions;
 };
+
+/*
+ * ESTABLISH_INTRO cell extensions.
+ */
+
+const TRUNNEL_CELL_EXTENSION_TYPE_DOS = 0x01;
+
+/* DoS Parameter types. */
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_RATE_PER_SEC = 0x01;
+const TRUNNEL_DOS_PARAM_TYPE_INTRO2_BURST_PER_SEC = 0x02;
+
+/*
+ * DoS Parameters Extension. See proposal 305 for more details.
+ */
+struct trn_cell_extension_dos_param {
+  u8 type;
+  u64 value;
+};
+
+struct trn_cell_extension_dos {
+  u8 n_params;
+  struct trn_cell_extension_dos_param params[n_params];
+};
diff --git a/src/trunnel/hs/cell_introduce1.c b/src/trunnel/hs/cell_introduce1.c
index 53b3d299f2..016c9fa8d6 100644
--- a/src/trunnel/hs/cell_introduce1.c
+++ b/src/trunnel/hs/cell_introduce1.c
@@ -1,4 +1,4 @@
-/* cell_introduce1.c -- generated by Trunnel v1.5.2.
+/* cell_introduce1.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_introduce1.h b/src/trunnel/hs/cell_introduce1.h
index 986a531ca7..8dabff3cb5 100644
--- a/src/trunnel/hs/cell_introduce1.h
+++ b/src/trunnel/hs/cell_introduce1.h
@@ -1,4 +1,4 @@
-/* cell_introduce1.h -- generated by Trunnel v1.5.2.
+/* cell_introduce1.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.c b/src/trunnel/hs/cell_rendezvous.c
index 53cb609138..1204e93cfc 100644
--- a/src/trunnel/hs/cell_rendezvous.c
+++ b/src/trunnel/hs/cell_rendezvous.c
@@ -1,4 +1,4 @@
-/* cell_rendezvous.c -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/hs/cell_rendezvous.h b/src/trunnel/hs/cell_rendezvous.h
index 39e14da25b..5a8c2ff52a 100644
--- a/src/trunnel/hs/cell_rendezvous.h
+++ b/src/trunnel/hs/cell_rendezvous.h
@@ -1,4 +1,4 @@
-/* cell_rendezvous.h -- generated by Trunnel v1.5.2.
+/* cell_rendezvous.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/include.am b/src/trunnel/include.am
index 03c1753e96..6c3a5ff06b 100644
--- a/src/trunnel/include.am
+++ b/src/trunnel/include.am
@@ -11,7 +11,9 @@ TRUNNELINPUTS = \
 	src/trunnel/link_handshake.trunnel \
 	src/trunnel/pwbox.trunnel \
 	src/trunnel/channelpadding_negotiation.trunnel \
-	src/trunner/socks5.trunnel
+	src/trunnel/sendme_cell.trunnel \
+	src/trunnel/socks5.trunnel \
+	src/trunnel/circpad_negotiation.trunnel
 
 TRUNNELSOURCES = \
 	src/ext/trunnel/trunnel.c \
@@ -23,7 +25,10 @@ TRUNNELSOURCES = \
 	src/trunnel/hs/cell_introduce1.c \
 	src/trunnel/hs/cell_rendezvous.c \
 	src/trunnel/channelpadding_negotiation.c \
-	src/trunnel/socks5.c
+	src/trunnel/sendme_cell.c                    \
+	src/trunnel/socks5.c \
+	src/trunnel/netinfo.c \
+	src/trunnel/circpad_negotiation.c
 
 TRUNNELHEADERS = \
 	src/ext/trunnel/trunnel.h		\
@@ -37,7 +42,10 @@ TRUNNELHEADERS = \
 	src/trunnel/hs/cell_introduce1.h \
 	src/trunnel/hs/cell_rendezvous.h \
 	src/trunnel/channelpadding_negotiation.h \
-	src/trunnel/socks5.h
+	src/trunnel/sendme_cell.h                    \
+	src/trunnel/socks5.h                    \
+	src/trunnel/netinfo.h \
+	src/trunnel/circpad_negotiation.h
 
 src_trunnel_libor_trunnel_a_SOURCES = $(TRUNNELSOURCES)
 src_trunnel_libor_trunnel_a_CPPFLAGS = \
diff --git a/src/trunnel/link_handshake.c b/src/trunnel/link_handshake.c
index 03ead31c62..76db4b0e29 100644
--- a/src/trunnel/link_handshake.c
+++ b/src/trunnel/link_handshake.c
@@ -1,4 +1,4 @@
-/* link_handshake.c -- generated by Trunnel v1.5.2.
+/* link_handshake.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/link_handshake.h b/src/trunnel/link_handshake.h
index 6a23483adc..0c7ac36b1b 100644
--- a/src/trunnel/link_handshake.h
+++ b/src/trunnel/link_handshake.h
@@ -1,4 +1,4 @@
-/* link_handshake.h -- generated by Trunnel v1.5.2.
+/* link_handshake.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/netinfo.c b/src/trunnel/netinfo.c
new file mode 100644
index 0000000000..d7d0cddc89
--- /dev/null
+++ b/src/trunnel/netinfo.c
@@ -0,0 +1,723 @@
+/* netinfo.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "netinfo.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int netinfo_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || netinfo_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+netinfo_addr_t *
+netinfo_addr_new(void)
+{
+  netinfo_addr_t *val = trunnel_calloc(1, sizeof(netinfo_addr_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_addr_clear(netinfo_addr_t *obj)
+{
+  (void) obj;
+}
+
+void
+netinfo_addr_free(netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_addr_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_addr_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+netinfo_addr_get_addr_type(const netinfo_addr_t *inp)
+{
+  return inp->addr_type;
+}
+int
+netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->addr_type = val;
+  return 0;
+}
+uint8_t
+netinfo_addr_get_len(const netinfo_addr_t *inp)
+{
+  return inp->len;
+}
+int
+netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val)
+{
+  inp->len = val;
+  return 0;
+}
+uint32_t
+netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp)
+{
+  return inp->addr_ipv4;
+}
+int
+netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val)
+{
+  inp->addr_ipv4 = val;
+  return 0;
+}
+size_t
+netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp)
+{
+  (void)inp;  return 16;
+}
+
+uint8_t
+netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx)
+{
+  trunnel_assert(idx < 16);
+  return inp->addr_ipv6[idx];
+}
+
+uint8_t
+netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx)
+{
+  return netinfo_addr_get_addr_ipv6((netinfo_addr_t*)inp, idx);
+}
+int
+netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < 16);
+  inp->addr_ipv6[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp)
+{
+  return inp->addr_ipv6;
+}
+const uint8_t  *
+netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp)
+{
+  return (const uint8_t  *)netinfo_addr_getarray_addr_ipv6((netinfo_addr_t*)inp);
+}
+const char *
+netinfo_addr_check(const netinfo_addr_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+      break;
+
+    default:
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+netinfo_addr_encoded_len(const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_addr_check(obj))
+     return -1;
+
+
+  /* Length of u8 addr_type */
+  result += 1;
+
+  /* Length of u8 len */
+  result += 1;
+  switch (obj->addr_type) {
+
+    case NETINFO_ADDR_TYPE_IPV4:
+
+      /* Length of u32 addr_ipv4 */
+      result += 4;
+      break;
+
+    case NETINFO_ADDR_TYPE_IPV6:
+
+      /* Length of u8 addr_ipv6[16] */
+      result += 16;
+      break;
+
+    default:
+      break;
+  }
+  return result;
+}
+int
+netinfo_addr_clear_errors(netinfo_addr_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_addr_encode(uint8_t *output, const size_t avail, const netinfo_addr_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_addr_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_len = NULL;
+
+  if (NULL != (msg = netinfo_addr_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 addr_type */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->addr_type));
+  written += 1; ptr += 1;
+
+  /* Encode u8 len */
+  backptr_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->len));
+  written += 1; ptr += 1;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union addr[addr_type] */
+    trunnel_assert(written <= avail);
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Encode u32 addr_ipv4 */
+        trunnel_assert(written <= avail);
+        if (avail - written < 4)
+          goto truncated;
+        trunnel_set_uint32(ptr, trunnel_htonl(obj->addr_ipv4));
+        written += 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Encode u8 addr_ipv6[16] */
+        trunnel_assert(written <= avail);
+        if (avail - written < 16)
+          goto truncated;
+        memcpy(ptr, obj->addr_ipv6, 16);
+        written += 16; ptr += 16;
+        break;
+
+      default:
+        break;
+    }
+    /* Write the length field back to len */
+    trunnel_assert(written >= written_before_union);
+#if UINT8_MAX < SIZE_MAX
+    if (written - written_before_union > UINT8_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint8(backptr_len, (written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_addr_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_addr_parse_into(netinfo_addr_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 addr_type */
+  CHECK_REMAINING(1, truncated);
+  obj->addr_type = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse u8 len */
+  CHECK_REMAINING(1, truncated);
+  obj->len = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->len, truncated);
+    remaining_after = remaining - obj->len;
+    remaining = obj->len;
+
+    /* Parse union addr[addr_type] */
+    switch (obj->addr_type) {
+
+      case NETINFO_ADDR_TYPE_IPV4:
+
+        /* Parse u32 addr_ipv4 */
+        CHECK_REMAINING(4, fail);
+        obj->addr_ipv4 = trunnel_ntohl(trunnel_get_uint32(ptr));
+        remaining -= 4; ptr += 4;
+        break;
+
+      case NETINFO_ADDR_TYPE_IPV6:
+
+        /* Parse u8 addr_ipv6[16] */
+        CHECK_REMAINING(16, fail);
+        memcpy(obj->addr_ipv6, ptr, 16);
+        remaining -= 16; ptr += 16;
+        break;
+
+      default:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_addr_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_addr_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_addr_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
+netinfo_cell_t *
+netinfo_cell_new(void)
+{
+  netinfo_cell_t *val = trunnel_calloc(1, sizeof(netinfo_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+netinfo_cell_clear(netinfo_cell_t *obj)
+{
+  (void) obj;
+  netinfo_addr_free(obj->other_addr);
+  obj->other_addr = NULL;
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      netinfo_addr_free(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  TRUNNEL_DYNARRAY_WIPE(&obj->my_addrs);
+  TRUNNEL_DYNARRAY_CLEAR(&obj->my_addrs);
+}
+
+void
+netinfo_cell_free(netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  netinfo_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(netinfo_cell_t));
+  trunnel_free_(obj);
+}
+
+uint32_t
+netinfo_cell_get_timestamp(const netinfo_cell_t *inp)
+{
+  return inp->timestamp;
+}
+int
+netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val)
+{
+  inp->timestamp = val;
+  return 0;
+}
+struct netinfo_addr_st *
+netinfo_cell_get_other_addr(netinfo_cell_t *inp)
+{
+  return inp->other_addr;
+}
+const struct netinfo_addr_st *
+netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp)
+{
+  return netinfo_cell_get_other_addr((netinfo_cell_t*) inp);
+}
+int
+netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  if (inp->other_addr && inp->other_addr != val)
+    netinfo_addr_free(inp->other_addr);
+  return netinfo_cell_set0_other_addr(inp, val);
+}
+int
+netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val)
+{
+  inp->other_addr = val;
+  return 0;
+}
+uint8_t
+netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp)
+{
+  return inp->n_my_addrs;
+}
+int
+netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val)
+{
+  inp->n_my_addrs = val;
+  return 0;
+}
+size_t
+netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp)
+{
+  return TRUNNEL_DYNARRAY_LEN(&inp->my_addrs);
+}
+
+struct netinfo_addr_st *
+netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx)
+{
+  return TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+}
+
+ const struct netinfo_addr_st *
+netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx)
+{
+  return netinfo_cell_get_my_addrs((netinfo_cell_t*)inp, idx);
+}
+int
+netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  netinfo_addr_t *oldval = TRUNNEL_DYNARRAY_GET(&inp->my_addrs, idx);
+  if (oldval && oldval != elt)
+    netinfo_addr_free(oldval);
+  return netinfo_cell_set0_my_addrs(inp, idx, elt);
+}
+int
+netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt)
+{
+  TRUNNEL_DYNARRAY_SET(&inp->my_addrs, idx, elt);
+  return 0;
+}
+int
+netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt)
+{
+#if SIZE_MAX >= UINT8_MAX
+  if (inp->my_addrs.n_ == UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  TRUNNEL_DYNARRAY_ADD(struct netinfo_addr_st *, &inp->my_addrs, elt, {});
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+
+struct netinfo_addr_st * *
+netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp)
+{
+  return inp->my_addrs.elts_;
+}
+const struct netinfo_addr_st *  const  *
+netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp)
+{
+  return (const struct netinfo_addr_st *  const  *)netinfo_cell_getarray_my_addrs((netinfo_cell_t*)inp);
+}
+int
+netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen)
+{
+  struct netinfo_addr_st * *newptr;
+#if UINT8_MAX < SIZE_MAX
+  if (newlen > UINT8_MAX)
+    goto trunnel_alloc_failed;
+#endif
+  newptr = trunnel_dynarray_setlen(&inp->my_addrs.allocated_,
+                 &inp->my_addrs.n_, inp->my_addrs.elts_, newlen,
+                 sizeof(inp->my_addrs.elts_[0]), (trunnel_free_fn_t) netinfo_addr_free,
+                 &inp->trunnel_error_code_);
+  if (newlen != 0 && newptr == NULL)
+    goto trunnel_alloc_failed;
+  inp->my_addrs.elts_ = newptr;
+  return 0;
+ trunnel_alloc_failed:
+  TRUNNEL_SET_ERROR_CODE(inp);
+  return -1;
+}
+const char *
+netinfo_cell_check(const netinfo_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  {
+    const char *msg;
+    if (NULL != (msg = netinfo_addr_check(obj->other_addr)))
+      return msg;
+  }
+  {
+    const char *msg;
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      if (NULL != (msg = netinfo_addr_check(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx))))
+        return msg;
+    }
+  }
+  if (TRUNNEL_DYNARRAY_LEN(&obj->my_addrs) != obj->n_my_addrs)
+    return "Length mismatch for my_addrs";
+  return NULL;
+}
+
+ssize_t
+netinfo_cell_encoded_len(const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != netinfo_cell_check(obj))
+     return -1;
+
+
+  /* Length of u32 timestamp */
+  result += 4;
+
+  /* Length of struct netinfo_addr other_addr */
+  result += netinfo_addr_encoded_len(obj->other_addr);
+
+  /* Length of u8 n_my_addrs */
+  result += 1;
+
+  /* Length of struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      result += netinfo_addr_encoded_len(TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+    }
+  }
+  return result;
+}
+int
+netinfo_cell_clear_errors(netinfo_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+netinfo_cell_encode(uint8_t *output, const size_t avail, const netinfo_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = netinfo_cell_encoded_len(obj);
+#endif
+
+  if (NULL != (msg = netinfo_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u32 timestamp */
+  trunnel_assert(written <= avail);
+  if (avail - written < 4)
+    goto truncated;
+  trunnel_set_uint32(ptr, trunnel_htonl(obj->timestamp));
+  written += 4; ptr += 4;
+
+  /* Encode struct netinfo_addr other_addr */
+  trunnel_assert(written <= avail);
+  result = netinfo_addr_encode(ptr, avail - written, obj->other_addr);
+  if (result < 0)
+    goto fail; /* XXXXXXX !*/
+  written += result; ptr += result;
+
+  /* Encode u8 n_my_addrs */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->n_my_addrs));
+  written += 1; ptr += 1;
+
+  /* Encode struct netinfo_addr my_addrs[n_my_addrs] */
+  {
+
+    unsigned idx;
+    for (idx = 0; idx < TRUNNEL_DYNARRAY_LEN(&obj->my_addrs); ++idx) {
+      trunnel_assert(written <= avail);
+      result = netinfo_addr_encode(ptr, avail - written, TRUNNEL_DYNARRAY_GET(&obj->my_addrs, idx));
+      if (result < 0)
+        goto fail; /* XXXXXXX !*/
+      written += result; ptr += result;
+    }
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As netinfo_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+netinfo_cell_parse_into(netinfo_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u32 timestamp */
+  CHECK_REMAINING(4, truncated);
+  obj->timestamp = trunnel_ntohl(trunnel_get_uint32(ptr));
+  remaining -= 4; ptr += 4;
+
+  /* Parse struct netinfo_addr other_addr */
+  result = netinfo_addr_parse(&obj->other_addr, ptr, remaining);
+  if (result < 0)
+    goto relay_fail;
+  trunnel_assert((size_t)result <= remaining);
+  remaining -= result; ptr += result;
+
+  /* Parse u8 n_my_addrs */
+  CHECK_REMAINING(1, truncated);
+  obj->n_my_addrs = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+
+  /* Parse struct netinfo_addr my_addrs[n_my_addrs] */
+  TRUNNEL_DYNARRAY_EXPAND(netinfo_addr_t *, &obj->my_addrs, obj->n_my_addrs, {});
+  {
+    netinfo_addr_t * elt;
+    unsigned idx;
+    for (idx = 0; idx < obj->n_my_addrs; ++idx) {
+      result = netinfo_addr_parse(&elt, ptr, remaining);
+      if (result < 0)
+        goto relay_fail;
+      trunnel_assert((size_t)result <= remaining);
+      remaining -= result; ptr += result;
+      TRUNNEL_DYNARRAY_ADD(netinfo_addr_t *, &obj->my_addrs, elt, {netinfo_addr_free(elt);});
+    }
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ relay_fail:
+  trunnel_assert(result < 0);
+  return result;
+ trunnel_alloc_failed:
+  return -1;
+}
+
+ssize_t
+netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = netinfo_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = netinfo_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    netinfo_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/netinfo.h b/src/trunnel/netinfo.h
new file mode 100644
index 0000000000..37c2ae3c2d
--- /dev/null
+++ b/src/trunnel/netinfo.h
@@ -0,0 +1,226 @@
+/* netinfo.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_NETINFO_H
+#define TRUNNEL_NETINFO_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define NETINFO_ADDR_TYPE_IPV4 4
+#define NETINFO_ADDR_TYPE_IPV6 6
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_ADDR)
+struct netinfo_addr_st {
+  uint8_t addr_type;
+  uint8_t len;
+  uint32_t addr_ipv4;
+  uint8_t addr_ipv6[16];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_addr_st netinfo_addr_t;
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_NETINFO_CELL)
+struct netinfo_cell_st {
+  uint32_t timestamp;
+  struct netinfo_addr_st *other_addr;
+  uint8_t n_my_addrs;
+  TRUNNEL_DYNARRAY_HEAD(, struct netinfo_addr_st *) my_addrs;
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct netinfo_cell_st netinfo_cell_t;
+/** Return a newly allocated netinfo_addr with all elements set to
+ * zero.
+ */
+netinfo_addr_t *netinfo_addr_new(void);
+/** Release all storage held by the netinfo_addr in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_addr_free(netinfo_addr_t *victim);
+/** Try to parse a netinfo_addr from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_addr_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_addr_parse(netinfo_addr_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_addr in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_addr_encoded_len(const netinfo_addr_t *obj);
+/** Try to encode the netinfo_addr from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_addr_encode(uint8_t *output, size_t avail, const netinfo_addr_t *input);
+/** Check whether the internal state of the netinfo_addr in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_addr_check(const netinfo_addr_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_addr_clear_errors(netinfo_addr_t *obj);
+/** Return the value of the addr_type field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint8_t netinfo_addr_get_addr_type(const netinfo_addr_t *inp);
+/** Set the value of the addr_type field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_type(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the len field of the netinfo_addr_t in 'inp'
+ */
+uint8_t netinfo_addr_get_len(const netinfo_addr_t *inp);
+/** Set the value of the len field of the netinfo_addr_t in 'inp' to
+ * 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_addr_set_len(netinfo_addr_t *inp, uint8_t val);
+/** Return the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp'
+ */
+uint32_t netinfo_addr_get_addr_ipv4(const netinfo_addr_t *inp);
+/** Set the value of the addr_ipv4 field of the netinfo_addr_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_addr_set_addr_ipv4(netinfo_addr_t *inp, uint32_t val);
+/** Return the (constant) length of the array holding the addr_ipv6
+ * field of the netinfo_addr_t in 'inp'.
+ */
+size_t netinfo_addr_getlen_addr_ipv6(const netinfo_addr_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp'.
+ */
+uint8_t netinfo_addr_get_addr_ipv6(netinfo_addr_t *inp, size_t idx);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+uint8_t netinfo_addr_getconst_addr_ipv6(const netinfo_addr_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * addr_ipv6 of the netinfo_addr_t in 'inp', so that it will hold the
+ * value 'elt'.
+ */
+int netinfo_addr_set_addr_ipv6(netinfo_addr_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the 16-element array field addr_ipv6 of 'inp'.
+ */
+uint8_t * netinfo_addr_getarray_addr_ipv6(netinfo_addr_t *inp);
+/** As netinfo_addr_get_addr_ipv6, but take and return a const pointer
+ */
+const uint8_t  * netinfo_addr_getconstarray_addr_ipv6(const netinfo_addr_t *inp);
+/** Return a newly allocated netinfo_cell with all elements set to
+ * zero.
+ */
+netinfo_cell_t *netinfo_cell_new(void);
+/** Release all storage held by the netinfo_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void netinfo_cell_free(netinfo_cell_t *victim);
+/** Try to parse a netinfo_cell from the buffer in 'input', using up
+ * to 'len_in' bytes from the input buffer. On success, return the
+ * number of bytes consumed and set *output to the newly allocated
+ * netinfo_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t netinfo_cell_parse(netinfo_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * netinfo_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t netinfo_cell_encoded_len(const netinfo_cell_t *obj);
+/** Try to encode the netinfo_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t netinfo_cell_encode(uint8_t *output, size_t avail, const netinfo_cell_t *input);
+/** Check whether the internal state of the netinfo_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *netinfo_cell_check(const netinfo_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int netinfo_cell_clear_errors(netinfo_cell_t *obj);
+/** Return the value of the timestamp field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint32_t netinfo_cell_get_timestamp(const netinfo_cell_t *inp);
+/** Set the value of the timestamp field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_timestamp(netinfo_cell_t *inp, uint32_t val);
+/** Return the value of the other_addr field of the netinfo_cell_t in
+ * 'inp'
+ */
+struct netinfo_addr_st * netinfo_cell_get_other_addr(netinfo_cell_t *inp);
+/** As netinfo_cell_get_other_addr, but take and return a const
+ * pointer
+ */
+const struct netinfo_addr_st * netinfo_cell_getconst_other_addr(const netinfo_cell_t *inp);
+/** Set the value of the other_addr field of the netinfo_cell_t in
+ * 'inp' to 'val'. Free the old value if any. Steals the referenceto
+ * 'val'.Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_set_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** As netinfo_cell_set_other_addr, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_other_addr(netinfo_cell_t *inp, struct netinfo_addr_st *val);
+/** Return the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp'
+ */
+uint8_t netinfo_cell_get_n_my_addrs(const netinfo_cell_t *inp);
+/** Set the value of the n_my_addrs field of the netinfo_cell_t in
+ * 'inp' to 'val'. Return 0 on success; return -1 and set the error
+ * code on 'inp' on failure.
+ */
+int netinfo_cell_set_n_my_addrs(netinfo_cell_t *inp, uint8_t val);
+/** Return the length of the dynamic array holding the my_addrs field
+ * of the netinfo_cell_t in 'inp'.
+ */
+size_t netinfo_cell_getlen_my_addrs(const netinfo_cell_t *inp);
+/** Return the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp'.
+ */
+struct netinfo_addr_st * netinfo_cell_get_my_addrs(netinfo_cell_t *inp, size_t idx);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+ const struct netinfo_addr_st * netinfo_cell_getconst_my_addrs(const netinfo_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the dynamic array field
+ * my_addrs of the netinfo_cell_t in 'inp', so that it will hold the
+ * value 'elt'. Free the previous value, if any.
+ */
+int netinfo_cell_set_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** As netinfo_cell_set_my_addrs, but does not free the previous
+ * value.
+ */
+int netinfo_cell_set0_my_addrs(netinfo_cell_t *inp, size_t idx, struct netinfo_addr_st * elt);
+/** Append a new element 'elt' to the dynamic array field my_addrs of
+ * the netinfo_cell_t in 'inp'.
+ */
+int netinfo_cell_add_my_addrs(netinfo_cell_t *inp, struct netinfo_addr_st * elt);
+/** Return a pointer to the variable-length array field my_addrs of
+ * 'inp'.
+ */
+struct netinfo_addr_st * * netinfo_cell_getarray_my_addrs(netinfo_cell_t *inp);
+/** As netinfo_cell_get_my_addrs, but take and return a const pointer
+ */
+const struct netinfo_addr_st *  const  * netinfo_cell_getconstarray_my_addrs(const netinfo_cell_t *inp);
+/** Change the length of the variable-length array field my_addrs of
+ * 'inp' to 'newlen'.Fill extra elements with NULL; free removed
+ * elements. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int netinfo_cell_setlen_my_addrs(netinfo_cell_t *inp, size_t newlen);
+
+
+#endif
diff --git a/src/trunnel/netinfo.trunnel b/src/trunnel/netinfo.trunnel
new file mode 100644
index 0000000000..2c4b7a7591
--- /dev/null
+++ b/src/trunnel/netinfo.trunnel
@@ -0,0 +1,24 @@
+// Warning: make sure these values are consistent with RESOLVED_TYPE_*
+// constants in Tor code and numbers in Section 6.4 of tor-spec.txt.
+
+const NETINFO_ADDR_TYPE_IPV4 = 4;
+const NETINFO_ADDR_TYPE_IPV6 = 6;
+
+struct netinfo_addr {
+  u8 addr_type;
+  u8 len;
+  union addr[addr_type] with length len {
+    NETINFO_ADDR_TYPE_IPV4: u32 ipv4;
+    NETINFO_ADDR_TYPE_IPV6: u8 ipv6[16];
+    default: ignore;
+  };
+
+}
+
+struct netinfo_cell {
+  u32 timestamp;
+  struct netinfo_addr other_addr;
+  u8 n_my_addrs;
+  struct netinfo_addr my_addrs[n_my_addrs];
+}
+
diff --git a/src/trunnel/pwbox.c b/src/trunnel/pwbox.c
index c356515d36..c159a5e687 100644
--- a/src/trunnel/pwbox.c
+++ b/src/trunnel/pwbox.c
@@ -1,4 +1,4 @@
-/* pwbox.c -- generated by Trunnel v1.5.2.
+/* pwbox.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/pwbox.h b/src/trunnel/pwbox.h
index a9a421408a..36d595f4ef 100644
--- a/src/trunnel/pwbox.h
+++ b/src/trunnel/pwbox.h
@@ -1,4 +1,4 @@
-/* pwbox.h -- generated by Trunnel v1.5.2.
+/* pwbox.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/sendme_cell.c b/src/trunnel/sendme_cell.c
new file mode 100644
index 0000000000..b9f8fe967f
--- /dev/null
+++ b/src/trunnel/sendme_cell.c
@@ -0,0 +1,347 @@
+/* sendme_cell.c -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#include <stdlib.h>
+#include "trunnel-impl.h"
+
+#include "sendme_cell.h"
+
+#define TRUNNEL_SET_ERROR_CODE(obj) \
+  do {                              \
+    (obj)->trunnel_error_code_ = 1; \
+  } while (0)
+
+#if defined(__COVERITY__) || defined(__clang_analyzer__)
+/* If we're running a static analysis tool, we don't want it to complain
+ * that some of our remaining-bytes checks are dead-code. */
+int sendmecell_deadcode_dummy__ = 0;
+#define OR_DEADCODE_DUMMY || sendmecell_deadcode_dummy__
+#else
+#define OR_DEADCODE_DUMMY
+#endif
+
+#define CHECK_REMAINING(nbytes, label)                           \
+  do {                                                           \
+    if (remaining < (nbytes) OR_DEADCODE_DUMMY) {                \
+      goto label;                                                \
+    }                                                            \
+  } while (0)
+
+sendme_cell_t *
+sendme_cell_new(void)
+{
+  sendme_cell_t *val = trunnel_calloc(1, sizeof(sendme_cell_t));
+  if (NULL == val)
+    return NULL;
+  return val;
+}
+
+/** Release all storage held inside 'obj', but do not free 'obj'.
+ */
+static void
+sendme_cell_clear(sendme_cell_t *obj)
+{
+  (void) obj;
+}
+
+void
+sendme_cell_free(sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return;
+  sendme_cell_clear(obj);
+  trunnel_memwipe(obj, sizeof(sendme_cell_t));
+  trunnel_free_(obj);
+}
+
+uint8_t
+sendme_cell_get_version(const sendme_cell_t *inp)
+{
+  return inp->version;
+}
+int
+sendme_cell_set_version(sendme_cell_t *inp, uint8_t val)
+{
+  if (! ((val == 0 || val == 1))) {
+     TRUNNEL_SET_ERROR_CODE(inp);
+     return -1;
+  }
+  inp->version = val;
+  return 0;
+}
+uint16_t
+sendme_cell_get_data_len(const sendme_cell_t *inp)
+{
+  return inp->data_len;
+}
+int
+sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val)
+{
+  inp->data_len = val;
+  return 0;
+}
+size_t
+sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp)
+{
+  (void)inp;  return TRUNNEL_SENDME_V1_DIGEST_LEN;
+}
+
+uint8_t
+sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  return inp->data_v1_digest[idx];
+}
+
+uint8_t
+sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx)
+{
+  return sendme_cell_get_data_v1_digest((sendme_cell_t*)inp, idx);
+}
+int
+sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt)
+{
+  trunnel_assert(idx < TRUNNEL_SENDME_V1_DIGEST_LEN);
+  inp->data_v1_digest[idx] = elt;
+  return 0;
+}
+
+uint8_t *
+sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp)
+{
+  return inp->data_v1_digest;
+}
+const uint8_t  *
+sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp)
+{
+  return (const uint8_t  *)sendme_cell_getarray_data_v1_digest((sendme_cell_t*)inp);
+}
+const char *
+sendme_cell_check(const sendme_cell_t *obj)
+{
+  if (obj == NULL)
+    return "Object was NULL";
+  if (obj->trunnel_error_code_)
+    return "A set function failed on this object";
+  if (! (obj->version == 0 || obj->version == 1))
+    return "Integer out of bounds";
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+      break;
+
+    default:
+        return "Bad tag for union";
+      break;
+  }
+  return NULL;
+}
+
+ssize_t
+sendme_cell_encoded_len(const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+
+  if (NULL != sendme_cell_check(obj))
+     return -1;
+
+
+  /* Length of u8 version IN [0, 1] */
+  result += 1;
+
+  /* Length of u16 data_len */
+  result += 2;
+  switch (obj->version) {
+
+    case 0:
+      break;
+
+    case 1:
+
+      /* Length of u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+      result += TRUNNEL_SENDME_V1_DIGEST_LEN;
+      break;
+
+    default:
+      trunnel_assert(0);
+      break;
+  }
+  return result;
+}
+int
+sendme_cell_clear_errors(sendme_cell_t *obj)
+{
+  int r = obj->trunnel_error_code_;
+  obj->trunnel_error_code_ = 0;
+  return r;
+}
+ssize_t
+sendme_cell_encode(uint8_t *output, const size_t avail, const sendme_cell_t *obj)
+{
+  ssize_t result = 0;
+  size_t written = 0;
+  uint8_t *ptr = output;
+  const char *msg;
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  const ssize_t encoded_len = sendme_cell_encoded_len(obj);
+#endif
+
+  uint8_t *backptr_data_len = NULL;
+
+  if (NULL != (msg = sendme_cell_check(obj)))
+    goto check_failed;
+
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  trunnel_assert(encoded_len >= 0);
+#endif
+
+  /* Encode u8 version IN [0, 1] */
+  trunnel_assert(written <= avail);
+  if (avail - written < 1)
+    goto truncated;
+  trunnel_set_uint8(ptr, (obj->version));
+  written += 1; ptr += 1;
+
+  /* Encode u16 data_len */
+  backptr_data_len = ptr;
+  trunnel_assert(written <= avail);
+  if (avail - written < 2)
+    goto truncated;
+  trunnel_set_uint16(ptr, trunnel_htons(obj->data_len));
+  written += 2; ptr += 2;
+  {
+    size_t written_before_union = written;
+
+    /* Encode union data[version] */
+    trunnel_assert(written <= avail);
+    switch (obj->version) {
+
+      case 0:
+        break;
+
+      case 1:
+
+        /* Encode u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        trunnel_assert(written <= avail);
+        if (avail - written < TRUNNEL_SENDME_V1_DIGEST_LEN)
+          goto truncated;
+        memcpy(ptr, obj->data_v1_digest, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        written += TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        trunnel_assert(0);
+        break;
+    }
+    /* Write the length field back to data_len */
+    trunnel_assert(written >= written_before_union);
+#if UINT16_MAX < SIZE_MAX
+    if (written - written_before_union > UINT16_MAX)
+      goto check_failed;
+#endif
+    trunnel_set_uint16(backptr_data_len, trunnel_htons(written - written_before_union));
+  }
+
+
+  trunnel_assert(ptr == output + written);
+#ifdef TRUNNEL_CHECK_ENCODED_LEN
+  {
+    trunnel_assert(encoded_len >= 0);
+    trunnel_assert((size_t)encoded_len == written);
+  }
+
+#endif
+
+  return written;
+
+ truncated:
+  result = -2;
+  goto fail;
+ check_failed:
+  (void)msg;
+  result = -1;
+  goto fail;
+ fail:
+  trunnel_assert(result < 0);
+  return result;
+}
+
+/** As sendme_cell_parse(), but do not allocate the output object.
+ */
+static ssize_t
+sendme_cell_parse_into(sendme_cell_t *obj, const uint8_t *input, const size_t len_in)
+{
+  const uint8_t *ptr = input;
+  size_t remaining = len_in;
+  ssize_t result = 0;
+  (void)result;
+
+  /* Parse u8 version IN [0, 1] */
+  CHECK_REMAINING(1, truncated);
+  obj->version = (trunnel_get_uint8(ptr));
+  remaining -= 1; ptr += 1;
+  if (! (obj->version == 0 || obj->version == 1))
+    goto fail;
+
+  /* Parse u16 data_len */
+  CHECK_REMAINING(2, truncated);
+  obj->data_len = trunnel_ntohs(trunnel_get_uint16(ptr));
+  remaining -= 2; ptr += 2;
+  {
+    size_t remaining_after;
+    CHECK_REMAINING(obj->data_len, truncated);
+    remaining_after = remaining - obj->data_len;
+    remaining = obj->data_len;
+
+    /* Parse union data[version] */
+    switch (obj->version) {
+
+      case 0:
+        /* Skip to end of union */
+        ptr += remaining; remaining = 0;
+        break;
+
+      case 1:
+
+        /* Parse u8 data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN] */
+        CHECK_REMAINING(TRUNNEL_SENDME_V1_DIGEST_LEN, fail);
+        memcpy(obj->data_v1_digest, ptr, TRUNNEL_SENDME_V1_DIGEST_LEN);
+        remaining -= TRUNNEL_SENDME_V1_DIGEST_LEN; ptr += TRUNNEL_SENDME_V1_DIGEST_LEN;
+        break;
+
+      default:
+        goto fail;
+        break;
+    }
+    if (remaining != 0)
+      goto fail;
+    remaining = remaining_after;
+  }
+  trunnel_assert(ptr + remaining == input + len_in);
+  return len_in - remaining;
+
+ truncated:
+  return -2;
+ fail:
+  result = -1;
+  return result;
+}
+
+ssize_t
+sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in)
+{
+  ssize_t result;
+  *output = sendme_cell_new();
+  if (NULL == *output)
+    return -1;
+  result = sendme_cell_parse_into(*output, input, len_in);
+  if (result < 0) {
+    sendme_cell_free(*output);
+    *output = NULL;
+  }
+  return result;
+}
diff --git a/src/trunnel/sendme_cell.h b/src/trunnel/sendme_cell.h
new file mode 100644
index 0000000000..45efb9f10d
--- /dev/null
+++ b/src/trunnel/sendme_cell.h
@@ -0,0 +1,101 @@
+/* sendme_cell.h -- generated by Trunnel v1.5.3.
+ * https://gitweb.torproject.org/trunnel.git
+ * You probably shouldn't edit this file.
+ */
+#ifndef TRUNNEL_SENDME_CELL_H
+#define TRUNNEL_SENDME_CELL_H
+
+#include <stdint.h>
+#include "trunnel.h"
+
+#define TRUNNEL_SENDME_V1_DIGEST_LEN 20
+#if !defined(TRUNNEL_OPAQUE) && !defined(TRUNNEL_OPAQUE_SENDME_CELL)
+struct sendme_cell_st {
+  uint8_t version;
+  uint16_t data_len;
+  uint8_t data_v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  uint8_t trunnel_error_code_;
+};
+#endif
+typedef struct sendme_cell_st sendme_cell_t;
+/** Return a newly allocated sendme_cell with all elements set to
+ * zero.
+ */
+sendme_cell_t *sendme_cell_new(void);
+/** Release all storage held by the sendme_cell in 'victim'. (Do
+ * nothing if 'victim' is NULL.)
+ */
+void sendme_cell_free(sendme_cell_t *victim);
+/** Try to parse a sendme_cell from the buffer in 'input', using up to
+ * 'len_in' bytes from the input buffer. On success, return the number
+ * of bytes consumed and set *output to the newly allocated
+ * sendme_cell_t. On failure, return -2 if the input appears
+ * truncated, and -1 if the input is otherwise invalid.
+ */
+ssize_t sendme_cell_parse(sendme_cell_t **output, const uint8_t *input, const size_t len_in);
+/** Return the number of bytes we expect to need to encode the
+ * sendme_cell in 'obj'. On failure, return a negative value. Note
+ * that this value may be an overestimate, and can even be an
+ * underestimate for certain unencodeable objects.
+ */
+ssize_t sendme_cell_encoded_len(const sendme_cell_t *obj);
+/** Try to encode the sendme_cell from 'input' into the buffer at
+ * 'output', using up to 'avail' bytes of the output buffer. On
+ * success, return the number of bytes used. On failure, return -2 if
+ * the buffer was not long enough, and -1 if the input was invalid.
+ */
+ssize_t sendme_cell_encode(uint8_t *output, size_t avail, const sendme_cell_t *input);
+/** Check whether the internal state of the sendme_cell in 'obj' is
+ * consistent. Return NULL if it is, and a short message if it is not.
+ */
+const char *sendme_cell_check(const sendme_cell_t *obj);
+/** Clear any errors that were set on the object 'obj' by its setter
+ * functions. Return true iff errors were cleared.
+ */
+int sendme_cell_clear_errors(sendme_cell_t *obj);
+/** Return the value of the version field of the sendme_cell_t in
+ * 'inp'
+ */
+uint8_t sendme_cell_get_version(const sendme_cell_t *inp);
+/** Set the value of the version field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_version(sendme_cell_t *inp, uint8_t val);
+/** Return the value of the data_len field of the sendme_cell_t in
+ * 'inp'
+ */
+uint16_t sendme_cell_get_data_len(const sendme_cell_t *inp);
+/** Set the value of the data_len field of the sendme_cell_t in 'inp'
+ * to 'val'. Return 0 on success; return -1 and set the error code on
+ * 'inp' on failure.
+ */
+int sendme_cell_set_data_len(sendme_cell_t *inp, uint16_t val);
+/** Return the (constant) length of the array holding the
+ * data_v1_digest field of the sendme_cell_t in 'inp'.
+ */
+size_t sendme_cell_getlen_data_v1_digest(const sendme_cell_t *inp);
+/** Return the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp'.
+ */
+uint8_t sendme_cell_get_data_v1_digest(sendme_cell_t *inp, size_t idx);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+uint8_t sendme_cell_getconst_data_v1_digest(const sendme_cell_t *inp, size_t idx);
+/** Change the element at position 'idx' of the fixed array field
+ * data_v1_digest of the sendme_cell_t in 'inp', so that it will hold
+ * the value 'elt'.
+ */
+int sendme_cell_set_data_v1_digest(sendme_cell_t *inp, size_t idx, uint8_t elt);
+/** Return a pointer to the TRUNNEL_SENDME_V1_DIGEST_LEN-element array
+ * field data_v1_digest of 'inp'.
+ */
+uint8_t * sendme_cell_getarray_data_v1_digest(sendme_cell_t *inp);
+/** As sendme_cell_get_data_v1_digest, but take and return a const
+ * pointer
+ */
+const uint8_t  * sendme_cell_getconstarray_data_v1_digest(const sendme_cell_t *inp);
+
+
+#endif
diff --git a/src/trunnel/sendme_cell.trunnel b/src/trunnel/sendme_cell.trunnel
new file mode 100644
index 0000000000..300963e679
--- /dev/null
+++ b/src/trunnel/sendme_cell.trunnel
@@ -0,0 +1,19 @@
+/* This file contains the SENDME cell definition. */
+
+/* v1 digest length in bytes. */
+const TRUNNEL_SENDME_V1_DIGEST_LEN = 20;
+
+/* SENDME cell declaration. */
+struct sendme_cell {
+  /* Version field. */
+  u8 version IN [0x00, 0x01];
+
+  /* Length of data contained in this cell. */
+  u16 data_len;
+
+  /* The data content depends on the version. */
+  union data[version] with length data_len {
+    0x00: ignore;
+    0x01: u8 v1_digest[TRUNNEL_SENDME_V1_DIGEST_LEN];
+  };
+}
diff --git a/src/trunnel/socks5.c b/src/trunnel/socks5.c
index 9e5f6fcfed..f32862e353 100644
--- a/src/trunnel/socks5.c
+++ b/src/trunnel/socks5.c
@@ -1,4 +1,4 @@
-/* socks5.c -- generated by Trunnel v1.5.2.
+/* socks5.c -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
@@ -1694,7 +1694,6 @@ socks4_server_reply_new(void)
   socks4_server_reply_t *val = trunnel_calloc(1, sizeof(socks4_server_reply_t));
   if (NULL == val)
     return NULL;
-  val->version = 4;
   return val;
 }
 
@@ -1724,7 +1723,7 @@ socks4_server_reply_get_version(const socks4_server_reply_t *inp)
 int
 socks4_server_reply_set_version(socks4_server_reply_t *inp, uint8_t val)
 {
-  if (! ((val == 4))) {
+  if (! ((val == 0 || val == 4))) {
      TRUNNEL_SET_ERROR_CODE(inp);
      return -1;
   }
@@ -1771,7 +1770,7 @@ socks4_server_reply_check(const socks4_server_reply_t *obj)
     return "Object was NULL";
   if (obj->trunnel_error_code_)
     return "A set function failed on this object";
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     return "Integer out of bounds";
   return NULL;
 }
@@ -1785,7 +1784,7 @@ socks4_server_reply_encoded_len(const socks4_server_reply_t *obj)
      return -1;
 
 
-  /* Length of u8 version IN [4] */
+  /* Length of u8 version IN [0, 4] */
   result += 1;
 
   /* Length of u8 status */
@@ -1823,7 +1822,7 @@ socks4_server_reply_encode(uint8_t *output, const size_t avail, const socks4_ser
   trunnel_assert(encoded_len >= 0);
 #endif
 
-  /* Encode u8 version IN [4] */
+  /* Encode u8 version IN [0, 4] */
   trunnel_assert(written <= avail);
   if (avail - written < 1)
     goto truncated;
@@ -1886,11 +1885,11 @@ socks4_server_reply_parse_into(socks4_server_reply_t *obj, const uint8_t *input,
   ssize_t result = 0;
   (void)result;
 
-  /* Parse u8 version IN [4] */
+  /* Parse u8 version IN [0, 4] */
   CHECK_REMAINING(1, truncated);
   obj->version = (trunnel_get_uint8(ptr));
   remaining -= 1; ptr += 1;
-  if (! (obj->version == 4))
+  if (! (obj->version == 0 || obj->version == 4))
     goto fail;
 
   /* Parse u8 status */
diff --git a/src/trunnel/socks5.h b/src/trunnel/socks5.h
index d3bea152e7..23ac64faba 100644
--- a/src/trunnel/socks5.h
+++ b/src/trunnel/socks5.h
@@ -1,4 +1,4 @@
-/* socks5.h -- generated by Trunnel v1.5.2.
+/* socks5.h -- generated by Trunnel v1.5.3.
  * https://gitweb.torproject.org/trunnel.git
  * You probably shouldn't edit this file.
  */
diff --git a/src/trunnel/socks5.trunnel b/src/trunnel/socks5.trunnel
index b86ec03b9d..17fa2ed996 100644
--- a/src/trunnel/socks5.trunnel
+++ b/src/trunnel/socks5.trunnel
@@ -86,7 +86,7 @@ struct socks4_client_request {
 }
 
 struct socks4_server_reply {
-   u8 version IN [4];
+   u8 version IN [0,4];
    u8 status;
    u16 port;
    u32 addr;
diff --git a/src/trunnel/trunnel-local.h b/src/trunnel/trunnel-local.h
index c4118fce4c..80da371560 100644
--- a/src/trunnel/trunnel-local.h
+++ b/src/trunnel/trunnel-local.h
@@ -14,5 +14,6 @@
 #define trunnel_reallocarray tor_reallocarray
 #define trunnel_assert tor_assert
 #define trunnel_memwipe(mem, len) memwipe((mem), 0, (len))
+#define trunnel_abort tor_abort_
 
 #endif
diff --git a/src/win32/orconfig.h b/src/win32/orconfig.h
index c97889ce46..b26aac0fa4 100644
--- a/src/win32/orconfig.h
+++ b/src/win32/orconfig.h
@@ -102,7 +102,6 @@
 /* Define to 1 if you have the <sys/socket.h> header file. */
 #undef HAVE_SYS_SOCKET_H
 
-
 /* Define to 1 if you have the <sys/stat.h> header file. */
 #define HAVE_SYS_STAT_H
 
@@ -218,9 +217,7 @@
 #define USING_TWOS_COMPLEMENT
 
 /* Version number of package */
-#define VERSION "0.3.5.15-dev"
-
-
+#define VERSION "0.4.4.9"
 
 #define HAVE_STRUCT_SOCKADDR_IN6
 #define HAVE_STRUCT_IN6_ADDR